{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"As mentioned in this [thread](https://www.kaggle.com/c/h-and-m-personalized-fashion-recommendations/discussion/307288), one way to approach this problem is to generate candidates with different models and then rank them using item features and user features. This notebook provides basic `item features` that you can use using ranking models, for example [this](https://www.kaggle.com/c/h-and-m-personalized-fashion-recommendations/discussion/309220).\n\nMy previous notebook with [user features](https://www.kaggle.com/alexvishnevskiy/ranking-user-features/edit/run/88745460).","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom tqdm import tqdm\nfrom abc import ABC, abstractmethod\nfrom typing import List, Dict, Any, Union","metadata":{"execution":{"iopub.status.busy":"2022-02-26T13:26:43.524603Z","iopub.execute_input":"2022-02-26T13:26:43.525145Z","iopub.status.idle":"2022-02-26T13:26:43.530292Z","shell.execute_reply.started":"2022-02-26T13:26:43.525097Z","shell.execute_reply":"2022-02-26T13:26:43.529525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = Path('../input/h-and-m-personalized-fashion-recommendations')\ntransactions_train = pd.read_csv(data_path/'transactions_train.csv')\ntransactions_train['t_dat'] = pd.to_datetime(transactions_train['t_dat'])\ncustomers_df = pd.read_csv(data_path/'customers.csv')\narticles_df = pd.read_csv(data_path/'articles.csv')","metadata":{"execution":{"iopub.status.busy":"2022-02-26T13:16:10.645466Z","iopub.execute_input":"2022-02-26T13:16:10.645799Z","iopub.status.idle":"2022-02-26T13:17:38.982751Z","shell.execute_reply.started":"2022-02-26T13:16:10.645765Z","shell.execute_reply":"2022-02-26T13:17:38.981349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Basically, I am using abstraction like this below. Every class should have `get` method and should output pandas DataFrame. Then, collect all features using another class `UserFeaturesCollector`.","metadata":{}},{"cell_type":"code","source":"class ItemFeatures(ABC):\n    @abstractmethod\n    def get(self, *args, **kwargs) -> pd.DataFrame:\n        \"\"\"\n        article_id -> features\n        \"\"\"\n        pass","metadata":{"execution":{"iopub.status.busy":"2022-02-26T13:19:20.072256Z","iopub.execute_input":"2022-02-26T13:19:20.073039Z","iopub.status.idle":"2022-02-26T13:19:20.077311Z","shell.execute_reply.started":"2022-02-26T13:19:20.072992Z","shell.execute_reply":"2022-02-26T13:19:20.076705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CategoryTransform(ItemFeatures):\n    \"\"\"\n    factorize all articles columns\n    \"\"\"\n    def __init__(self, articles_df: pd.DataFrame):\n        self.articles_df = articles_df\n\n    def get(self):\n        self.__feature_columns = list(filter(lambda x: 'name' in x, self.articles_df.columns))[1:]\n        filtered_articles = self.articles_df[self.__feature_columns]\n        filtered_articles = filtered_articles.apply(lambda x: pd.factorize(x)[0])\n        filtered_articles['article_id'] = self.articles_df['article_id']\n\n        features = filtered_articles.set_index('article_id').astype('int8')\n        return features\n\n    def get_columns(self):\n        return self.__feature_columns","metadata":{"execution":{"iopub.status.busy":"2022-02-26T14:33:18.69364Z","iopub.execute_input":"2022-02-26T14:33:18.693953Z","iopub.status.idle":"2022-02-26T14:33:18.702682Z","shell.execute_reply.started":"2022-02-26T14:33:18.69392Z","shell.execute_reply":"2022-02-26T14:33:18.701477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AggrTransform(ItemFeatures):\n    \"\"\"\n    aggregation transactions features : mean, max and etc...\n    \"\"\"\n    def __init__(self, articles_df: pd.DataFrame, transactions_df: pd.DataFrame):\n        self.articles_df = articles_df\n        self.transactions_df = transactions_df\n\n    def get(self):\n        stats = self._get_stats()\n        return stats\n\n    def _get_stats(self):\n        transactions_more = self.transactions_df.merge(self.articles_df, on = ('article_id'))\n        grouped = (\n            transactions_more.\n            groupby('article_id')\n        )\n\n        counts = (\n            grouped['article_id']\n            .count()\n            .to_frame()\n            .rename(columns = {'article_id': 'count'})\n            .astype('int16')\n            .reset_index()\n            .set_index('article_id')\n        )\n        sums = (\n            grouped['price']\n            .sum()\n            .to_frame()\n            .astype('float32')\n            .rename(columns = {\n                'price': 'sum_price'\n            })\n        )\n        means = (\n            grouped['price']\n            .mean()\n            .to_frame()\n            .astype('float32')\n            .rename(columns = {\n                'price': 'mean_price'\n            })\n        )\n        mins = (\n            grouped['price']\n            .min()\n            .to_frame()\n            .astype('float32')\n            .rename(columns = {\n               'price': 'min_price' \n            })\n        )\n        maxs = (\n            grouped['price']\n            .max()\n            .to_frame()\n            .astype('float32')\n            .rename(columns = {\n                'price': 'max_price'\n            })\n        )\n        \n        output_df = (\n            counts\n            .merge(sums, on = ('article_id'))\n            .merge(means, on = ('article_id'))\n            .merge(mins, on = ('article_id'))\n            .merge(maxs, on = ('article_id'))\n        )\n        return output_df","metadata":{"execution":{"iopub.status.busy":"2022-02-26T14:34:21.471817Z","iopub.execute_input":"2022-02-26T14:34:21.472471Z","iopub.status.idle":"2022-02-26T14:34:21.485879Z","shell.execute_reply.started":"2022-02-26T14:34:21.472429Z","shell.execute_reply":"2022-02-26T14:34:21.4849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TopTransforms(ItemFeatures):\n    \"\"\"\n    whether category appears in top categories\n    \"\"\"\n    def __init__(self, articles_df: pd.DataFrame, topk = 3):\n        self.articles_df = articles_df\n        self.topk = topk\n    \n    def get(self):\n        name_cols = list(filter(lambda x: 'name' in x, self.articles_df.columns))  \n        \n        value_counts = self._get_value_counts(name_cols)\n        value_counts = {\n            f'{k}_{self.topk}': self.articles_df[k].isin(v).astype('int8') for k, v in value_counts.items()\n        }\n        \n        output_df = self.articles_df.assign(**value_counts)\n        output_df = output_df[['article_id'] + list(value_counts.keys())].set_index('article_id')\n        return output_df\n        \n    def _get_value_counts(self, name_cols: List[str]):\n        value_counts = self.articles_df[name_cols].apply(pd.Series.value_counts)\n        get_index = lambda x: value_counts.sort_values(x, ascending = False)[x][:self.topk].index  \n        value_counts = dict(zip(name_cols, map(lambda x: get_index(x), name_cols)))\n        return value_counts","metadata":{"execution":{"iopub.status.busy":"2022-02-26T14:34:53.009813Z","iopub.execute_input":"2022-02-26T14:34:53.010128Z","iopub.status.idle":"2022-02-26T14:34:53.020187Z","shell.execute_reply.started":"2022-02-26T14:34:53.010096Z","shell.execute_reply":"2022-02-26T14:34:53.019565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ItemFeaturesCollector:\n    @staticmethod\n    def collect(features: Union[List[ItemFeatures], List[str]], **kwargs) -> pd.DataFrame:\n        output_df = None\n\n        for feature in tqdm(features):\n            if isinstance(feature, ItemFeatures):\n                feature_out = feature.get(**kwargs)\n            if isinstance(feature, str):\n                try:\n                    feature_out = pd.read_csv(feature)\n                except:\n                    feature_out = pd.read_parquet(feature)\n\n            if output_df is None:\n                output_df = feature_out\n            else:\n                output_df = output_df.merge(feature_out, on = ('article_id'))\n        return output_df","metadata":{"execution":{"iopub.status.busy":"2022-02-26T13:31:50.217194Z","iopub.execute_input":"2022-02-26T13:31:50.217934Z","iopub.status.idle":"2022-02-26T13:31:50.225223Z","shell.execute_reply.started":"2022-02-26T13:31:50.217892Z","shell.execute_reply":"2022-02-26T13:31:50.224326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For simplicity let's take only first 100k transactions.","metadata":{}},{"cell_type":"code","source":"item_features = ItemFeaturesCollector.collect([\n    CategoryTransform(articles_df),\n    AggrTransform(articles_df, transactions_train.iloc[:100_000]),\n    TopTransforms(articles_df)\n])","metadata":{"execution":{"iopub.status.busy":"2022-02-26T14:25:03.143689Z","iopub.execute_input":"2022-02-26T14:25:03.144638Z","iopub.status.idle":"2022-02-26T14:25:04.385765Z","shell.execute_reply.started":"2022-02-26T14:25:03.144585Z","shell.execute_reply":"2022-02-26T14:25:04.385171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_features.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-26T14:25:25.710915Z","iopub.execute_input":"2022-02-26T14:25:25.711198Z","iopub.status.idle":"2022-02-26T14:25:25.733908Z","shell.execute_reply.started":"2022-02-26T14:25:25.71117Z","shell.execute_reply":"2022-02-26T14:25:25.73321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_features.to_parquet('item_features.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-02-26T14:25:40.132197Z","iopub.execute_input":"2022-02-26T14:25:40.133113Z","iopub.status.idle":"2022-02-26T14:25:40.235017Z","shell.execute_reply.started":"2022-02-26T14:25:40.133069Z","shell.execute_reply":"2022-02-26T14:25:40.234375Z"},"trusted":true},"execution_count":null,"outputs":[]}]}