{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport gc\n\nfrom collections import defaultdict","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-19T09:56:51.580462Z","iopub.execute_input":"2022-04-19T09:56:51.581088Z","iopub.status.idle":"2022-04-19T09:56:51.603749Z","shell.execute_reply.started":"2022-04-19T09:56:51.580964Z","shell.execute_reply":"2022-04-19T09:56:51.60283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_df = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\", dtype={'article_id': str})\n","metadata":{"execution":{"iopub.status.busy":"2022-04-19T09:56:56.488736Z","iopub.execute_input":"2022-04-19T09:56:56.489005Z","iopub.status.idle":"2022-04-19T09:56:57.68119Z","shell.execute_reply.started":"2022-04-19T09:56:56.488975Z","shell.execute_reply":"2022-04-19T09:56:57.680396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntransaction_df = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\", \n                             dtype={'article_id': str},\n                             usecols=['t_dat', 'customer_id', 'article_id'])\n\ntransaction_df = transaction_df.groupby(['customer_id', 't_dat'], as_index=False)[['article_id']].agg(list)\n#transaction_df = transaction_df[transaction_df.t_dat >= '2019-09-01']\ntransaction_df['num_articles'] = transaction_df.article_id.apply(lambda x: len(set(x)))\ntransaction_df = transaction_df[transaction_df.num_articles > 1]\ntransaction_df = transaction_df[transaction_df.num_articles<18]\ntransaction_df['article_id'] = transaction_df['article_id'].apply(lambda x: list(set(x)))\n\ntransaction_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T09:56:57.684052Z","iopub.execute_input":"2022-04-19T09:56:57.68438Z","iopub.status.idle":"2022-04-19T10:00:31.455367Z","shell.execute_reply.started":"2022-04-19T09:56:57.684319Z","shell.execute_reply":"2022-04-19T10:00:31.454328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transaction_df.num_articles.describe()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:00:31.45656Z","iopub.execute_input":"2022-04-19T10:00:31.456837Z","iopub.status.idle":"2022-04-19T10:00:31.557289Z","shell.execute_reply.started":"2022-04-19T10:00:31.456805Z","shell.execute_reply":"2022-04-19T10:00:31.556447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transaction_df.num_articles.quantile(0.99)","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:00:31.558941Z","iopub.execute_input":"2022-04-19T10:00:31.559362Z","iopub.status.idle":"2022-04-19T10:00:31.603502Z","shell.execute_reply.started":"2022-04-19T10:00:31.559316Z","shell.execute_reply":"2022-04-19T10:00:31.602687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"number of transactions:\", len(transaction_df))","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:00:31.604693Z","iopub.execute_input":"2022-04-19T10:00:31.604903Z","iopub.status.idle":"2022-04-19T10:00:31.609488Z","shell.execute_reply.started":"2022-04-19T10:00:31.604868Z","shell.execute_reply":"2022-04-19T10:00:31.608478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_map=defaultdict(int)\nfor it, article_ids in enumerate(transaction_df.article_id.values):\n    num_articles = len(article_ids)\n    for article_id in article_ids:\n        item_map[article_id] += 1\n        \nitem_df = pd.DataFrame.from_dict({\n    'item': item_map.keys(),\n    'freq': item_map.values()\n})\nitem_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:00:31.611223Z","iopub.execute_input":"2022-04-19T10:00:31.611798Z","iopub.status.idle":"2022-04-19T10:00:40.143113Z","shell.execute_reply.started":"2022-04-19T10:00:31.611754Z","shell.execute_reply":"2022-04-19T10:00:40.142321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_df = item_df[ item_df.freq > 20]\ncandidate_items = set(item_df.item.values)","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:03:07.103277Z","iopub.execute_input":"2022-04-19T10:03:07.103821Z","iopub.status.idle":"2022-04-19T10:03:07.119661Z","shell.execute_reply.started":"2022-04-19T10:03:07.103782Z","shell.execute_reply":"2022-04-19T10:03:07.118808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npair_map={}\nfor it, article_ids in enumerate(transaction_df.article_id.values):\n    if it%1000000 == 0:\n        print(it)\n    num_articles = len(article_ids)\n    for i in range(num_articles):\n        item1 = article_ids[i]\n        if item1 not in candidate_items:\n            continue\n            \n        for j in range(i+1, num_articles):\n            item2 = article_ids[j]\n            if item2 not in candidate_items:\n                continue\n                \n            if item1 not in pair_map:\n                pair_map[item1] = {}\n            if item2 not in pair_map:\n                pair_map[item2] = {}\n            \n            if item2 not in pair_map[item1]:\n                pair_map[item1][item2] = 0\n            if item1 not in pair_map[item2]:\n                pair_map[item2][item1] = 0\n            \n            pair_map[item1][item2] += 1\n            pair_map[item2][item1] += 1","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:04:18.894071Z","iopub.execute_input":"2022-04-19T10:04:18.894405Z","iopub.status.idle":"2022-04-19T10:05:39.077254Z","shell.execute_reply.started":"2022-04-19T10:04:18.894364Z","shell.execute_reply":"2022-04-19T10:05:39.07618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item1 = []\nitem2 = []\nfreq  = []\n\nfor i1 in pair_map.keys():\n    for i2 in pair_map[i1].keys():\n        v = pair_map[i1][i2]\n        if v <= 20:\n            continue\n        item1.append(i1)\n        item2.append(i2)\n        freq.append(v)\n\npair_df = pd.DataFrame.from_dict({ 'item1': item1, 'item2': item2, 'joint_freq': freq})\npair_df['item_freq1'] = pair_df.item1.apply(lambda k: item_map[k])\npair_df['item_freq2'] = pair_df.item2.apply(lambda k: item_map[k])\npair_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:05:39.080736Z","iopub.execute_input":"2022-04-19T10:05:39.08102Z","iopub.status.idle":"2022-04-19T10:07:16.098725Z","shell.execute_reply.started":"2022-04-19T10:05:39.080987Z","shell.execute_reply":"2022-04-19T10:07:16.098143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pair_df = pair_df[pair_df.joint_freq>20]\npair_df['confidence'] = pair_df.joint_freq.div(pair_df['item_freq1'])\npair_df = pair_df.sort_values(['item1', 'confidence'], ascending=[True, False])\n\npair_df = pair_df.groupby('item1').head(10)\npair_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:17:06.720591Z","iopub.execute_input":"2022-04-19T10:17:06.720851Z","iopub.status.idle":"2022-04-19T10:17:06.830925Z","shell.execute_reply.started":"2022-04-19T10:17:06.720823Z","shell.execute_reply":"2022-04-19T10:17:06.830073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"number of pairs:\", len(pair_df)//2)","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:17:09.474966Z","iopub.execute_input":"2022-04-19T10:17:09.475249Z","iopub.status.idle":"2022-04-19T10:17:09.480935Z","shell.execute_reply.started":"2022-04-19T10:17:09.475215Z","shell.execute_reply":"2022-04-19T10:17:09.480042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pair_df.confidence.describe()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:17:16.945759Z","iopub.execute_input":"2022-04-19T10:17:16.946375Z","iopub.status.idle":"2022-04-19T10:17:16.959547Z","shell.execute_reply.started":"2022-04-19T10:17:16.946316Z","shell.execute_reply":"2022-04-19T10:17:16.958413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pair_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:17:30.062217Z","iopub.execute_input":"2022-04-19T10:17:30.062754Z","iopub.status.idle":"2022-04-19T10:17:30.073228Z","shell.execute_reply.started":"2022-04-19T10:17:30.06271Z","shell.execute_reply":"2022-04-19T10:17:30.072668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:19:05.241733Z","iopub.execute_input":"2022-04-19T10:19:05.242251Z","iopub.status.idle":"2022-04-19T10:19:08.179106Z","shell.execute_reply.started":"2022-04-19T10:19:05.242208Z","shell.execute_reply":"2022-04-19T10:19:08.178386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# predict based on the association","metadata":{}},{"cell_type":"code","source":"transaction_df = transaction_df.groupby('customer_id', as_index=False)[['article_id']].agg(list)\ntransaction_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:22:11.272883Z","iopub.execute_input":"2022-04-19T10:22:11.273159Z","iopub.status.idle":"2022-04-19T10:22:22.041346Z","shell.execute_reply.started":"2022-04-19T10:22:11.273132Z","shell.execute_reply":"2022-04-19T10:22:22.04046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"popular_articles = \"0706016001 0720125001 0706016002 0372860001 0759871002 0610776002 0751471001 0372860002 0673677002 0706016003 0464297007 0562245046\"\n","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:24:31.256724Z","iopub.execute_input":"2022-04-19T10:24:31.257207Z","iopub.status.idle":"2022-04-19T10:24:31.26038Z","shell.execute_reply.started":"2022-04-19T10:24:31.257141Z","shell.execute_reply":"2022-04-19T10:24:31.259865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_recommended_items(articles):\n    df = pair_df[pair_df.item1.isin(articles)]\n    df.groupby('item2', as_index=False)[['joint_freq']].sum().sort_values('joint_freq', ascending=False)\n    df = df.head(12)\n    items = df.item2.values\n    items = ' '.join(items)\n    return items","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:31:46.215817Z","iopub.execute_input":"2022-04-19T10:31:46.216152Z","iopub.status.idle":"2022-04-19T10:31:46.22197Z","shell.execute_reply.started":"2022-04-19T10:31:46.216117Z","shell.execute_reply":"2022-04-19T10:31:46.22133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_candidate_items = set(pair_df.item1.values)\nprint(len(final_candidate_items))","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:42:53.683255Z","iopub.execute_input":"2022-04-19T10:42:53.68412Z","iopub.status.idle":"2022-04-19T10:42:53.694178Z","shell.execute_reply.started":"2022-04-19T10:42:53.684078Z","shell.execute_reply":"2022-04-19T10:42:53.693295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds=[]\nfor it, row in transaction_df.iterrows():\n    customer_id = row.customer_id\n    articles = []\n    for article_lst in row.article_id:\n        articles += article_lst\n    articles = set(articles)\n    cur_articles = final_candidate_items.intersection(articles)\n    \n    if len(cur_articles) == 0:\n        continue\n        \n    pred_items = get_recommended_items(cur_articles)\n    preds.append({\n        'customer_id': customer_id,\n        'prediction': pred_items\n    })\n    if it%100000==0:\n        print(it)\npred_df = pd.DataFrame.from_dict(preds)\npred_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:44:03.128431Z","iopub.execute_input":"2022-04-19T10:44:03.128989Z","iopub.status.idle":"2022-04-19T10:44:03.156624Z","shell.execute_reply.started":"2022-04-19T10:44:03.128948Z","shell.execute_reply":"2022-04-19T10:44:03.155745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsub_df = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/sample_submission.csv\", usecols=['customer_id'])\nsub_df = sub_df.merge(pred_df, how='left')\nsub_df.prediction.fillna(popular_articles, inplace=True)\nsub_df.to_csv(\"submission.csv\", index=False)\n\nsub_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T10:44:09.778098Z","iopub.execute_input":"2022-04-19T10:44:09.77857Z","iopub.status.idle":"2022-04-19T10:44:25.704898Z","shell.execute_reply.started":"2022-04-19T10:44:09.778524Z","shell.execute_reply":"2022-04-19T10:44:25.704322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}