{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview\n\nThis notebook is written based on the notebook created by LichtLab. \nIn this notebook, I attempt to improve the calculation process.\n\nAlso see Byfone and CHris' notebook for the underlying algorithm.\n\nThanks to \n- LichtLab : https://www.kaggle.com/lichtlab/0-0226-byfone-chris-combination-approach\n- Byfone: https://www.kaggle.com/byfone/h-m-trending-products-weekly\n- Chris : https://www.kaggle.com/cdeotte/recommend-items-purchased-together-0-021  \n","metadata":{}},{"cell_type":"markdown","source":"# About Improvements\n\n- Accelerate date type conversion process in transaction data sets\n- The process of generating candidate predictions got about 5 times faster by applying a native sorting process instead of `pandas.Series`.\n- To improve processing speed, we select the necessary columns from the data frame and transform them into lists.  \n","metadata":{}},{"cell_type":"code","source":"import gc\nimport numpy as np\nimport os\nimport pandas as pd\nfrom math import sqrt\nfrom pathlib import Path\nfrom tqdm import tqdm\n\ntqdm.pandas()\n\n# size of prediction candidates\nN = 12\n\ndf_trans = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv',\n                       dtype={'article_id': str})\ndf_trans['t_dat'] = pd.to_datetime(df_trans['t_dat'], format='%Y-%m-%d')","metadata":{"execution":{"iopub.status.busy":"2022-03-06T16:11:16.409366Z","iopub.execute_input":"2022-03-06T16:11:16.409701Z","iopub.status.idle":"2022-03-06T16:12:25.639010Z","shell.execute_reply.started":"2022-03-06T16:11:16.409617Z","shell.execute_reply":"2022-03-06T16:12:25.638347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculate distance from latest date\n\nCalculate the distance after rounding the date to the nearest week.  \nHowever, it seems to take a lot of time when processing with `apply`.\nWe will try to speed up the process by calculating by columns.\n\nFor rounding for the `datetime` type, see the following entry.  \nhttps://pandas.pydata.org/docs/reference/api/pandas.DatetimeIndex.floor.html\n","metadata":{}},{"cell_type":"code","source":"# Step1\ndf = df_trans[['t_dat', 'customer_id', 'article_id']].copy()\nlast_ts = df['t_dat'].max()\n\n# df['ldbw'] = df['t_dat'].progress_apply(lambda d: last_ts - (last_ts - d).floor('7D'))\ndf['offset_dat'] = (last_ts - df['t_dat']).dt.floor('7D')\ndf['ldbw'] = last_ts - df['offset_dat']","metadata":{"execution":{"iopub.status.busy":"2022-03-06T16:13:08.818363Z","iopub.execute_input":"2022-03-06T16:13:08.818664Z","iopub.status.idle":"2022-03-06T16:13:12.820822Z","shell.execute_reply.started":"2022-03-06T16:13:08.818626Z","shell.execute_reply":"2022-03-06T16:13:12.820231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weekly_sales = df.drop(['customer_id', 'offset_dat'], axis=1).groupby(['ldbw', 'article_id']).count()\nweekly_sales = weekly_sales.rename(columns={'t_dat': 'count'})\ndf = df.join(weekly_sales, on=['ldbw', 'article_id'])\nweekly_sales = weekly_sales.reset_index().set_index('article_id')\nlast_day = last_ts.strftime('%Y-%m-%d')","metadata":{"execution":{"iopub.status.busy":"2022-03-06T16:13:59.796910Z","iopub.execute_input":"2022-03-06T16:13:59.797256Z","iopub.status.idle":"2022-03-06T16:14:20.266177Z","shell.execute_reply.started":"2022-03-06T16:13:59.797219Z","shell.execute_reply":"2022-03-06T16:14:20.265379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.join(\n    weekly_sales.loc[weekly_sales['ldbw']==last_day, ['count']],\n    on='article_id', rsuffix=\"_targ\")\n\ndf['count_targ'].fillna(0, inplace=True)\ndel weekly_sales, df_trans\ngc.collect()\ndf['quotient'] = df['count_targ'] / df['count']\n","metadata":{"execution":{"iopub.status.busy":"2022-03-06T16:14:24.312146Z","iopub.execute_input":"2022-03-06T16:14:24.312656Z","iopub.status.idle":"2022-03-06T16:14:34.626643Z","shell.execute_reply.started":"2022-03-06T16:14:24.312624Z","shell.execute_reply":"2022-03-06T16:14:34.625732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate purchase history dictionary\n\nTo improve processing speed, we select the necessary columns from the data frame and transform them into lists.  \nThen we store the list of values for each column in dictonary `calc_buffer`.  \nBy referring to the list, we can expect to improve processing speed considerably.","metadata":{}},{"cell_type":"code","source":"purchase_dict = {}\n\n# Processed by list to achieve faster speeds\ncols_to_list = ['customer_id', 'article_id', 't_dat', 'quotient']\ncalc_buffer = {_c: df[_c].to_list() for _c in cols_to_list}","metadata":{"execution":{"iopub.status.busy":"2022-03-06T16:15:02.048593Z","iopub.execute_input":"2022-03-06T16:15:02.048846Z","iopub.status.idle":"2022-03-06T16:16:08.518869Z","shell.execute_reply.started":"2022-03-06T16:15:02.048819Z","shell.execute_reply":"2022-03-06T16:16:08.517885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in tqdm(range(0, len(calc_buffer['customer_id']))):\n    cust_id = calc_buffer['customer_id'][i]\n    art_id = calc_buffer['article_id'][i]\n    t_dat = calc_buffer['t_dat'][i]\n\n    if cust_id not in purchase_dict:\n        purchase_dict[cust_id] = {}\n\n    if art_id not in purchase_dict[cust_id]:\n        purchase_dict[cust_id][art_id] = 0\n    \n    x = max(1, (last_ts - t_dat).days)\n\n    a, b, c, d = 2.5e4, 1.5e5, 2e-1, 1e3\n    y = a / np.sqrt(x) + b * np.exp(-c*x) - d\n\n    value = calc_buffer['quotient'][i] * max(0, y)\n    purchase_dict[cust_id][art_id] += value\n\ndel calc_buffer\ngc.collect()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-06T16:16:32.658977Z","iopub.execute_input":"2022-03-06T16:16:32.659284Z","iopub.status.idle":"2022-03-06T16:23:39.809738Z","shell.execute_reply.started":"2022-03-06T16:16:32.659247Z","shell.execute_reply":"2022-03-06T16:23:39.808790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_sales = df.drop('customer_id', axis=1).groupby('article_id')['quotient'].sum()\ngeneral_pred = target_sales.nlargest(N).index.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-03-06T16:30:29.538660Z","iopub.execute_input":"2022-03-06T16:30:29.539067Z","iopub.status.idle":"2022-03-06T16:30:38.296253Z","shell.execute_reply.started":"2022-03-06T16:30:29.539029Z","shell.execute_reply":"2022-03-06T16:30:38.295581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"general_pred","metadata":{"execution":{"iopub.status.busy":"2022-03-06T16:30:55.167329Z","iopub.execute_input":"2022-03-06T16:30:55.167973Z","iopub.status.idle":"2022-03-06T16:30:55.175881Z","shell.execute_reply.started":"2022-03-06T16:30:55.167921Z","shell.execute_reply":"2022-03-06T16:30:55.174867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Step2 & Step3\npairs = np.load('../input/hmitempairs/pairs_cudf.npy',allow_pickle=True).item()\nsub = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-03-06T16:31:04.389305Z","iopub.execute_input":"2022-03-06T16:31:04.389793Z","iopub.status.idle":"2022-03-06T16:31:09.200217Z","shell.execute_reply.started":"2022-03-06T16:31:04.389759Z","shell.execute_reply":"2022-03-06T16:31:09.199471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_list = []\nfor cust_id in tqdm(sub['customer_id']):\n    # in case of the customer who has purchase history\n    if cust_id in purchase_dict:\n        # get purchase history\n        #series = pd.Series(purchase_dict[cust_id])\n        #series = series[series > 0]\n        purchased = sorted(purchase_dict[cust_id].items(), key=lambda x:x[1], reverse=True)\n        # Get up to 12 cases in order of likelihood\n        #l = series.nlargest(N).index.tolist()\n        l = [_[0] for _ in purchased if _[1] > 0]\n        tmp_l = l.copy()\n        for elm in tmp_l:\n            # If the number of recommendation candidates is less than 12, \n            # add products to the recommendation candidates for possible simultaneous purchase.\n            if len(l) < N and int(elm) in pairs.keys():\n                itm = pairs[int(elm)]\n                l.append('0' + str(itm))\n        if len(l) < N:\n            # If the 12 recommended candidate slots are not filled, \n            # pick up from the general forecast candidates to fill the slots\n            l = l + general_pred[:(N-len(l))]\n    else:\n        # If no purchase history is available, apply general prediction candidates\n        l = general_pred\n    pred_list.append(' '.join(l))\n\nsub['prediction'] = pred_list\nsub.to_csv(f'submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-06T16:31:22.652123Z","iopub.execute_input":"2022-03-06T16:31:22.652416Z","iopub.status.idle":"2022-03-06T16:31:59.949157Z","shell.execute_reply.started":"2022-03-06T16:31:22.652387Z","shell.execute_reply":"2022-03-06T16:31:59.948203Z"},"trusted":true},"execution_count":null,"outputs":[]}]}