{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is intended to make a faster version of this notebook: (**63x**)\n* https://www.kaggle.com/byfone/h-m-trending-products-weekly/notebook \n\nThe formula above is already well explained in this notebook  : \n* https://www.kaggle.com/lichtlab/0-0226-byfone-chris-combination-approach/notebook \n\nPlease upvote those notebook notebook, and this notebook too if you find this helpful :D \n\nThis notebook has reduce **63x** the processing time from **2 hours to 2 minutes**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nfrom datetime import datetime, timedelta\nimport gc\n\nimport cudf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-06T03:44:52.178569Z","iopub.execute_input":"2022-03-06T03:44:52.178864Z","iopub.status.idle":"2022-03-06T03:44:55.78635Z","shell.execute_reply.started":"2022-03-06T03:44:52.178784Z","shell.execute_reply":"2022-03-06T03:44:55.78557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Read the transactions data","metadata":{}},{"cell_type":"markdown","source":"Changes: \n* read_csv using cudf\n* article_id types from string to int32 \n* reduce memory on customer_id ","metadata":{}},{"cell_type":"code","source":"N = 12","metadata":{"execution":{"iopub.status.busy":"2022-03-06T03:44:55.790449Z","iopub.execute_input":"2022-03-06T03:44:55.791051Z","iopub.status.idle":"2022-03-06T03:44:55.79838Z","shell.execute_reply.started":"2022-03-06T03:44:55.791019Z","shell.execute_reply":"2022-03-06T03:44:55.797652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df  = cudf.read_csv('../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv',\n                            usecols= ['t_dat', 'customer_id', 'article_id'], \n                            dtype={'article_id': 'int32', 't_dat': 'string', 'customer_id': 'string'})\ndf ['customer_id'] = df ['customer_id'].str[-16:].str.hex_to_int().astype('int64')\n\ndf['t_dat'] = cudf.to_datetime(df['t_dat'])\nlast_ts = df['t_dat'].max()","metadata":{"execution":{"iopub.status.busy":"2022-03-06T03:44:55.80349Z","iopub.execute_input":"2022-03-06T03:44:55.804044Z","iopub.status.idle":"2022-03-06T03:45:40.765348Z","shell.execute_reply.started":"2022-03-06T03:44:55.804006Z","shell.execute_reply":"2022-03-06T03:45:40.764564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add the last day of billing week","metadata":{}},{"cell_type":"markdown","source":"Note: \n* In the original code, `ldbw` is the next nearest (or current) Tuesday from `t_dat` (since 2020-09-22 is Tuesday)\n    \nChanges: \n* Use `pandas.Series.dt.dayofweek` and `pd.TimedeltaIndex` \n* Do not use `last_ts`\n\nSteps: \n1. Get the day of week, Monday = 0, Tuesday = 1 and so on \n2. Truncated t_dat into the Tuesday of t_dat week, saved as ldbw \n3. for t_dat Wednesday until Sunday add 7 days to get next Tuesday, do nothing for t_dat Monday and Tuesday\n\nTime reduction: \n* Before: 55 min 47 sec\n* After: 8 sec","metadata":{}},{"cell_type":"code","source":"%%time\ntmp = df[['t_dat']].copy().to_pandas()\ntmp['dow'] = tmp['t_dat'].dt.dayofweek\ntmp['ldbw'] = tmp['t_dat'] - pd.TimedeltaIndex(tmp['dow'] - 1, unit='D')\ntmp.loc[tmp['dow'] >=2 , 'ldbw'] = tmp.loc[tmp['dow'] >=2 , 'ldbw'] + pd.TimedeltaIndex(np.ones(len(tmp.loc[tmp['dow'] >=2])) * 7, unit='D')\n\ndf['ldbw'] = tmp['ldbw'].values","metadata":{"execution":{"iopub.status.busy":"2022-03-06T03:48:44.319723Z","iopub.execute_input":"2022-03-06T03:48:44.320001Z","iopub.status.idle":"2022-03-06T03:48:52.136963Z","shell.execute_reply.started":"2022-03-06T03:48:44.319969Z","shell.execute_reply":"2022-03-06T03:48:52.136201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Count the number of transactions per week ","metadata":{"execution":{"iopub.status.busy":"2022-03-06T01:38:34.458819Z","iopub.execute_input":"2022-03-06T01:38:34.459249Z","iopub.status.idle":"2022-03-06T01:38:34.5231Z","shell.execute_reply.started":"2022-03-06T01:38:34.45921Z","shell.execute_reply":"2022-03-06T01:38:34.522386Z"}}},{"cell_type":"markdown","source":"Changes: \n* Do reset index after groupby \n* df.join to df.merge ","metadata":{}},{"cell_type":"code","source":"weekly_sales = df.drop('customer_id', axis=1).groupby(['ldbw', 'article_id']).count().reset_index()\nweekly_sales = weekly_sales.rename(columns={'t_dat': 'count'})","metadata":{"execution":{"iopub.status.busy":"2022-03-06T03:48:52.13847Z","iopub.execute_input":"2022-03-06T03:48:52.138878Z","iopub.status.idle":"2022-03-06T03:48:52.231024Z","shell.execute_reply.started":"2022-03-06T03:48:52.138833Z","shell.execute_reply":"2022-03-06T03:48:52.23037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.merge(weekly_sales, on=['ldbw', 'article_id'], how = 'left')","metadata":{"execution":{"iopub.status.busy":"2022-03-06T03:48:52.232024Z","iopub.execute_input":"2022-03-06T03:48:52.232348Z","iopub.status.idle":"2022-03-06T03:48:52.322184Z","shell.execute_reply.started":"2022-03-06T03:48:52.232314Z","shell.execute_reply":"2022-03-06T03:48:52.321548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Let's assume that in the target week sales will be similar to the last week of the training data","metadata":{}},{"cell_type":"code","source":"weekly_sales = weekly_sales.reset_index().set_index('article_id')\n\ndf = df.merge(\n    weekly_sales.loc[weekly_sales['ldbw']==last_ts, ['count']],\n    on='article_id', suffixes=(\"\", \"_targ\"))\n\ndf['count_targ'].fillna(0, inplace=True)\ndel weekly_sales","metadata":{"execution":{"iopub.status.busy":"2022-03-06T03:48:52.323948Z","iopub.execute_input":"2022-03-06T03:48:52.324262Z","iopub.status.idle":"2022-03-06T03:48:52.412549Z","shell.execute_reply.started":"2022-03-06T03:48:52.324228Z","shell.execute_reply":"2022-03-06T03:48:52.411899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Calculate sales rate adjusted for changes in product popularity ","metadata":{}},{"cell_type":"code","source":"df['quotient'] = df['count_targ'] / df['count']","metadata":{"execution":{"iopub.status.busy":"2022-03-06T03:48:52.413619Z","iopub.execute_input":"2022-03-06T03:48:52.413934Z","iopub.status.idle":"2022-03-06T03:48:52.423043Z","shell.execute_reply.started":"2022-03-06T03:48:52.413897Z","shell.execute_reply":"2022-03-06T03:48:52.422218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Take supposedly popular products","metadata":{}},{"cell_type":"code","source":"target_sales = df.drop('customer_id', axis=1).groupby('article_id')['quotient'].sum()\ngeneral_pred = target_sales.nlargest(N).index.to_pandas().tolist()\ngeneral_pred = ['0' + str(article_id) for article_id in general_pred]\ngeneral_pred_str =  ' '.join(general_pred)\ndel target_sales","metadata":{"execution":{"iopub.status.busy":"2022-03-06T03:48:52.424464Z","iopub.execute_input":"2022-03-06T03:48:52.424956Z","iopub.status.idle":"2022-03-06T03:48:52.480994Z","shell.execute_reply.started":"2022-03-06T03:48:52.424886Z","shell.execute_reply":"2022-03-06T03:48:52.480371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"general_pred # Exactly same with the original notebook! ","metadata":{"execution":{"iopub.status.busy":"2022-03-06T03:48:52.482866Z","iopub.execute_input":"2022-03-06T03:48:52.483259Z","iopub.status.idle":"2022-03-06T03:48:52.489346Z","shell.execute_reply.started":"2022-03-06T03:48:52.483225Z","shell.execute_reply":"2022-03-06T03:48:52.488487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"general_pred_str","metadata":{"execution":{"iopub.status.busy":"2022-03-06T03:48:52.490818Z","iopub.execute_input":"2022-03-06T03:48:52.491114Z","iopub.status.idle":"2022-03-06T03:48:52.499814Z","shell.execute_reply.started":"2022-03-06T03:48:52.491079Z","shell.execute_reply":"2022-03-06T03:48:52.499131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fill in purchase dictionary","metadata":{}},{"cell_type":"markdown","source":"Changes: \n* avoid looping, use pandas series instead \n* Use dataframe instead of dictionary (faster on submission part)\n\nTime reduction:\n* Before: 39 min 09 sec\n* After: 28 sec\n","metadata":{}},{"cell_type":"code","source":"%%time\npurchase_dict = {}\n\ntmp = df.copy().to_pandas()\ntmp['x'] = ((last_ts - tmp['t_dat']) / np.timedelta64(1, 'D')).astype(int)\ntmp['dummy_1'] = 1 \ntmp['x'] = tmp[[\"x\", \"dummy_1\"]].max(axis=1)\n\na, b, c, d = 2.5e4, 1.5e5, 2e-1, 1e3\ntmp['y'] = a / np.sqrt(tmp['x']) + b * np.exp(-c*tmp['x']) - d\n\ntmp['dummy_0'] = 0 \ntmp['y'] = tmp[[\"y\", \"dummy_0\"]].max(axis=1)\ntmp['value'] = tmp['quotient'] * tmp['y'] \n\ntmp = tmp.groupby(['customer_id', 'article_id']).agg({'value': 'sum'})\ntmp = tmp.reset_index()\n\ntmp = tmp.loc[tmp['value'] > 100]\ntmp['rank'] = tmp.groupby(\"customer_id\")[\"value\"].rank(\"dense\", ascending=False)\ntmp = tmp.loc[tmp['rank'] <= 12]\n\n# for customer_id in tmp['customer_id'].unique():\n#     purchase_dict[customer_id] = {} \n\n# for customer_id, article_id, value in zip(tmp['customer_id'], tmp['article_id'], tmp['value']):\n#     purchase_dict[customer_id][article_id] = value\n\npurchase_df = tmp.sort_values(['customer_id', 'value'], ascending = False).reset_index(drop = True)\npurchase_df['prediction'] = '0' + purchase_df['article_id'].astype(str) + ' '\npurchase_df = purchase_df.groupby('customer_id').agg({'prediction': sum}).reset_index()\npurchase_df['prediction'] = purchase_df['prediction'].str.strip()\npurchase_df = cudf.DataFrame(purchase_df)","metadata":{"execution":{"iopub.status.busy":"2022-03-06T03:48:58.502817Z","iopub.execute_input":"2022-03-06T03:48:58.503341Z","iopub.status.idle":"2022-03-06T03:49:26.474923Z","shell.execute_reply.started":"2022-03-06T03:48:58.503299Z","shell.execute_reply":"2022-03-06T03:49:26.474206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make a submission","metadata":{}},{"cell_type":"markdown","source":"Changes: \n* read_csv using cudf \n* avoid looping, use pandas series instead \n* Use dataframe instead of dictionary \n* Concatenate prediction string (tricks found in https://www.kaggle.com/cdeotte/recommend-items-purchased-together-0-021) \n\nTime reduction:\n* Before: 15 min 44 sec\n* After: 15 sec\n","metadata":{}},{"cell_type":"code","source":"%%time\nsub  = cudf.read_csv('../input/h-and-m-personalized-fashion-recommendations/sample_submission.csv',\n                            usecols= ['customer_id'], \n                            dtype={'customer_id': 'string'})\n\nsub['customer_id2'] = sub['customer_id'].str[-16:].str.hex_to_int().astype('int64')\n\nsub = sub.merge(purchase_df, left_on = 'customer_id2', right_on = 'customer_id', how = 'left',\n               suffixes = ('', '_ignored'))\n\nsub = sub.to_pandas()\nsub['prediction'] = sub['prediction'].fillna(general_pred_str)\nsub['prediction'] = sub['prediction'] + ' ' +  general_pred_str\nsub['prediction'] = sub['prediction'].str.strip()\nsub['prediction'] = sub['prediction'].str[:131]\nsub = sub[['customer_id', 'prediction']]\nsub.to_csv(f'submission.csv',index=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-06T03:05:27.489311Z","iopub.execute_input":"2022-03-06T03:05:27.489594Z","iopub.status.idle":"2022-03-06T03:05:42.88965Z","shell.execute_reply.started":"2022-03-06T03:05:27.489559Z","shell.execute_reply":"2022-03-06T03:05:42.88881Z"},"trusted":true},"execution_count":null,"outputs":[]}]}