Libraries
","metadata":{}},{"cell_type":"code","source":"import pandas as pd, numpy as np\n\nimport glob\n\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom datetime import datetime\n\nfrom tqdm import tqdm\n\ntype_labels = {'clicks':1, 'carts':2, 'orders':3}","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:09:32.576803Z","iopub.execute_input":"2024-06-08T03:09:32.577196Z","iopub.status.idle":"2024-06-08T03:09:33.694191Z","shell.execute_reply.started":"2024-06-08T03:09:32.577166Z","shell.execute_reply":"2024-06-08T03:09:33.693170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##Data Loading
","metadata":{}},{"cell_type":"code","source":"def load(which): \n dfs = []\n\n test_files = glob.glob('/kaggle/input/otto-chunk-data-inparquet-format/'+which+'_parquet/*')\n \n for e, chunk_file in enumerate(test_files):\n chunk = pd.read_parquet(chunk_file)\n chunk.ts = (chunk.ts/1000).astype('int32')\n chunk['type'] = chunk['type'].map(type_labels).astype('int8')\n dfs.append(chunk)\n \n return pd.concat(dfs).reset_index(drop=True) #.astype({\"ts\": \"datetime64[ms]\"})","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:09:33.696141Z","iopub.execute_input":"2024-06-08T03:09:33.696605Z","iopub.status.idle":"2024-06-08T03:09:33.703370Z","shell.execute_reply.started":"2024-06-08T03:09:33.696574Z","shell.execute_reply":"2024-06-08T03:09:33.702289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_df = load('test')\ntrain_df = load('train')","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:09:33.709612Z","iopub.execute_input":"2024-06-08T03:09:33.709977Z","iopub.status.idle":"2024-06-08T03:10:34.315324Z","shell.execute_reply.started":"2024-06-08T03:09:33.709937Z","shell.execute_reply":"2024-06-08T03:10:34.314174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:10:34.317114Z","iopub.execute_input":"2024-06-08T03:10:34.317857Z","iopub.status.idle":"2024-06-08T03:10:34.338130Z","shell.execute_reply.started":"2024-06-08T03:10:34.317805Z","shell.execute_reply":"2024-06-08T03:10:34.336747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##Generation of small dataset
","metadata":{}},{"cell_type":"markdown","source":"## To reduce runtime, only consider data from the last 4 days as train_df.","metadata":{}},{"cell_type":"code","source":"datetime.strptime(\"2022-08-25 00:00:00\", \"%Y-%m-%d %H:%M:%S\").timestamp()","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:10:34.339868Z","iopub.execute_input":"2024-06-08T03:10:34.340201Z","iopub.status.idle":"2024-06-08T03:10:34.347287Z","shell.execute_reply.started":"2024-06-08T03:10:34.340172Z","shell.execute_reply":"2024-06-08T03:10:34.346185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum(train_df['ts']>=1661385600)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:10:34.348776Z","iopub.execute_input":"2024-06-08T03:10:34.349141Z","iopub.status.idle":"2024-06-08T03:10:53.124265Z","shell.execute_reply.started":"2024-06-08T03:10:34.349102Z","shell.execute_reply":"2024-06-08T03:10:53.123195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df[train_df['ts']>=1661385600].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:10:53.125789Z","iopub.execute_input":"2024-06-08T03:10:53.126176Z","iopub.status.idle":"2024-06-08T03:10:54.836896Z","shell.execute_reply.started":"2024-06-08T03:10:53.126147Z","shell.execute_reply":"2024-06-08T03:10:54.835522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['datetime'] = train_df['ts'].apply(lambda x: datetime.fromtimestamp(x))","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:10:54.838614Z","iopub.execute_input":"2024-06-08T03:10:54.839243Z","iopub.status.idle":"2024-06-08T03:11:34.462297Z","shell.execute_reply.started":"2024-06-08T03:10:54.839201Z","shell.execute_reply":"2024-06-08T03:11:34.461088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:11:34.467894Z","iopub.execute_input":"2024-06-08T03:11:34.468495Z","iopub.status.idle":"2024-06-08T03:11:34.477839Z","shell.execute_reply.started":"2024-06-08T03:11:34.468441Z","shell.execute_reply":"2024-06-08T03:11:34.475947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['session'].nunique()","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:11:34.479804Z","iopub.execute_input":"2024-06-08T03:11:34.480370Z","iopub.status.idle":"2024-06-08T03:11:35.279974Z","shell.execute_reply.started":"2024-06-08T03:11:34.480321Z","shell.execute_reply":"2024-06-08T03:11:35.278754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['aid'].nunique()","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:11:35.281365Z","iopub.execute_input":"2024-06-08T03:11:35.281727Z","iopub.status.idle":"2024-06-08T03:11:36.012740Z","shell.execute_reply.started":"2024-06-08T03:11:35.281696Z","shell.execute_reply":"2024-06-08T03:11:36.011154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:11:36.014116Z","iopub.execute_input":"2024-06-08T03:11:36.014470Z","iopub.status.idle":"2024-06-08T03:11:36.029837Z","shell.execute_reply.started":"2024-06-08T03:11:36.014432Z","shell.execute_reply":"2024-06-08T03:11:36.028464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##Division of training set and validation set
","metadata":{}},{"cell_type":"markdown","source":"## Choose the last day as the validation set.","metadata":{}},{"cell_type":"code","source":"train_df['datetime'].dt.day.unique()","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:11:36.031749Z","iopub.execute_input":"2024-06-08T03:11:36.032248Z","iopub.status.idle":"2024-06-08T03:11:37.063728Z","shell.execute_reply.started":"2024-06-08T03:11:36.032205Z","shell.execute_reply":"2024-06-08T03:11:37.062468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['day'] = train_df['datetime'].dt.day","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:11:37.065062Z","iopub.execute_input":"2024-06-08T03:11:37.065506Z","iopub.status.idle":"2024-06-08T03:11:37.932760Z","shell.execute_reply.started":"2024-06-08T03:11:37.065464Z","shell.execute_reply":"2024-06-08T03:11:37.931147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_val_offline = train_df[train_df['day']==28].reset_index(drop=True)\ndf_train_offline = train_df[train_df['day']<28].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:11:37.934627Z","iopub.execute_input":"2024-06-08T03:11:37.935380Z","iopub.status.idle":"2024-06-08T03:11:39.943267Z","shell.execute_reply.started":"2024-06-08T03:11:37.935334Z","shell.execute_reply":"2024-06-08T03:11:39.942157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##Recommendation based on item popularity
","metadata":{}},{"cell_type":"markdown","source":"## Analyze item popularity based on different types of behavior.","metadata":{}},{"cell_type":"code","source":"aid_pop_sta = df_train_offline.groupby(['aid','type'],as_index=False)['session'].count()","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:11:39.944900Z","iopub.execute_input":"2024-06-08T03:11:39.945366Z","iopub.status.idle":"2024-06-08T03:11:43.425241Z","shell.execute_reply.started":"2024-06-08T03:11:39.945323Z","shell.execute_reply":"2024-06-08T03:11:43.423894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aid_pop_sta.columns = ['aid','type','pop']","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:11:43.427142Z","iopub.execute_input":"2024-06-08T03:11:43.427527Z","iopub.status.idle":"2024-06-08T03:11:43.433431Z","shell.execute_reply.started":"2024-06-08T03:11:43.427492Z","shell.execute_reply":"2024-06-08T03:11:43.432079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aid_pop_sta","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:11:43.435032Z","iopub.execute_input":"2024-06-08T03:11:43.435436Z","iopub.status.idle":"2024-06-08T03:11:43.458656Z","shell.execute_reply.started":"2024-06-08T03:11:43.435377Z","shell.execute_reply":"2024-06-08T03:11:43.457239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Obtain the top 200 most popular items.","metadata":{}},{"cell_type":"code","source":"#aid_pop_sta = aid_pop_sta.sort_values(['type','pop'],ascending=False).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:18:57.408154Z","iopub.execute_input":"2024-06-08T03:18:57.408607Z","iopub.status.idle":"2024-06-08T03:18:57.414305Z","shell.execute_reply.started":"2024-06-08T03:18:57.408571Z","shell.execute_reply":"2024-06-08T03:18:57.412933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aid_pop_sta = aid_pop_sta.sort_values('pop', ascending=False).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:26:17.263513Z","iopub.execute_input":"2024-06-08T03:26:17.263973Z","iopub.status.idle":"2024-06-08T03:26:17.390473Z","shell.execute_reply.started":"2024-06-08T03:26:17.263942Z","shell.execute_reply":"2024-06-08T03:26:17.389261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aid_pop_sta","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:26:19.486494Z","iopub.execute_input":"2024-06-08T03:26:19.486905Z","iopub.status.idle":"2024-06-08T03:26:19.500435Z","shell.execute_reply.started":"2024-06-08T03:26:19.486872Z","shell.execute_reply":"2024-06-08T03:26:19.499280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top200_aid_click = list(aid_pop_sta.loc[aid_pop_sta['type']==1,'aid'][:200])\ntop200_aid_cart = list(aid_pop_sta.loc[aid_pop_sta['type']==2,'aid'][:200])\ntop200_aid_order = list(aid_pop_sta.loc[aid_pop_sta['type']==3,'aid'][:200])","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:26:30.587344Z","iopub.execute_input":"2024-06-08T03:26:30.587775Z","iopub.status.idle":"2024-06-08T03:26:30.620993Z","shell.execute_reply.started":"2024-06-08T03:26:30.587744Z","shell.execute_reply":"2024-06-08T03:26:30.619607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Obtain the historical behavior list of the target user. The users in the validation set are the target users.","metadata":{}},{"cell_type":"code","source":"target_users = list( df_val_offline['session'].unique() )","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:19:43.449312Z","iopub.execute_input":"2024-06-08T03:19:43.449726Z","iopub.status.idle":"2024-06-08T03:19:43.743743Z","shell.execute_reply.started":"2024-06-08T03:19:43.449687Z","shell.execute_reply":"2024-06-08T03:19:43.742503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(target_users)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:19:45.729162Z","iopub.execute_input":"2024-06-08T03:19:45.729569Z","iopub.status.idle":"2024-06-08T03:19:45.736124Z","shell.execute_reply.started":"2024-06-08T03:19:45.729534Z","shell.execute_reply":"2024-06-08T03:19:45.734879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_his = df_train_offline.loc[df_train_offline['session'].isin(target_users)].groupby(['session','type'],as_index=False)['aid'].agg(list)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:11:43.972333Z","iopub.execute_input":"2024-06-08T03:11:43.972656Z","iopub.status.idle":"2024-06-08T03:12:03.322530Z","shell.execute_reply.started":"2024-06-08T03:11:43.972628Z","shell.execute_reply":"2024-06-08T03:12:03.321271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_his","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:12:03.323931Z","iopub.execute_input":"2024-06-08T03:12:03.324273Z","iopub.status.idle":"2024-06-08T03:12:03.341936Z","shell.execute_reply.started":"2024-06-08T03:12:03.324243Z","shell.execute_reply":"2024-06-08T03:12:03.340830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_his['session'].nunique()","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:12:03.343464Z","iopub.execute_input":"2024-06-08T03:12:03.344428Z","iopub.status.idle":"2024-06-08T03:12:03.391879Z","shell.execute_reply.started":"2024-06-08T03:12:03.344351Z","shell.execute_reply":"2024-06-08T03:12:03.390751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Only 558344 out of 1350502 target users have historical behavior.**","metadata":{}},{"cell_type":"code","source":"his_click_dict = dict(zip(session_his.loc[session_his['type']==1,'session'],session_his.loc[session_his['type']==1,'aid']))\nhis_cart_dict = dict(zip(session_his.loc[session_his['type']==2,'session'],session_his.loc[session_his['type']==2,'aid']))\nhis_order_dict = dict(zip(session_his.loc[session_his['type']==3,'session'],session_his.loc[session_his['type']==3,'aid']))","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:12:03.398332Z","iopub.execute_input":"2024-06-08T03:12:03.398681Z","iopub.status.idle":"2024-06-08T03:12:03.789109Z","shell.execute_reply.started":"2024-06-08T03:12:03.398655Z","shell.execute_reply":"2024-06-08T03:12:03.787958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Recommend to the target users based on item popularity, while excluding items that have appeared in the users' historical behavior records.","metadata":{}},{"cell_type":"code","source":"res = []\n\nfor each_session in tqdm(target_users):\n\n if each_session in his_click_dict:\n his_click_tmp = his_click_dict[each_session]\n res.append([str(each_session)+'_clicks'] + [' '.join([str(x) for x in top200_aid_click if x not in his_click_tmp][:20])] )\n else:\n res.append([str(each_session)+'_clicks'] + [' '.join([str(x) for x in top200_aid_click][:20])])\n \n \n if each_session in his_cart_dict:\n his_cart_tmp = his_cart_dict[each_session]\n res.append([str(each_session)+'_carts'] + [' '.join([str(x) for x in top200_aid_cart if x not in his_cart_tmp][:20])] )\n else:\n res.append([str(each_session)+'_carts'] + [' '.join([str(x) for x in top200_aid_cart][:20])])\n \n if each_session in his_order_dict:\n his_order_tmp = his_order_dict[each_session]\n res.append([str(each_session)+'_orders'] + [' '.join([str(x) for x in top200_aid_order if x not in his_order_tmp][:20])] )\n else:\n res.append([str(each_session)+'_orders'] + [' '.join([str(x) for x in top200_aid_order][:20])])\n ","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:31:19.027143Z","iopub.execute_input":"2024-06-08T03:31:19.027631Z","iopub.status.idle":"2024-06-08T03:34:27.680660Z","shell.execute_reply.started":"2024-06-08T03:31:19.027596Z","shell.execute_reply":"2024-06-08T03:34:27.679523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_res = pd.DataFrame(res)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:34:31.688048Z","iopub.execute_input":"2024-06-08T03:34:31.688462Z","iopub.status.idle":"2024-06-08T03:34:32.727636Z","shell.execute_reply.started":"2024-06-08T03:34:31.688430Z","shell.execute_reply":"2024-06-08T03:34:32.725473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_res","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:34:35.994621Z","iopub.execute_input":"2024-06-08T03:34:35.995029Z","iopub.status.idle":"2024-06-08T03:34:36.008310Z","shell.execute_reply.started":"2024-06-08T03:34:35.994997Z","shell.execute_reply":"2024-06-08T03:34:36.006899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_dict = dict(zip(df_res[0],df_res[1]))","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:35:11.230137Z","iopub.execute_input":"2024-06-08T03:35:11.230568Z","iopub.status.idle":"2024-06-08T03:35:13.900623Z","shell.execute_reply.started":"2024-06-08T03:35:11.230533Z","shell.execute_reply":"2024-06-08T03:35:13.899344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##Evaluate performance
","metadata":{}},{"cell_type":"code","source":"session_items_val = df_val_offline.groupby(['session','type'],as_index=False)['aid'].agg(set)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:35:18.520615Z","iopub.execute_input":"2024-06-08T03:35:18.521001Z","iopub.status.idle":"2024-06-08T03:36:00.854976Z","shell.execute_reply.started":"2024-06-08T03:35:18.520971Z","shell.execute_reply":"2024-06-08T03:36:00.853705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_items_val ","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:36:07.595625Z","iopub.execute_input":"2024-06-08T03:36:07.596559Z","iopub.status.idle":"2024-06-08T03:36:07.611659Z","shell.execute_reply.started":"2024-06-08T03:36:07.596518Z","shell.execute_reply":"2024-06-08T03:36:07.610493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getScore(df, type_, suffix):\n df_tmp = df[df['type']==type_].reset_index(drop=True)\n \n score_Numerator = 0\n score_Denominator = 0\n\n for i,row in tqdm( df_tmp.iterrows(),total=len(df_tmp) ):\n recom_tmp = [int(x) for x in res_dict[str(row['session'])+suffix].split(' ')]\n score_Numerator += len(row['aid'] & set(recom_tmp))\n score_Denominator += min(20,len(row['aid']))\n \n return score_Numerator/score_Denominator","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:36:19.159803Z","iopub.execute_input":"2024-06-08T03:36:19.160196Z","iopub.status.idle":"2024-06-08T03:36:19.169151Z","shell.execute_reply.started":"2024-06-08T03:36:19.160165Z","shell.execute_reply":"2024-06-08T03:36:19.167948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_click = getScore(session_items_val,1,'_clicks')","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:36:24.217907Z","iopub.execute_input":"2024-06-08T03:36:24.218850Z","iopub.status.idle":"2024-06-08T03:38:08.918051Z","shell.execute_reply.started":"2024-06-08T03:36:24.218813Z","shell.execute_reply":"2024-06-08T03:38:08.916553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_cart = getScore(session_items_val,2,'_carts')","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:38:11.337255Z","iopub.execute_input":"2024-06-08T03:38:11.340354Z","iopub.status.idle":"2024-06-08T03:38:33.353938Z","shell.execute_reply.started":"2024-06-08T03:38:11.340299Z","shell.execute_reply":"2024-06-08T03:38:33.352789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_order = getScore(session_items_val,3,'_orders')","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:38:36.248664Z","iopub.execute_input":"2024-06-08T03:38:36.249110Z","iopub.status.idle":"2024-06-08T03:38:44.216804Z","shell.execute_reply.started":"2024-06-08T03:38:36.249075Z","shell.execute_reply":"2024-06-08T03:38:44.215637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_click * 0.1 + score_cart * 0.3 + score_order * 0.6","metadata":{"execution":{"iopub.status.busy":"2024-06-08T03:38:46.577278Z","iopub.execute_input":"2024-06-08T03:38:46.577778Z","iopub.status.idle":"2024-06-08T03:38:46.586323Z","shell.execute_reply.started":"2024-06-08T03:38:46.577739Z","shell.execute_reply":"2024-06-08T03:38:46.584938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}