{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"1. This notebook is originally derived from Chris Deotte notebook at https://www.kaggle.com/code/cdeotte/candidate-rerank-model-lb-0-575\n2. Few modifications were made in the forked version at https://www.kaggle.com/code/hoangnguyen719/candidate-rerank-model-lb-0-575\n3. This notebook, forked from the forked version (2), provides covisitation matrices built using only `train` set. They will be used in training and validation.","metadata":{"papermill":{"duration":0.00373,"end_time":"2022-11-10T16:03:20.9748","exception":false,"start_time":"2022-11-10T16:03:20.97107","status":"completed"},"tags":[]}},{"cell_type":"code","source":"VER = 2\n\nimport pandas as pd, numpy as np\nfrom tqdm.notebook import tqdm\nimport os, sys, pickle, glob, gc\nfrom collections import Counter\nimport cudf, itertools\nprint('We will use RAPIDS version',cudf.__version__)","metadata":{"papermill":{"duration":3.036143,"end_time":"2022-11-10T16:03:24.014816","exception":false,"start_time":"2022-11-10T16:03:20.978673","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T10:49:21.739803Z","iopub.execute_input":"2023-01-30T10:49:21.740225Z","iopub.status.idle":"2023-01-30T10:49:24.256363Z","shell.execute_reply.started":"2023-01-30T10:49:21.740134Z","shell.execute_reply":"2023-01-30T10:49:24.255433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TAIL = 40\n\nCART_ORDER_TOP = 40\nCART_ORDER_PIECES = 5\n\nBUY2BUY_TOP = 40\nBUY2BUY_PIECES = 2\n\nCLICK_TIME_TOP = 50\nCLICK_PIECES = 5\n\nTRAINING = False\n\n# Processing specification\nREAD_CT = 1 # Number of files in a group\nCHUNKS = 10 # Number of chunks\n# Number of groups will be decided below","metadata":{"execution":{"iopub.status.busy":"2023-01-30T10:49:24.258296Z","iopub.execute_input":"2023-01-30T10:49:24.258655Z","iopub.status.idle":"2023-01-30T10:49:24.264871Z","shell.execute_reply.started":"2023-01-30T10:49:24.258619Z","shell.execute_reply":"2023-01-30T10:49:24.263303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Compute Three Co-visitation Matrices with RAPIDS\nWe will compute 3 co-visitation matrices using RAPIDS cuDF on GPU. This is 30x faster than using Pandas CPU like other public notebooks! For maximum speed, set the variable `DISK_PIECES` to the smallest number possible based on the GPU you are using without incurring memory errors. If you run this code offline with 32GB GPU ram, then you can use `DISK_PIECES = 1` and compute each co-visitation matrix in almost 1 minute! Kaggle's GPU only has 16GB ram, so we use `DISK_PIECES = 4` and it takes an amazing 3 minutes each! Below are some of the tricks to speed up computation\n* Use RAPIDS cuDF GPU instead of Pandas CPU\n* Read disk once and save in CPU RAM for later GPU multiple use\n* Process largest amount of data possible on GPU at one time\n* Merge data in two stages. Multiple small to single medium. Multiple medium to single large.\n* Write result as parquet instead of dictionary","metadata":{"papermill":{"duration":0.00424,"end_time":"2022-11-10T16:03:24.023816","exception":false,"start_time":"2022-11-10T16:03:24.019576","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n# CACHE FUNCTIONS\ndef read_file(f):\n    return cudf.DataFrame( data_cache[f] )\ndef read_file_to_cache(f):\n    df = pd.read_parquet(f)\n    df.ts = (df.ts/1000).astype('int32')\n    df['type'] = df['type'].map(type_labels).astype('int8')\n    return df\n\n# CACHE THE DATA ON CPU BEFORE PROCESSING ON GPU\ndata_cache = {}\ntype_labels = {'clicks':0, 'carts':1, 'orders':2}\nif TRAINING:\n    files = glob.glob('../input/otto-chunk-data-inparquet-format/train_parquet/*') # Getting train only\nelse:\n    files = glob.glob('../input/otto-chunk-data-inparquet-format/*_parquet/*')\nfor f in files: data_cache[f] = read_file_to_cache(f)\n\n# CHUNK PARAMETERS\n# READ_CT = 5\n# CHUNK = int( np.ceil( len(files)/6 ))\n# print(f'We will process {len(files)} files, in groups of {READ_CT} and chunks of {CHUNK}.')","metadata":{"papermill":{"duration":0.063943,"end_time":"2022-11-10T16:03:24.091816","exception":false,"start_time":"2022-11-10T16:03:24.027873","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-30T10:49:24.623809Z","iopub.execute_input":"2023-01-30T10:49:24.624167Z","iopub.status.idle":"2023-01-30T10:50:23.704108Z","shell.execute_reply.started":"2023-01-30T10:49:24.624135Z","shell.execute_reply":"2023-01-30T10:50:23.702994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CHUNK = int( np.ceil( len(files)/CHUNKS ))\nprint(f'We will process {len(files)} files, in groups of {READ_CT} and chunks of {CHUNK}.')","metadata":{"execution":{"iopub.status.busy":"2023-01-30T10:50:23.706997Z","iopub.execute_input":"2023-01-30T10:50:23.708658Z","iopub.status.idle":"2023-01-30T10:50:23.715977Z","shell.execute_reply.started":"2023-01-30T10:50:23.708618Z","shell.execute_reply":"2023-01-30T10:50:23.714690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1) \"Carts Orders\" Co-visitation Matrix - Type Weighted","metadata":{"papermill":{"duration":0.004089,"end_time":"2022-11-10T16:03:24.100502","exception":false,"start_time":"2022-11-10T16:03:24.096413","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n# type_weight = {0:1, 1:6, 2:3}\ntype_weight = {0:1, 1:3, 2:6} # Using a different and potentially correct weight\n\n# USE SMALLEST DISK_PIECES POSSIBLE WITHOUT MEMORY ERROR\n# DISK_PIECES = 4\nSIZE = 1.86e6/CART_ORDER_PIECES\n\n# COMPUTE IN PARTS FOR MEMORY MANGEMENT\nfor PART in range(CART_ORDER_PIECES):\n    print()\n    print('### DISK PART',PART+1)\n    \n    # MERGE IS FASTEST PROCESSING CHUNKS WITHIN CHUNKS\n    # => OUTER CHUNKS\n    for j in range(CHUNKS):\n        a = j*CHUNK\n        b = min( (j+1)*CHUNK, len(files) )\n        print(f'Processing files {a} thru {b-1} in groups of {READ_CT}...')\n        \n        # => INNER CHUNKS\n        for k in range(a,b,READ_CT):\n            # READ FILE\n            if READ_CT==1:\n                df = read_file(files[k])\n            else:\n                df = [read_file(files[k])]\n                for i in range(1,READ_CT): \n                    if k+i<b: df.append( read_file(files[k+i]) )\n                df = cudf.concat(df,ignore_index=True,axis=0)\n            df = df.sort_values(['session','ts'],ascending=[True,False])\n            # USE TAIL OF SESSION\n            df = df.reset_index(drop=True)\n            df['n'] = df.groupby('session').cumcount()\n            df = df.loc[df.n<TAIL].drop('n',axis=1)\n            # CREATE PAIRS\n            df = df.merge(df,on='session')\n            df = df.loc[ ((df.ts_x - df.ts_y).abs()< 24 * 60 * 60) & (df.aid_x != df.aid_y) ]\n            # MEMORY MANAGEMENT COMPUTE IN PARTS\n            df = df.loc[(df.aid_x >= PART*SIZE)&(df.aid_x < (PART+1)*SIZE)]\n            # ASSIGN WEIGHTS\n#             df = df[['session', 'aid_x', 'aid_y','type_y']].drop_duplicates(['session', 'aid_x', 'aid_y'])\n            df = df[['session', 'aid_x', 'aid_y','type_y']].drop_duplicates(['session', 'aid_x', 'aid_y','type_y'])\n            df['wgt'] = df.type_y.map(type_weight)\n            df = df[['aid_x','aid_y','wgt']]\n            df.wgt = df.wgt.astype('float32')\n            df = df.groupby(['aid_x','aid_y']).wgt.sum()\n            # COMBINE INNER CHUNKS\n            if k==a: tmp2 = df\n            else: tmp2 = tmp2.add(df, fill_value=0)\n            print(k,', ',end='')\n            del df\n            gc.collect()\n        print()\n        # COMBINE OUTER CHUNKS\n        if a==0: tmp = tmp2\n        else: tmp = tmp.add(tmp2, fill_value=0)\n        del tmp2#, df\n        gc.collect()\n    # CONVERT MATRIX TO DICTIONARY\n    tmp = tmp.reset_index()\n    tmp = tmp.sort_values(['aid_x','wgt'],ascending=[True,False])\n    # SAVE TOP 40\n    tmp = tmp.reset_index(drop=True)\n    tmp['n'] = tmp.groupby('aid_x').aid_y.cumcount()\n    tmp = tmp.loc[tmp.n<CART_ORDER_TOP].drop('n',axis=1)\n    # SAVE PART TO DISK (convert to pandas first uses less memory)\n    tmp.to_pandas().to_parquet(f'top_{CART_ORDER_TOP}_carts_orders_v{VER}_{PART}.pqt')\n    del tmp\n    gc.collect()","metadata":{"papermill":{"duration":566.561189,"end_time":"2022-11-10T16:12:50.666123","exception":false,"start_time":"2022-11-10T16:03:24.104934","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T10:50:23.718250Z","iopub.execute_input":"2023-01-30T10:50:23.719161Z","iopub.status.idle":"2023-01-30T10:59:32.071877Z","shell.execute_reply.started":"2023-01-30T10:50:23.719115Z","shell.execute_reply":"2023-01-30T10:59:32.070794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2) \"Buy2Buy\" Co-visitation Matrix","metadata":{"papermill":{"duration":0.03219,"end_time":"2022-11-10T16:12:50.730634","exception":false,"start_time":"2022-11-10T16:12:50.698444","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n# USE SMALLEST DISK_PIECES POSSIBLE WITHOUT MEMORY ERROR\n# DISK_PIECES = 1\nSIZE = 1.86e6/BUY2BUY_PIECES\n\n# COMPUTE IN PARTS FOR MEMORY MANGEMENT\nfor PART in range(BUY2BUY_PIECES):\n    print()\n    print('### DISK PART',PART+1)\n    \n    # MERGE IS FASTEST PROCESSING CHUNKS WITHIN CHUNKS\n    # => OUTER CHUNKS\n    for j in range(CHUNKS):\n        a = j*CHUNK\n        b = min( (j+1)*CHUNK, len(files) )\n        print(f'Processing files {a} thru {b-1} in groups of {READ_CT}...')\n        \n        # => INNER CHUNKS\n        for k in range(a,b,READ_CT):\n            # READ FILE\n            df = [read_file(files[k])]\n            for i in range(1,READ_CT): \n                if k+i<b: df.append( read_file(files[k+i]) )\n            df = cudf.concat(df,ignore_index=True,axis=0)\n            df = df.loc[df['type'].isin([1,2])] # ONLY WANT CARTS AND ORDERS\n            df = df.sort_values(['session','ts'],ascending=[True,False])\n            # USE TAIL OF SESSION\n            df = df.reset_index(drop=True)\n            df['n'] = df.groupby('session').cumcount()\n            df = df.loc[df.n<TAIL].drop('n',axis=1)\n            # CREATE PAIRS\n            df = df.merge(df,on='session')\n            df = df.loc[ ((df.ts_x - df.ts_y).abs()< 14 * 24 * 60 * 60) & (df.aid_x != df.aid_y) ] # 14 DAYS\n            # MEMORY MANAGEMENT COMPUTE IN PARTS\n            df = df.loc[(df.aid_x >= PART*SIZE)&(df.aid_x < (PART+1)*SIZE)]\n            # ASSIGN WEIGHTS\n            df = df[['session', 'aid_x', 'aid_y','type_y']].drop_duplicates(['session', 'aid_x', 'aid_y'])\n            df['wgt'] = 1\n            df = df[['aid_x','aid_y','wgt']]\n            df.wgt = df.wgt.astype('float32')\n            df = df.groupby(['aid_x','aid_y']).wgt.sum()\n            # COMBINE INNER CHUNKS\n            if k==a: tmp2 = df\n            else: tmp2 = tmp2.add(df, fill_value=0)\n            print(k,', ',end='')\n        print()\n        # COMBINE OUTER CHUNKS\n        if a==0: tmp = tmp2\n        else: tmp = tmp.add(tmp2, fill_value=0)\n        del tmp2, df\n        gc.collect()\n    # CONVERT MATRIX TO DICTIONARY\n    tmp = tmp.reset_index()\n    tmp = tmp.sort_values(['aid_x','wgt'],ascending=[True,False])\n    # SAVE TOP 40\n    tmp = tmp.reset_index(drop=True)\n    tmp['n'] = tmp.groupby('aid_x').aid_y.cumcount()\n    tmp = tmp.loc[tmp.n<BUY2BUY_TOP].drop('n',axis=1)\n    # SAVE PART TO DISK (convert to pandas first uses less memory)\n    tmp.to_pandas().to_parquet(f'top_{BUY2BUY_TOP}_buy2buy_v{VER}_{PART}.pqt')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":113.735315,"end_time":"2022-11-10T16:14:44.498182","exception":false,"start_time":"2022-11-10T16:12:50.762867","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T10:59:32.074318Z","iopub.execute_input":"2023-01-30T10:59:32.074714Z","iopub.status.idle":"2023-01-30T11:00:52.920814Z","shell.execute_reply.started":"2023-01-30T10:59:32.074675Z","shell.execute_reply":"2023-01-30T11:00:52.919684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3) \"Clicks\" Co-visitation Matrix - Time Weighted","metadata":{"papermill":{"duration":0.04526,"end_time":"2022-11-10T16:14:44.58589","exception":false,"start_time":"2022-11-10T16:14:44.54063","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n# USE SMALLEST DISK_PIECES POSSIBLE WITHOUT MEMORY ERROR\n# DISK_PIECES = 4\nSIZE = 1.86e6/CLICK_PIECES\n\n# COMPUTE IN PARTS FOR MEMORY MANGEMENT\nfor PART in range(CLICK_PIECES):\n    print()\n    print('### DISK PART',PART+1)\n    \n    # MERGE IS FASTEST PROCESSING CHUNKS WITHIN CHUNKS\n    # => OUTER CHUNKS\n    for j in range(CHUNKS):\n        a = j*CHUNK\n        b = min( (j+1)*CHUNK, len(files) )\n        print(f'Processing files {a} thru {b-1} in groups of {READ_CT}...')\n        \n        # => INNER CHUNKS\n        for k in range(a,b,READ_CT):\n            # READ FILE\n            df = [read_file(files[k])]\n            for i in range(1,READ_CT): \n                if k+i<b: df.append( read_file(files[k+i]) )\n            df = cudf.concat(df,ignore_index=True,axis=0)\n            df = df.sort_values(['session','ts'],ascending=[True,False])\n            # USE TAIL OF SESSION\n            df = df.reset_index(drop=True)\n            df['n'] = df.groupby('session').cumcount()\n            df = df.loc[df.n<TAIL].drop('n',axis=1)\n            # CREATE PAIRS\n            df = df.merge(df,on='session')\n            df = df.loc[ ((df.ts_x - df.ts_y).abs()< 24 * 60 * 60) & (df.aid_x != df.aid_y) ]\n            # MEMORY MANAGEMENT COMPUTE IN PARTS\n            df = df.loc[(df.aid_x >= PART*SIZE)&(df.aid_x < (PART+1)*SIZE)]\n            # ASSIGN WEIGHTS\n            df = df[['session', 'aid_x', 'aid_y','ts_x']].drop_duplicates(['session', 'aid_x', 'aid_y'])\n            df['wgt'] = 1 + 3*(df.ts_x - 1659304800)/(1662328791-1659304800)\n            df = df[['aid_x','aid_y','wgt']]\n            df.wgt = df.wgt.astype('float32')\n            df = df.groupby(['aid_x','aid_y']).wgt.sum()\n            # COMBINE INNER CHUNKS\n            if k==a: tmp2 = df\n            else: tmp2 = tmp2.add(df, fill_value=0)\n            print(k,', ',end='')\n        print()\n        # COMBINE OUTER CHUNKS\n        if a==0: tmp = tmp2\n        else: tmp = tmp.add(tmp2, fill_value=0)\n        del tmp2, df\n        gc.collect()\n    # CONVERT MATRIX TO DICTIONARY\n    tmp = tmp.reset_index()\n    tmp = tmp.sort_values(['aid_x','wgt'],ascending=[True,False])\n    # SAVE TOP 40\n    tmp = tmp.reset_index(drop=True)\n    tmp['n'] = tmp.groupby('aid_x').aid_y.cumcount()\n    tmp = tmp.loc[tmp.n<CLICK_TIME_TOP].drop('n',axis=1)\n    # SAVE PART TO DISK (convert to pandas first uses less memory)\n    tmp.to_pandas().to_parquet(f'top_{CLICK_TIME_TOP}_clicks_v{VER}_{PART}.pqt')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":null,"end_time":null,"exception":false,"start_time":"2022-11-10T16:14:44.629032","status":"running"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T11:00:52.922347Z","iopub.execute_input":"2023-01-30T11:00:52.922924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FREE MEMORY\ndel data_cache, tmp\n_ = gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step 2 - ReRank (choose 20) using handcrafted rules\nWe are not inferencing with this notebook","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# def load_test():    \n#     dfs = []\n#     for e, chunk_file in enumerate(glob.glob('../input/otto-chunk-data-inparquet-format/test_parquet/*')):\n#         chunk = pd.read_parquet(chunk_file)\n#         chunk.ts = (chunk.ts/1000).astype('int32')\n#         chunk['type'] = chunk['type'].map(type_labels).astype('int8')\n#         dfs.append(chunk)\n#     return pd.concat(dfs).reset_index(drop=True) #.astype({\"ts\": \"datetime64[ms]\"})\n\n# test_df = load_test()\n# print('Test data has shape',test_df.shape)\n# test_df.head()","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# def pqt_to_dict(df):\n#     return df.groupby('aid_x').aid_y.apply(list).to_dict()\n# # LOAD THREE CO-VISITATION MATRICES\n# top_20_clicks = pqt_to_dict( pd.read_parquet(f'top_20_clicks_v{VER}_0.pqt') )\n# for k in range(1,DISK_PIECES): \n#     top_20_clicks.update( pqt_to_dict( pd.read_parquet(f'top_20_clicks_v{VER}_{k}.pqt') ) )\n# top_20_buys = pqt_to_dict( pd.read_parquet(f'top_15_carts_orders_v{VER}_0.pqt') )\n# for k in range(1,DISK_PIECES): \n#     top_20_buys.update( pqt_to_dict( pd.read_parquet(f'top_15_carts_orders_v{VER}_{k}.pqt') ) )\n# top_20_buy2buy = pqt_to_dict( pd.read_parquet(f'top_15_buy2buy_v{VER}_0.pqt') )\n\n# # TOP CLICKS AND ORDERS IN TEST\n# # top_clicks = test_df.loc[test_df['type']=='clicks','aid'].value_counts().index.values[:20]\n# # top_orders = test_df.loc[test_df['type']=='orders','aid'].value_counts().index.values[:20]\n# top_clicks = test_df.loc[test_df['type']==0,'aid'].value_counts().index.values[:20]\n# top_orders = test_df.loc[test_df['type']==2,'aid'].value_counts().index.values[:20]\n\n# print('Here are size of our 3 co-visitation matrices:')\n# print( len( top_20_clicks ), len( top_20_buy2buy ), len( top_20_buys ) )","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #type_weight_multipliers = {'clicks': 1, 'carts': 6, 'orders': 3}\n# type_weight_multipliers = {0: 1, 1: 6, 2: 3}\n\n# def suggest_clicks(df):\n#     # USER HISTORY AIDS AND TYPES\n#     aids=df.aid.tolist()\n#     types = df.type.tolist()\n#     unique_aids = list(dict.fromkeys(aids[::-1] ))\n#     # RERANK CANDIDATES USING WEIGHTS\n#     if len(unique_aids)>=20:\n#         weights=np.logspace(0.1,1,len(aids),base=2, endpoint=True)-1\n#         aids_temp = Counter() \n#         # RERANK BASED ON REPEAT ITEMS AND TYPE OF ITEMS\n#         for aid,w,t in zip(aids,weights,types): \n#             aids_temp[aid] += w * type_weight_multipliers[t]\n#         sorted_aids = [k for k,v in aids_temp.most_common(20)]\n#         return sorted_aids\n#     # USE \"CLICKS\" CO-VISITATION MATRIX\n#     aids2 = list(itertools.chain(*[top_20_clicks[aid] for aid in unique_aids if aid in top_20_clicks]))\n#     # RERANK CANDIDATES\n#     top_aids2 = [aid2 for aid2, cnt in Counter(aids2).most_common(20) if aid2 not in unique_aids]    \n#     result = unique_aids + top_aids2[:20 - len(unique_aids)]\n#     # USE TOP20 TEST CLICKS\n#     return result + list(top_clicks)[:20-len(result)]\n\n# def suggest_buys(df):\n#     # USER HISTORY AIDS AND TYPES\n#     aids=df.aid.tolist()\n#     types = df.type.tolist()\n#     # UNIQUE AIDS AND UNIQUE BUYS\n#     unique_aids = list(dict.fromkeys(aids[::-1] ))\n#     df = df.loc[(df['type']==1)|(df['type']==2)]\n#     unique_buys = list(dict.fromkeys( df.aid.tolist()[::-1] ))\n#     # RERANK CANDIDATES USING WEIGHTS\n#     if len(unique_aids)>=20:\n#         weights=np.logspace(0.5,1,len(aids),base=2, endpoint=True)-1\n#         aids_temp = Counter() \n#         # RERANK BASED ON REPEAT ITEMS AND TYPE OF ITEMS\n#         for aid,w,t in zip(aids,weights,types): \n#             aids_temp[aid] += w * type_weight_multipliers[t]\n#         # RERANK CANDIDATES USING \"BUY2BUY\" CO-VISITATION MATRIX\n#         aids3 = list(itertools.chain(*[top_20_buy2buy[aid] for aid in unique_buys if aid in top_20_buy2buy]))\n#         for aid in aids3: aids_temp[aid] += 0.1\n#         sorted_aids = [k for k,v in aids_temp.most_common(20)]\n#         return sorted_aids\n#     # USE \"CART ORDER\" CO-VISITATION MATRIX\n#     aids2 = list(itertools.chain(*[top_20_buys[aid] for aid in unique_aids if aid in top_20_buys]))\n#     # USE \"BUY2BUY\" CO-VISITATION MATRIX\n#     aids3 = list(itertools.chain(*[top_20_buy2buy[aid] for aid in unique_buys if aid in top_20_buy2buy]))\n#     # RERANK CANDIDATES\n#     top_aids2 = [aid2 for aid2, cnt in Counter(aids2+aids3).most_common(20) if aid2 not in unique_aids] \n#     result = unique_aids + top_aids2[:20 - len(unique_aids)]\n#     # USE TOP20 TEST ORDERS\n#     return result + list(top_orders)[:20-len(result)]","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Submission CSV\nInferring test data with Pandas groupby is slow. We need to accelerate the following code.","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# %%time\n# pred_df_clicks = test_df.sort_values([\"session\", \"ts\"]).groupby([\"session\"]).apply(\n#     lambda x: suggest_clicks(x)\n# )\n\n# pred_df_buys = test_df.sort_values([\"session\", \"ts\"]).groupby([\"session\"]).apply(\n#     lambda x: suggest_buys(x)\n# )","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clicks_pred_df = pd.DataFrame(pred_df_clicks.add_suffix(\"_clicks\"), columns=[\"labels\"]).reset_index()\n# orders_pred_df = pd.DataFrame(pred_df_buys.add_suffix(\"_orders\"), columns=[\"labels\"]).reset_index()\n# carts_pred_df = pd.DataFrame(pred_df_buys.add_suffix(\"_carts\"), columns=[\"labels\"]).reset_index()","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_df = pd.concat([clicks_pred_df, orders_pred_df, carts_pred_df])\n# pred_df.columns = [\"session_type\", \"labels\"]\n# pred_df[\"labels\"] = pred_df.labels.apply(lambda x: \" \".join(map(str,x)))\n# pred_df.to_csv(\"submission.csv\", index=False)\n# pred_df.head()","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}