{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd, numpy as np\nimport os, sys, pickle, glob, gc, itertools, math, json\nimport cudf\nfrom datetime import datetime as dt\n# import matplotlib.pyplot as plt\nfrom sklearn.model_selection import StratifiedKFold as skfold\n# from collections import Counter\nimport xgboost as xgb\n\nprint('We will use RAPIDS version',cudf.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-31T20:31:16.286133Z","iopub.execute_input":"2023-01-31T20:31:16.287205Z","iopub.status.idle":"2023-01-31T20:31:19.865325Z","shell.execute_reply.started":"2023-01-31T20:31:16.287075Z","shell.execute_reply":"2023-01-31T20:31:19.864234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Parameters\nVER = 2\n\n# Data paths\nFT_DS = '/kaggle/input/otto-te-cand40-v2-tail40-top404050/'\nCANDIDATES = 40\n\nCLICK_SAVE_NAME = f'Rerank_Click_Infer_Can{CANDIDATES}_V{VER}_Tail40_Top404050.pqt'\nCART_SAVE_NAME = f'Rerank_Cart_Infer_Can{CANDIDATES}_V{VER}_Tail40_Top404050.pqt'\nORDER_SAVE_NAME = f'Rerank_Order_Infer_Can{CANDIDATES}_V{VER}_Tail40_Top404050.pqt'","metadata":{"execution":{"iopub.status.busy":"2023-01-31T20:31:19.869461Z","iopub.execute_input":"2023-01-31T20:31:19.870245Z","iopub.status.idle":"2023-01-31T20:31:19.879579Z","shell.execute_reply.started":"2023-01-31T20:31:19.870184Z","shell.execute_reply":"2023-01-31T20:31:19.877954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{}},{"cell_type":"code","source":"def timer(sta):\n    return round((dt.now() - sta).seconds, 3)\n\ndef load_pqt(path):\n    return pd.read_parquet(path)\n\ndef load2cudf(path):\n    return cudf.from_pandas(load_pqt(path))\n\ndef pd2cudf(df):\n    return cudf.from_pandas(df)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T20:31:46.895130Z","iopub.execute_input":"2023-01-31T20:31:46.895530Z","iopub.status.idle":"2023-01-31T20:31:46.902309Z","shell.execute_reply.started":"2023-01-31T20:31:46.895497Z","shell.execute_reply":"2023-01-31T20:31:46.901331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ui_ft = load2cudf(FT_DS + f'ui_event_wgt_v{VER}.pqt')","metadata":{"execution":{"iopub.status.busy":"2023-01-31T20:31:47.689856Z","iopub.execute_input":"2023-01-31T20:31:47.690222Z","iopub.status.idle":"2023-01-31T20:31:50.678768Z","shell.execute_reply.started":"2023-01-31T20:31:47.690187Z","shell.execute_reply":"2023-01-31T20:31:50.677739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsorting_cols = ['clk_cov_wgt', 'top_clk_wk', 'ui_clk_wgt']\n\nclicks = load2cudf(FT_DS + f'click_candidates_top{CANDIDATES}_v{VER}.pqt')\nclicks = clicks.merge(ui_ft, on=['session', 'aid'], how='left')\nclicks = clicks.loc[:, ['session', 'aid'] + sorting_cols]\nclicks = clicks.fillna(0)\n\nclicks = clicks.sort_values(['session'] + sorting_cols, ascending=False)\nclicks = clicks.reset_index(drop=True)\nclicks['n'] = clicks.groupby('session').cumcount()\nclicks['predict_final'] = (clicks.n < 20).astype('uint16')\nclicks = clicks[['session', 'aid', 'predict_final']]\n\nclicks.to_pandas().to_parquet(CLICK_SAVE_NAME)\nclicks.head(30)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T20:31:51.514119Z","iopub.execute_input":"2023-01-31T20:31:51.514502Z","iopub.status.idle":"2023-01-31T20:32:05.558690Z","shell.execute_reply.started":"2023-01-31T20:31:51.514468Z","shell.execute_reply":"2023-01-31T20:32:05.557509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsorting_cols = ['ui_order_wgt', 'b2b_cov_wgt', 'buy_cov_wgt', 'top_buy_wk']\n\ncarts = load2cudf(FT_DS + f'order_carts_candidates_top{CANDIDATES}_v{VER}.pqt')\ncarts = carts.merge(ui_ft, on=['session', 'aid'], how='left')\ncarts = carts.loc[:, ['session', 'aid'] + sorting_cols]\ncarts = carts.fillna(0)\n\ncarts = carts.sort_values(['session'] + sorting_cols, ascending=False)\ncarts = carts.reset_index(drop=True)\ncarts['n'] = carts.groupby('session').cumcount()\ncarts['predict_final'] = (carts.n < 20).astype('uint16')\ncarts = carts[['session', 'aid', 'predict_final']]\n\ncarts.to_pandas().to_parquet(CART_SAVE_NAME)\ncarts.to_pandas().to_parquet(ORDER_SAVE_NAME)\ncarts.head(30)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T20:32:09.157611Z","iopub.execute_input":"2023-01-31T20:32:09.157975Z","iopub.status.idle":"2023-01-31T20:32:25.173504Z","shell.execute_reply.started":"2023-01-31T20:32:09.157943Z","shell.execute_reply":"2023-01-31T20:32:25.172526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntmp = clicks.loc[clicks.predict_final == 1, ['session', 'aid']]\ntmp['session'] = tmp.session.astype(str) + '_clicks'\ntmp = tmp.to_pandas()\ntmp = tmp.groupby('session')['aid'].apply(lambda x: ' '.join(map(str,x)))\ntmp = tmp.reset_index()\n\nsubmission = [tmp]\n\nfor e in ['carts', 'orders']:\n    tmp = carts.loc[carts.predict_final == 1, ['session', 'aid']]\n    tmp['session'] = tmp.session.astype(str) + '_' + e\n    tmp = tmp.to_pandas()\n    tmp = tmp.groupby('session')['aid'].apply(lambda x: ' '.join(map(str,x)))\n    tmp = tmp.reset_index()\n    submission.append(tmp)\n    \ndel clicks, carts, tmp\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T20:32:25.175558Z","iopub.execute_input":"2023-01-31T20:32:25.176221Z","iopub.status.idle":"2023-01-31T20:33:05.479631Z","shell.execute_reply.started":"2023-01-31T20:32:25.176183Z","shell.execute_reply":"2023-01-31T20:33:05.478557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsubmission = pd.concat(submission)\nsubmission = submission.rename(columns={'session':'session_type', 'aid':'labels'})\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T20:24:42.733538Z","iopub.execute_input":"2023-01-31T20:24:42.734257Z","iopub.status.idle":"2023-01-31T20:25:03.660192Z","shell.execute_reply.started":"2023-01-31T20:24:42.734218Z","shell.execute_reply":"2023-01-31T20:25:03.659095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ses = 1671803\nlen_submit = len(submission)\nprint('Validating')\nprint(f'len={len_submit}')\nprint(f'sessions = {round(len_submit/3)}/{test_ses} ({round(len_submit/len_submit,2)})')","metadata":{"execution":{"iopub.status.busy":"2023-01-31T20:25:03.665255Z","iopub.execute_input":"2023-01-31T20:25:03.667326Z","iopub.status.idle":"2023-01-31T20:25:03.677499Z","shell.execute_reply.started":"2023-01-31T20:25:03.667297Z","shell.execute_reply":"2023-01-31T20:25:03.676534Z"},"trusted":true},"execution_count":null,"outputs":[]}]}