{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Description\n- V1 -> V9: build, full data seems to be large, subsample has problem with mismatch session\n- V10: 1/5 data | fix bug mismatch ~ V7 dataset\n- V12: full data with chunking ~ V9 dataset\n- V13: ~ V12, num_history_candidates = 30\n- V16: add item feautures + re-chunking\n- V17: change num_chunk 5->10\n- V18: num_chunk = 20\n- V19: num_potential_cands 50->100 | num_his 30->50\n- V20: num_pot 50 | num_his 100\n- V22: new pipeline (candidates from notebook covi)\n- V23: ~V22, 100 cands better\n- V24: ~V23, 150 cands\n- V25: 50 cands\n- V28: ~V23, add more session + aid features\n- V29: ~V23 (timestamp day features)\n- V30: ~V23, timestamp day features | 20 chunks\n- V32: ~V23, timestamp day features | 10 chunks\n- V33: ~V23, modify aid features\n- V34: ~V33, 5 chunks\n- V37: ~V33, only session has groundtruth\n- V39: 37 features + only session has groundtruth\n- V44: 37 features + 5 chunks\n- V47: ~V44, add common test candidates","metadata":{}},{"cell_type":"code","source":"!pip install pyarrow fastparquet","metadata":{"execution":{"iopub.status.busy":"2023-01-31T10:28:49.881462Z","iopub.execute_input":"2023-01-31T10:28:49.882754Z","iopub.status.idle":"2023-01-31T10:29:06.776694Z","shell.execute_reply.started":"2023-01-31T10:28:49.882644Z","shell.execute_reply":"2023-01-31T10:29:06.775565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport time\nimport datetime\nfrom tqdm.notebook import tqdm\nimport os, sys, pickle, glob, gc\nfrom collections import Counter\nimport itertools\n\n# from multiprocessing import Pool\n# import psutil\n# N_CPU = psutil.cpu_count()\n# print(\"Number of cpu:\", N_CPU)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T10:29:06.780271Z","iopub.execute_input":"2023-01-31T10:29:06.780935Z","iopub.status.idle":"2023-01-31T10:29:06.915312Z","shell.execute_reply.started":"2023-01-31T10:29:06.780870Z","shell.execute_reply":"2023-01-31T10:29:06.913898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LEAK_DATA = True\nUSE_ONLY_GT_SESSION = True\nNUM_CHUNK = 5\n\nMIN_TS = 1661119200\nMAX_TS = 1661723999\nprint(\"Starting point of validA:\", datetime.datetime.fromtimestamp(MIN_TS))\nprint(\"Ending point of validA:\", datetime.datetime.fromtimestamp(MAX_TS))","metadata":{"execution":{"iopub.status.busy":"2023-01-31T10:29:06.919182Z","iopub.execute_input":"2023-01-31T10:29:06.920464Z","iopub.status.idle":"2023-01-31T10:29:06.928518Z","shell.execute_reply.started":"2023-01-31T10:29:06.920406Z","shell.execute_reply":"2023-01-31T10:29:06.927098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type_labels = {'clicks':0, 'carts':1, 'orders':2}\ndef read_file_to_cache(f):\n    df = pd.read_parquet(f)\n    df.ts = (df.ts/1000).astype('int32')\n    df['type'] = df['type'].map(type_labels).astype('int8')\n    return df\n\ntest_files = sorted(glob.glob('/kaggle/input/otto-validation/test_parquet/*'))\nfiles = [test_files[i] for i in range(len(test_files))]\n\ndfs = [read_file_to_cache(f) for f in files]\ntest_df = pd.concat(dfs, axis=0)\nlabel_df = pd.read_parquet(f\"/kaggle/input/otto-validation/test_labels.parquet\")\n\ndel dfs\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T10:29:06.931932Z","iopub.execute_input":"2023-01-31T10:29:06.932401Z","iopub.status.idle":"2023-01-31T10:29:12.164196Z","shell.execute_reply.started":"2023-01-31T10:29:06.932364Z","shell.execute_reply":"2023-01-31T10:29:12.162989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# features for history candidates\nhis_df = pd.get_dummies(data=test_df, columns=['type'])\nhis_df = his_df.groupby(['session','aid']).agg({'type_0':'sum','type_1':'sum','type_2':'sum','ts':['min','max']})\nhis_df = his_df.reset_index()\nhis_df.columns = ['session','aid','his_num_clicks','his_num_carts','his_num_orders','his_min_ts','his_max_ts']\nhis_df['his_min_ts'] = ((his_df['his_min_ts'] - MIN_TS)/(MAX_TS-MIN_TS)).astype('float32')\nhis_df['his_max_ts'] = ((his_df['his_max_ts'] - MIN_TS)/(MAX_TS-MIN_TS)).astype('float32')\nhis_df['his_num_clicks'] = his_df['his_num_clicks'].astype('int32')\nhis_df['his_num_carts'] = his_df['his_num_carts'].astype('int32')\nhis_df['his_num_orders'] = his_df['his_num_orders'].astype('int32')\nhis_df['his_num_actions'] = his_df['his_num_clicks'] + his_df['his_num_carts'] + his_df['his_num_orders']","metadata":{"execution":{"iopub.status.busy":"2023-01-31T10:29:12.166075Z","iopub.execute_input":"2023-01-31T10:29:12.166633Z","iopub.status.idle":"2023-01-31T10:29:19.712730Z","shell.execute_reply.started":"2023-01-31T10:29:12.166586Z","shell.execute_reply":"2023-01-31T10:29:19.711166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# User features","metadata":{}},{"cell_type":"code","source":"session_features = pd.read_parquet(\"/kaggle/input/100x-faster-feature-generation/sess_feature.parquet\")\nsession_features = session_features[['session', 'ts_max',\n       'ts_min', 'ts_mean']]\nsession_features['ts_max'] = ((session_features['ts_max'] - MIN_TS)/(MAX_TS-MIN_TS)).astype('float32')\nsession_features['ts_min'] = ((session_features['ts_min'] - MIN_TS)/(MAX_TS-MIN_TS)).astype('float32')\nsession_features['ts_mean'] = ((session_features['ts_mean'] - MIN_TS)/(MAX_TS-MIN_TS)).astype('float32')\n\ncolumns = []\nfor f in session_features.columns:\n    columns.append(f if \"sess\" in f else (\"sess_\" + f))\nsession_features.columns = columns\nprint(session_features.columns)\nprint(session_features.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T10:29:19.714431Z","iopub.execute_input":"2023-01-31T10:29:19.714799Z","iopub.status.idle":"2023-01-31T10:29:21.563917Z","shell.execute_reply.started":"2023-01-31T10:29:19.714766Z","shell.execute_reply":"2023-01-31T10:29:21.562258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Item features","metadata":{}},{"cell_type":"code","source":"# aid features from test set\ntest_item_features = pd.get_dummies(data=test_df, columns=['type'])\ntest_item_features = test_item_features.groupby(['aid']).agg({'type_0':'sum','type_1':'sum','type_2':'sum','ts':['min','max']})\ntest_item_features = test_item_features.reset_index()\ntest_item_features.columns = ['aid','aid_test_num_clicks','aid_test_num_carts','aid_test_num_orders','aid_test_min_ts','aid_test_max_ts']\ntest_item_features['aid_test_min_ts'] = ((test_item_features['aid_test_min_ts'] - MIN_TS)/(MAX_TS-MIN_TS)).astype('float32')\ntest_item_features['aid_test_max_ts'] = ((test_item_features['aid_test_max_ts'] - MIN_TS)/(MAX_TS-MIN_TS)).astype('float32')\ntest_item_features['aid_test_num_clicks'] = test_item_features['aid_test_num_clicks'].astype('int32')\ntest_item_features['aid_test_num_carts'] = test_item_features['aid_test_num_carts'].astype('int32')\ntest_item_features['aid_test_num_orders'] = test_item_features['aid_test_num_orders'].astype('int32')\ntest_item_features['aid_test_num_actions'] = test_item_features['aid_test_num_clicks'] + test_item_features['aid_test_num_carts'] + test_item_features['aid_test_num_orders']\nprint(test_item_features.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T10:29:21.565329Z","iopub.execute_input":"2023-01-31T10:29:21.565744Z","iopub.status.idle":"2023-01-31T10:29:24.257292Z","shell.execute_reply.started":"2023-01-31T10:29:21.565706Z","shell.execute_reply":"2023-01-31T10:29:24.255549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# aid features from train + test\nitem_features = pd.read_parquet(\"/kaggle/input/otto-item-features/item_features_full_valid.pqt\")\nitem_features.drop(columns=['aid_aid_count'], inplace=True)\nitem_features['aid_ts_min'] = ((item_features['aid_ts_min'] - 1659304800)/(MAX_TS-1659304800)).astype('float32')\nitem_features['aid_ts_max'] = ((item_features['aid_ts_max'] - 1659304800)/(MAX_TS-1659304800)).astype('float32')\nprint(item_features.columns)\nprint(item_features.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T10:29:24.258994Z","iopub.execute_input":"2023-01-31T10:29:24.259498Z","iopub.status.idle":"2023-01-31T10:29:25.111494Z","shell.execute_reply.started":"2023-01-31T10:29:24.259461Z","shell.execute_reply":"2023-01-31T10:29:25.110217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# aid features from train + test\nitem_features2 = pd.read_parquet(\"/kaggle/input/100x-faster-feature-generation/aid_features.parquet\")\nitem_features2 = item_features2[['aid','aid_ca_cl_ratio', 'aid_or_cl_ratio',\n       'aid_or_ca_ratio']]\nprint(item_features2.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T10:29:25.113075Z","iopub.execute_input":"2023-01-31T10:29:25.114286Z","iopub.status.idle":"2023-01-31T10:29:27.042947Z","shell.execute_reply.started":"2023-01-31T10:29:25.114236Z","shell.execute_reply":"2023-01-31T10:29:27.041542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_features = item_features.merge(item_features2, on='aid', how='left').merge(test_item_features, on='aid', how='left').fillna(0)\ndel item_features2, test_item_features\ngc.collect()\n\nprint(item_features.columns)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T10:29:27.046553Z","iopub.execute_input":"2023-01-31T10:29:27.047111Z","iopub.status.idle":"2023-01-31T10:29:29.496430Z","shell.execute_reply.started":"2023-01-31T10:29:27.047056Z","shell.execute_reply":"2023-01-31T10:29:29.494873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Interaction features","metadata":{}},{"cell_type":"code","source":"%%time\n\nfor _type in ['click','cart','order']:\n    # import history candidates from covi\n    candidates = pd.read_parquet(f\"/kaggle/input/otto-covi-candidates-validation/{_type}s_candidates.pqt\")\n\n    history_candidates = candidates.loc[candidates.type_candidate == 1].reset_index(drop=True)\n    history_candidates.rename(columns={'score':'his_covi_score'}, inplace=True)\n    history_candidates = history_candidates.merge(his_df, on=['session','aid'], how='left')\n    \n    common_test_cands = candidates.loc[candidates.type_candidate == 2].reset_index(drop=True)\n    common_test_cands.rename(columns={'score':'his_covi_score'}, inplace=True)\n    del candidates\n    gc.collect()\n\n    # import  potential candidates\n    dfs_list = []\n    files = glob.glob(f\"/kaggle/input/otto-interaction-features-dataset/{_type}*\")\n    for file in files: \n        dfs_list.append(pd.read_parquet(file))    \n    potential_candidates = pd.concat(dfs_list, axis=0, ignore_index=True)\n    del dfs_list\n    gc.collect()\n    \n    chunk_list = [0,4] if USE_ONLY_GT_SESSION else [0,2]\n    for chunk in chunk_list:\n        print(f\"{_type} | CHUNK {chunk}\")\n        if _type == 'click' or USE_ONLY_GT_SESSION is False:\n            sub_pot_cands = potential_candidates.loc[potential_candidates.session % NUM_CHUNK == chunk].reset_index(drop=True)\n            sub_his_cands = history_candidates.loc[history_candidates.session % NUM_CHUNK == chunk].reset_index(drop=True)\n            sub_common_test_cands = common_test_cands.loc[common_test_cands.session % NUM_CHUNK == chunk].reset_index(drop=True)\n        else:\n            chunk_buy = 0 if chunk == 0 else 1\n            sub_pot_cands = potential_candidates.loc[potential_candidates.session % 2 == chunk_buy].reset_index(drop=True)\n            sub_his_cands = history_candidates.loc[history_candidates.session % 2 == chunk_buy].reset_index(drop=True)\n            sub_common_test_cands = common_test_cands.loc[common_test_cands.session % 2 == chunk_buy].reset_index(drop=True)\n        candidates = pd.concat([sub_his_cands, sub_pot_cands.rename(columns={'aid_y':'aid'}), sub_common_test_cands], \n                               ignore_index=True, axis = 0).fillna(0).sort_values(by=['session'], ignore_index=True)\n\n        del sub_pot_cands, sub_his_cands, sub_common_test_cands\n        gc.collect()\n\n        # read labels\n        gt_df = label_df.loc[label_df.type == _type+\"s\"]\n        gt_df = gt_df.explode('ground_truth')\n        gt_df = gt_df.drop(columns='type').rename(columns={'ground_truth':'aid'})\n        gt_df['gt'] = 1\n\n        # merge gt\n        print(\"Number of session:\", candidates.session.nunique())\n        session_df = candidates[['session']].drop_duplicates()\n        candidates = candidates.merge(session_df, on=['session'], how='inner')\n        print(\"Number of session:\", candidates.session.nunique())\n        candidates = candidates.merge(gt_df, on=['session','aid'], how='left').fillna(0)\n        \n        # remove sessions that have no gt\n        if USE_ONLY_GT_SESSION:\n            candidates['have_gt'] = candidates.groupby(['session']).gt.transform(max)\n            candidates = candidates.loc[candidates.have_gt > 0]\n            candidates.drop(columns=['have_gt'], inplace=True)\n            print(\"Number of session:\", candidates.session.nunique())\n        candidates = candidates.merge(item_features, on=['aid'], how='left').merge(session_features, on=['session'], how='left').fillna(0)\n        print(\"Shape:\", candidates.shape)\n        candidates.to_parquet(f\"/kaggle/working/{_type}_features_{chunk}.pqt\")\n\n        del gt_df, candidates\n        gc.collect()\n\n    del history_candidates, potential_candidates, common_test_cands\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T10:29:29.497874Z","iopub.execute_input":"2023-01-31T10:29:29.498353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compute metric","metadata":{}},{"cell_type":"code","source":"%%time\nchunk_list = [0,4] if USE_ONLY_GT_SESSION else [0,2]\nfor chunk in chunk_list:\n    print(\"chunk \", chunk)\n    types = ['click','cart','order']\n    candidates = dict()\n    for _type in types:\n        cans = pd.read_parquet(f\"/kaggle/working/{_type}_features_{chunk}.pqt\")\n        candidates[_type+\"s\"] = cans[['session','aid']].groupby('session').agg({'aid': lambda x: list(x)}).reset_index().rename(columns={'aid':'labels'})\n    full_test_labels = pd.read_parquet('/kaggle/input/otto-validation/test_labels.parquet')\n\n    score = 0\n    weights = {'clicks': 0.10, 'carts': 0.30, 'orders': 0.60}\n    for t in ['clicks','carts','orders']:\n        sub = candidates[t]\n        test_labels = full_test_labels.loc[full_test_labels['type']==t]\n        test_labels = test_labels.merge(sub, how='inner', on=['session'])\n        test_labels['hits'] = test_labels.apply(lambda df: len(set(df.ground_truth).intersection(set(df.labels))), axis=1)\n        test_labels['gt_count'] = test_labels.ground_truth.str.len().clip(0,20)\n\n        recall = test_labels['hits'].sum() / test_labels['gt_count'].sum()\n        score += weights[t]*recall\n        print(f'{t} recall =',recall)\n    print(\"Finall recall:\", score)\n    \n    del candidates, full_test_labels, test_labels\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T09:19:35.638019Z","iopub.execute_input":"2023-01-25T09:19:35.638481Z","iopub.status.idle":"2023-01-25T09:20:56.735942Z","shell.execute_reply.started":"2023-01-25T09:19:35.638442Z","shell.execute_reply":"2023-01-25T09:20:56.734969Z"},"trusted":true},"execution_count":null,"outputs":[]}]}