{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Description\n- V17: 100 cands | LB 0.578\n- V18: 100 cands better | LB 0.578\n- V20: ~V18, better convergence | LB 0.579\n- V21: ~V18, max_depth=3 | LB 0.578\n- V22: 150 cands\n- V24: 50 cands | LB 0.578\n- V25: ~V22, negative sampling (is not better)\n- V34: V26 test | 100 features | LB 0.578\n- V31: add day timestamp features\n- V39: 20 chunks | 50+ features\n- V44: ~V39, 10 chunks | 50+ features\n- V45: 10 chunks | 24 features\n- V51: 24 features | aid features last week | 5 chunks\n- V54,V55,V56: ~V51, remove sessions have no gt\n- V59: V40 candidates | 37 features + remove session have no gt\n- V61: 37 features + 5 chunks\n- V62: ~V61 + add common test aid","metadata":{}},{"cell_type":"code","source":"!pip install scikit-learn==1.0.2 --upgrade\nimport sklearn\nprint(sklearn.__version__)\n\n!pip install xgboost pyarrow fastparquet\nUSE_TPU = True","metadata":{"execution":{"iopub.status.busy":"2023-01-30T15:40:52.012967Z","iopub.execute_input":"2023-01-30T15:40:52.013382Z","iopub.status.idle":"2023-01-30T15:40:52.043183Z","shell.execute_reply.started":"2023-01-30T15:40:52.013278Z","shell.execute_reply":"2023-01-30T15:40:52.042193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nfrom tqdm.notebook import tqdm\nimport os, sys, pickle, glob, gc\nfrom collections import Counter\nimport itertools\n\nfrom sklearn.model_selection import GroupKFold\nimport xgboost as xgb","metadata":{"execution":{"iopub.status.busy":"2023-01-30T15:40:52.045652Z","iopub.execute_input":"2023-01-30T15:40:52.046636Z","iopub.status.idle":"2023-01-30T15:40:53.301438Z","shell.execute_reply.started":"2023-01-30T15:40:52.046599Z","shell.execute_reply":"2023-01-30T15:40:53.300371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEATURES_PATH = \"/kaggle/input/otto-candidate-features\"","metadata":{"execution":{"iopub.status.busy":"2023-01-30T15:40:53.307147Z","iopub.execute_input":"2023-01-30T15:40:53.309625Z","iopub.status.idle":"2023-01-30T15:40:53.316440Z","shell.execute_reply.started":"2023-01-30T15:40:53.309583Z","shell.execute_reply":"2023-01-30T15:40:53.315367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"%%time\nchunk = 0\nfor _type in ['click','cart','order']:\n    candidates = pd.read_parquet(f\"/kaggle/input/otto-candidate-features/{_type}_features_{chunk}.pqt\").reset_index(drop=True)\n#     candidates['aid_num_actions'] = candidates['aid_num_clicks'] + candidates['aid_num_carts'] + candidates['aid_num_orders']\n#     candidates['aid_click_ratio'] = candidates['aid_num_clicks'] / candidates['aid_num_actions']\n#     candidates['aid_cart_ratio'] = candidates['aid_num_carts'] / candidates['aid_num_actions']\n#     candidates['aid_order_ratio'] = candidates['aid_num_orders'] / candidates['aid_num_actions']\n#     candidates.drop(columns=['aid_num_clicks','aid_num_carts','aid_num_orders'], inplace=True)\n#     positives = candidates.loc[candidates['gt']==1]\n#     negatives = candidates.loc[candidates['gt']==0].sample(frac=0.8)\n#     candidates = pd.concat([negatives,positives],axis=0,ignore_index=True).sort_values(by=['session','aid_num_orders']).reset_index(drop=True)\n#     del positives, negatives\n#    gc.collect()\n    \n    FEATURES = [f for f in candidates.columns[2:] if f != 'gt']\n    print(\"Number of features:\", len(FEATURES))\n    candidates['group_count'] = candidates.groupby(['session']).cumcount('aid') + 1\n\n    importances = []\n    skf = GroupKFold(n_splits=5)\n    \n    for fold,(train_idx, valid_idx) in enumerate(skf.split(candidates, candidates['gt'], groups=candidates['session'] )):\n\n        X_train = candidates.loc[train_idx, FEATURES]\n        y_train = candidates.loc[train_idx, 'gt']\n        X_valid = candidates.loc[valid_idx, FEATURES]\n        y_valid = candidates.loc[valid_idx, 'gt']\n        \n        # IF YOU HAVE 50 CANDIDATE WE USE 50 BELOW\n        train_groups = candidates.loc[train_idx, ['session','group_count']].groupby('session')['group_count'].last().to_numpy()\n        valid_groups = candidates.loc[valid_idx, ['session','group_count']].groupby('session')['group_count'].last().to_numpy()\n        \n        dtrain = xgb.DMatrix(X_train, y_train, group=train_groups) \n        dvalid = xgb.DMatrix(X_valid, y_valid, group=valid_groups) \n        \n        tree_method = 'hist' if USE_TPU else 'gpu_hist'\n        xgb_parms = {'objective':'rank:pairwise', 'tree_method':tree_method, 'eta':0.1, 'max_depth':4}\n\n        print('#'*25)\n        print('### Fold',fold+1)\n        print('### Train size',len(train_idx),' - Valid size',len(valid_idx))\n        print('#'*25)\n        \n        if _type == \"order\":\n            early_stopping_rounds = 50\n        elif _type == 'cart':\n            early_stopping_rounds = 40\n        else:\n            early_stopping_rounds = 40\n            \n        model = xgb.train(xgb_parms, \n            dtrain=dtrain,\n            evals=[(dtrain,'train'),(dvalid,'valid')],\n            num_boost_round=2000,\n            verbose_eval=50,\n            early_stopping_rounds=early_stopping_rounds)\n        model.save_model(f'/kaggle/working/XGB_fold{fold}_{_type}.xgb')\n\n        # GET FEATURE IMPORTANCE FOR FOLD K\n        dd = model.get_score(importance_type='weight')\n        df = pd.DataFrame({'feature':dd.keys(),f'importance_{fold}':dd.values()})\n        df.to_csv(f\"/kaggle/working/feat_importances_fold{fold}_{_type}.csv\")\n        importances.append(df)\n\n        del dtrain, dvalid, model\n        _ = gc.collect()\n    print(importances[0])\n    del candidates, importances\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T15:41:33.572646Z","iopub.execute_input":"2023-01-30T15:41:33.573036Z","iopub.status.idle":"2023-01-30T15:44:05.656968Z","shell.execute_reply.started":"2023-01-30T15:41:33.573001Z","shell.execute_reply":"2023-01-30T15:44:05.655965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validate","metadata":{}},{"cell_type":"code","source":"full_test_labels = pd.read_parquet('/kaggle/input/otto-validation/test_labels.parquet')\nrecall = dict()\nweights = {'click':0.1, 'cart':0.3, 'order':0.6}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor chunk in [0,4]:\n    print(\"Chunk \", chunk)\n    for _type in ['click','cart','order']:\n        test_candidates = pd.read_parquet(f\"/kaggle/input/otto-candidate-features/{_type}_features_{chunk}.pqt\")\n        FEATURES = [f for f in test_candidates.columns[2:] if f != 'gt']\n\n        preds = np.zeros(len(test_candidates))\n        for fold in range(5):\n            model = xgb.Booster()\n            model.load_model(f'XGB_fold{fold}_{_type}.xgb')\n            if USE_TPU:\n                model.set_param({'predictor': 'cpu_predictor'})\n            else:\n                model.set_param({'predictor': 'gpu_predictor'})\n            dtest = xgb.DMatrix(data=test_candidates[FEATURES])\n            preds += model.predict(dtest)/5\n\n            del model, dtest\n            gc.collect()\n        predictions = test_candidates[['session','aid']].copy()\n        predictions['pred'] = preds\n\n        predictions = predictions.sort_values(['session','pred'], ascending=[True,False]).reset_index(drop=True)\n        predictions['n'] = predictions.groupby('session').aid.cumcount().astype('int32')\n        predictions = predictions.loc[predictions.n<20]\n        sub = predictions.groupby('session').aid.apply(list)\n        sub = sub.to_frame().reset_index()\n        sub.columns = ['session', 'labels']\n\n        del test_candidates, predictions\n        gc.collect()\n\n        test_labels = full_test_labels.loc[full_test_labels['type']==(_type + \"s\")]\n        test_labels = test_labels.merge(sub, how='inner', on=['session'])\n        test_labels['hits'] = test_labels.apply(lambda df: len(set(df.ground_truth).intersection(set(df.labels))), axis=1)\n        test_labels['gt_count'] = test_labels.ground_truth.str.len().clip(0,20)\n\n        recall[_type] = test_labels['hits'].sum() / test_labels['gt_count'].sum()\n        print(f\"Recall type {_type}: {recall[_type]}\")\n\n        del test_labels, sub\n        gc.collect()\n\n    score = 0\n    for key in recall.keys():\n        score += recall[key]*weights[key]\n    print(\"Final score:\", score)","metadata":{"execution":{"iopub.status.busy":"2023-01-27T04:38:59.994318Z","iopub.execute_input":"2023-01-27T04:38:59.994715Z","iopub.status.idle":"2023-01-27T04:44:02.558710Z","shell.execute_reply.started":"2023-01-27T04:38:59.994659Z","shell.execute_reply":"2023-01-27T04:44:02.557814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pred","metadata":{}},{"cell_type":"code","source":"def add_top_common(pred, _type):\n    if len(pred) < 20:\n        top = [aid for aid in top_common_test[_type] if aid not in pred]\n        pred = pred + top[:20-len(pred)]\n    return pred\n\ntype_labels = {'clicks':0, 'carts':1, 'orders':2}\ndef load_test(path):    \n    dfs = []\n    for e, chunk_file in sorted(enumerate(glob.glob(path))):\n        chunk = pd.read_parquet(chunk_file)\n        chunk.ts = (chunk.ts/1000).astype('int32')\n        chunk['type'] = chunk['type'].map(type_labels).astype('int8')\n        dfs.append(chunk)\n    return pd.concat(dfs).reset_index(drop=True) #.astype({\"ts\": \"datetime64[ms]\"})\n\ntest_df = load_test('../input/otto-chunk-data-inparquet-format/test_parquet/*')\nprint('Test data has shape',test_df.shape)\n# test.head()\ntop_common_test = dict()\ntop_common_test['click'] = list(test_df.loc[test_df['type']== 0,'aid'].value_counts().index.values[:20]) \ntop_common_test['cart'] = list(test_df.loc[test_df['type']== 1,'aid'].value_counts().index.values[:20])\ntop_common_test['order'] = list(test_df.loc[test_df['type']== 2,'aid'].value_counts().index.values[:20])\ndel test_df\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npreds_df_list = []\nfor _type in ['click','cart','order']:\n    print(f\"{_type}\", end=',')\n    \n#     if _type == 'order':\n#         files = sorted(glob.glob(f\"/kaggle/input/fork-of-otto-feature-engineering-test-order/{_type}_features*.pqt\"))\n#     else:\n    files = sorted(glob.glob(f\"/kaggle/input/otto-candidate-features-test/{_type}_features*.pqt\"))\n    preds_df4type_list = []\n    for file in files:\n        print(f\"file:{file.split('/')[-1]}\", end='...')\n        test_candidates = pd.read_parquet(file)\n        FEATURES = test_candidates.columns[2:]\n        print(\"Number of features: \", len(FEATURES))\n        preds = np.zeros(len(test_candidates))\n        for fold in range(5):\n            model = xgb.Booster()\n            model.load_model(f'/kaggle/working/XGB_fold{fold}_{_type}.xgb')\n            if USE_TPU:\n                model.set_param({'predictor': 'cpu_predictor'})\n            else:\n                model.set_param({'predictor': 'gpu_predictor'})\n            dtest = xgb.DMatrix(data=test_candidates[FEATURES])\n            preds += model.predict(dtest)/5\n\n            del model, dtest\n            gc.collect()\n        predictions = test_candidates[['session','aid']].copy()\n        predictions['pred'] = preds\n\n        predictions = predictions.sort_values(['session','pred'], ascending=[True,False]).reset_index(drop=True)\n        predictions['n'] = predictions.groupby('session').aid.cumcount().astype('int32')\n        predictions = predictions.loc[predictions.n<20]\n        sub = predictions.groupby('session').aid.apply(list)\n        sub = sub.to_frame().reset_index()\n        sub.columns = ['session_type','labels']\n        sub['labels'] = sub['labels'].apply(lambda x: add_top_common(x, _type))\n        sub.labels = sub.labels.apply(lambda x: \" \".join(map(str,x)))\n        sub.session_type = sub.session_type.astype('str')+ \"_\" + _type + \"s\"\n        preds_df4type_list.append(sub)\n                             \n        del test_candidates, predictions\n        gc.collect()\n        print(\"Done!\")\n    \n    preds_df4type = pd.concat(preds_df4type_list, ignore_index=True, axis=0)\n    preds_df_list.append(preds_df4type)\n    del preds_df4type_list\n    gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pd.concat(preds_df_list, ignore_index=True, axis=0)\npred_df.to_csv(\"submission.csv\", index=False)\nprint(pred_df.shape)\npred_df.head()","metadata":{},"execution_count":null,"outputs":[]}]}