{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install polars","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:50:00.03025Z","iopub.execute_input":"2023-06-05T13:50:00.030812Z","iopub.status.idle":"2023-06-05T13:50:09.026302Z","shell.execute_reply.started":"2023-06-05T13:50:00.030689Z","shell.execute_reply":"2023-06-05T13:50:09.025233Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Requirement already satisfied: polars in /opt/conda/lib/python3.7/site-packages (0.18.0)\nRequirement already satisfied: typing_extensions>=4.0.1 in /opt/conda/lib/python3.7/site-packages (from polars) (4.1.1)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"import gc\nimport polars as pl","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:50:09.029297Z","iopub.execute_input":"2023-06-05T13:50:09.029702Z","iopub.status.idle":"2023-06-05T13:50:09.0755Z","shell.execute_reply.started":"2023-06-05T13:50:09.029663Z","shell.execute_reply":"2023-06-05T13:50:09.074467Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"train = pl.read_parquet(\"../input/otto-train-and-test-data-for-local-validation/train.parquet\")\nvalid = pl.read_parquet(\"../input/otto-train-and-test-data-for-local-validation/test.parquet\")\nvalid_labels = pl.read_parquet(\"../input/otto-train-and-test-data-for-local-validation/test_labels.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:50:09.077083Z","iopub.execute_input":"2023-06-05T13:50:09.07739Z","iopub.status.idle":"2023-06-05T13:50:13.557223Z","shell.execute_reply.started":"2023-06-05T13:50:09.077363Z","shell.execute_reply":"2023-06-05T13:50:13.556417Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"markdown","source":"# Generating the item features","metadata":{}},{"cell_type":"code","source":"# {'aid':'count', 'session':'nunique', 'type': 'mean'}\nitem_features = pl.concat([train, valid]).groupby('aid').agg([\n pl.count(\"aid\").alias(\"item_item_count\"), \n pl.n_unique(\"session\").alias(\"item_user_count\"), \n pl.mean(\"type\").alias(\"item_buy_ratio\").cast(pl.Float32)\n])\nitem_features.write_parquet('item_features.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:50:13.55813Z","iopub.execute_input":"2023-06-05T13:50:13.558476Z","iopub.status.idle":"2023-06-05T13:50:25.445315Z","shell.execute_reply.started":"2023-06-05T13:50:13.55844Z","shell.execute_reply":"2023-06-05T13:50:25.444338Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"item_features","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generating the user features","metadata":{}},{"cell_type":"code","source":"# {'session':'count','aid':'nunique','type':'mean'}\nuser_features = valid.groupby('session').agg([\n pl.count(\"session\").alias(\"user_user_count\"),\n pl.n_unique(\"aid\").alias(\"user_item_count\"),\n pl.mean(\"type\").alias(\"user_buy_ratio\").cast(pl.Float32)\n])\nuser_features.write_parquet('user_features.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:50:25.447471Z","iopub.execute_input":"2023-06-05T13:50:25.447781Z","iopub.status.idle":"2023-06-05T13:50:26.416219Z","shell.execute_reply.started":"2023-06-05T13:50:25.447753Z","shell.execute_reply":"2023-06-05T13:50:26.414958Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"user_features","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"event_paths = {\n \"clicks\": \"../input/train-577-score/top_20_clicks*\",\n \"carts\": \"../input/train-577-score/top_15_carts_orders*\",\n \"buys\": \"../input/train-577-score/top_15_buy2buy*\"\n}","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:50:26.417943Z","iopub.execute_input":"2023-06-05T13:50:26.418241Z","iopub.status.idle":"2023-06-05T13:50:26.423875Z","shell.execute_reply.started":"2023-06-05T13:50:26.418215Z","shell.execute_reply":"2023-06-05T13:50:26.422853Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"df = pl.read_parquet('../input/train-577-score/top_20_clicks*')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:50:56.09742Z","iopub.execute_input":"2023-06-05T13:50:56.098146Z","iopub.status.idle":"2023-06-05T13:50:56.684957Z","shell.execute_reply.started":"2023-06-05T13:50:56.098081Z","shell.execute_reply":"2023-06-05T13:50:56.683361Z"},"trusted":true},"execution_count":9,"outputs":[{"execution_count":9,"output_type":"execute_result","data":{"text/plain":"shape: (5, 4)\n┌───────┬─────────┬───────────┬───────────────────┐\n│ aid_x ┆ aid_y ┆ wgt ┆ __index_level_0__ │\n│ --- ┆ --- ┆ --- ┆ --- │\n│ i32 ┆ i32 ┆ f32 ┆ i64 │\n╞═══════╪═════════╪═══════════╪═══════════════════╡\n│ 0 ┆ 532042 ┆ 16.182125 ┆ 0 │\n│ 0 ┆ 643097 ┆ 14.165738 ┆ 1 │\n│ 0 ┆ 1848174 ┆ 13.299606 ┆ 2 │\n│ 0 ┆ 1735605 ┆ 8.174276 ┆ 3 │\n│ 0 ┆ 1363081 ┆ 5.821229 ┆ 4 │\n└───────┴─────────┴───────────┴───────────────────┘","text/html":"
\nshape: (5, 4)
aid_xaid_ywgt__index_level_0__
i32i32f32i64
053204216.1821250
064309714.1657381
0184817413.2996062
017356058.1742763
013630815.8212294
"},"metadata":{}}]},{"cell_type":"code","source":"del train, valid\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:50:26.425022Z","iopub.execute_input":"2023-06-05T13:50:26.425348Z","iopub.status.idle":"2023-06-05T13:50:26.672384Z","shell.execute_reply.started":"2023-06-05T13:50:26.425322Z","shell.execute_reply":"2023-06-05T13:50:26.671253Z"},"trusted":true},"execution_count":7,"outputs":[{"execution_count":7,"output_type":"execute_result","data":{"text/plain":"171"},"metadata":{}}]},{"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.model_selection import GroupKFold\n\n\ndef train_ranker(event, df_cands, n_splits=5):\n \n skf = GroupKFold(n_splits=n_splits)\n FEATURES = [\n 'session', 'item_item_count', 'item_user_count', 'item_buy_ratio', 'user_user_count', 'user_item_count', 'user_buy_ratio'\n ]\n TARGET = \"target\"\n for fold,(train_idx, valid_idx) in enumerate(skf.split(df_cands, df_cands['target'], groups=df_cands['session'])):\n\n X_train = df_cands.loc[train_idx, FEATURES]\n y_train = df_cands.loc[train_idx, TARGET]\n X_valid = df_cands.loc[valid_idx, FEATURES]\n y_valid = df_cands.loc[valid_idx, TARGET]\n\n X_train = X_train.sort_values(\"session\").reset_index(drop=True)\n X_valid = X_valid.sort_values(\"session\").reset_index(drop=True)\n\n train_group = X_train.groupby('session').session.agg('count').values\n valid_group = X_valid.groupby('session').session.agg('count').values\n\n #X_train = X_train.drop([\"session\"], axis=1)\n #X_valid = X_valid.drop([\"session\"], axis=1)\n\n dtrain = xgb.DMatrix(X_train, y_train, group=train_group) # [50] * (len(train_idx)//50) \n dvalid = xgb.DMatrix(X_valid, y_valid, group=valid_group) # [50] * (len(valid_idx)//50)\n xgb_parms = {\n 'objective':'rank:pairwise', \n 'tree_method':'hist',\n 'random_state': 42, \n 'learning_rate': 0.1,\n \"colsample_bytree\": 0.8, \n 'eta': 0.05, \n 'max_depth': 6,\n 'subsample': 0.75,\n # n_estimators=110,\n }\n model = xgb.train(\n xgb_parms, \n dtrain=dtrain,\n evals=[(dtrain,'train'), (dvalid,'valid')],\n num_boost_round=100,\n verbose_eval=20\n )\n model.save_model(f'XGB_fold{fold}_{event}.xgb')\n #gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtrain","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the Ranker","metadata":{}},{"cell_type":"code","source":"NEGATIVE_FRAC = 0.15\n\nfor event, path in event_paths.items():\n print(f\"Started ranking model for: {event}\")\n # Reading the candidates\n df_cands = pl.read_parquet(path)\n df_cands = df_cands.explode(\"candidates\").with_columns([\n pl.col(\"session\").cast(pl.Int32),\n pl.col(\"candidates\").cast(pl.Int32).alias(\"aid\")\n ]).drop(\"candidates\").unique(subset=[\"session\", \"aid\"])\n \n # Joining the item features\n df_cands = df_cands.join(item_features, on='aid', how='left').fill_nan(-1)\n # Joining the user features\n df_cands = df_cands.join(user_features, on='session', how='left').fill_nan(-1)\n cand_labels = valid_labels.filter(valid_labels[\"type\"] == event).explode(\"ground_truth\").with_columns([\n pl.col(\"session\").cast(pl.Int32),\n pl.col(\"ground_truth\").cast(pl.Int32)# .alias(\"aid\")\n ]).rename({\"ground_truth\": \"aid\"})\n cand_labels = cand_labels.with_column(pl.lit(1).alias(\"target\").cast(pl.Int8)).drop(\"type\")\n # Joining the labels\n df_cands = df_cands.join(cand_labels, on=[\"session\", \"aid\"], how=\"left\").fill_null(0)\n # Negative sampling\n df_cands = pl.concat([\n df_cands.filter(df_cands[\"target\"] == 0).sample(frac=NEGATIVE_FRAC, seed=42),\n df_cands.filter(df_cands[\"target\"] == 1)\n ])\n print(df_cands.groupby(\"target\").agg(pl.count()))\n df_cands = df_cands.to_pandas()\n df_cands = df_cands.sort_values(\"session\").reset_index(drop=True)\n print(f\"Event: {event} - started training...\")\n\n train_ranker(event, df_cands)\n del df_cands, cand_labels\n gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:50:26.75046Z","iopub.execute_input":"2023-06-05T13:50:26.751372Z","iopub.status.idle":"2023-06-05T13:50:27.372034Z","shell.execute_reply.started":"2023-06-05T13:50:26.751337Z","shell.execute_reply":"2023-06-05T13:50:27.37064Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stdout","text":"Started ranking model for: clicks\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mColumnNotFoundError\u001b[0m Traceback (most recent call last)","\u001b[0;32m/tmp/ipykernel_407/86103174.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;31m# Reading the candidates\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mdf_cands\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_parquet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m df_cands = df_cands.explode(\"candidates\").with_columns([\n\u001b[0m\u001b[1;32m 8\u001b[0m \u001b[0mpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"session\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mInt32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"candidates\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mInt32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0malias\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"aid\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/polars/dataframe/frame.py\u001b[0m in \u001b[0;36mexplode\u001b[0;34m(self, columns, *more_columns)\u001b[0m\n\u001b[1;32m 6291\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6292\u001b[0m \"\"\"\n\u001b[0;32m-> 6293\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlazy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexplode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmore_columns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mno_optimization\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6294\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6295\u001b[0m def pivot(\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/polars/lazyframe/frame.py\u001b[0m in \u001b[0;36mcollect\u001b[0;34m(self, type_coercion, predicate_pushdown, projection_pushdown, simplify_expression, no_optimization, slice_pushdown, common_subplan_elimination, streaming)\u001b[0m\n\u001b[1;32m 1499\u001b[0m \u001b[0mstreaming\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1500\u001b[0m )\n\u001b[0;32m-> 1501\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mwrap_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mldf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1502\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1503\u001b[0m def sink_parquet(\n","\u001b[0;31mColumnNotFoundError\u001b[0m: candidates"],"ename":"ColumnNotFoundError","evalue":"candidates","output_type":"error"}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}