{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Using Polars\n\nI learnt `polars` from last OTTO competition, which helped our team to do fast feature engineering when we faced `cudf`'s GPU memory errors.\n\nIn this competition, although the data size is not as large as OTTO's, it is still convenient to use `polars` with cpu for easy train-inference adaption. Moreover, from my own experiment, the `polars` is faster than `pandas` in Kaggle's notebook even there is only 2-core CPU avalaible, which restricts the full strength of `polars` parallelism.\n\nAnd last, the current Kaggle environment (2023-02-17) has `polars` supported! You can save about 40s from installing it.","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport sys\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nfrom catboost import CatBoostClassifier, Pool\n\nlevel_groups = [\"0-4\", \"5-12\", \"13-22\"]\nlevel_groups_reverse = {'0-4': 0, '5-12': 1, '13-22': 2}\nlevels = {'0-4': (0, 5), '5-12': (5, 13), '13-22': (13, 23)}\nquestions = {'0-4': (1, 4), '5-12': (4, 14), '13-22': (14, 19)}\nEVENTS = ['checkpoint', 'cutscene_click', 'map_click', 'map_hover', 'navigate_click', 'notebook_click', 'notification_click', 'object_click', 'object_hover', 'observation_click', 'person_click']\nCATS = ['event_name', 'name','fqid', 'room_fqid', 'text_fqid']\nNUMS = ['elapsed_time','level','page','room_coor_x', 'room_coor_y', 'screen_coor_x', 'screen_coor_y', 'hover_duration', \"time_past\"]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-19T15:23:42.055085Z","iopub.execute_input":"2023-02-19T15:23:42.055554Z","iopub.status.idle":"2023-02-19T15:23:43.055265Z","shell.execute_reply.started":"2023-02-19T15:23:42.055454Z","shell.execute_reply":"2023-02-19T15:23:43.054177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = [\n#     pl.col(\"page\").cast(pl.Float32),\n    (\n        (pl.col(\"elapsed_time\") - pl.col(\"elapsed_time\").shift(1))\n         .fill_null(0)\n         .clip(0, 1e9)\n         .over([\"session_id\", \"level_group\"])\n         .alias(\"time_past\")\n    ),\n]\naggs = [\n    *[pl.col(c).drop_nulls().n_unique().alias(f\"{c}_unique\") for c in CATS],\n    *[pl.col(c).mean().alias(f\"{c}_mean\") for c in NUMS],\n    *[pl.col(c).std().alias(f\"{c}_std\") for c in NUMS],\n    *[(pl.col(\"event_name\") == c).sum().alias(f\"{c}_sum\") for c in EVENTS],\n]\nmodels_list = [[CatBoostClassifier().load_model(\n    f\"/kaggle/input/cpu-catboost-baseline-using-polars-train/fold{fold}_q{q}.cbm\"\n) for fold in range(5)] for q in range(1, 19)]","metadata":{"execution":{"iopub.status.busy":"2023-02-19T15:23:43.057071Z","iopub.execute_input":"2023-02-19T15:23:43.057469Z","iopub.status.idle":"2023-02-19T15:23:43.862347Z","shell.execute_reply.started":"2023-02-19T15:23:43.057428Z","shell.execute_reply":"2023-02-19T15:23:43.861097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def trans(test):\n    return (pl.from_pandas(test)    \n            .with_columns(columns)\n            .select(aggs)\n            .fill_null(-1)\n            .to_pandas()\n            )","metadata":{"execution":{"iopub.status.busy":"2023-02-19T15:23:43.863761Z","iopub.execute_input":"2023-02-19T15:23:43.864107Z","iopub.status.idle":"2023-02-19T15:23:43.868961Z","shell.execute_reply.started":"2023-02-19T15:23:43.864078Z","shell.execute_reply":"2023-02-19T15:23:43.867703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T15:23:43.871041Z","iopub.execute_input":"2023-02-19T15:23:43.871600Z","iopub.status.idle":"2023-02-19T15:23:43.943267Z","shell.execute_reply.started":"2023-02-19T15:23:43.871565Z","shell.execute_reply":"2023-02-19T15:23:43.942237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (sample_submission, test) in iter_test:\n    target_level_group = level_groups_reverse[test.level_group.iloc[0]]\n    df = trans(test)\n    \n    fold = 0\n    preds = []\n    for q in range(*questions[level_groups[target_level_group]]):\n        model = models_list[q - 1][fold]\n        feature_cols = model.feature_names_\n        pred = model.predict_proba(df[feature_cols].astype(np.float32))[0,1]\n        preds.append(int(pred > 0.63))\n\n    sample_submission[\"correct\"] = preds\n\n    env.predict(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-02-19T15:23:43.944546Z","iopub.execute_input":"2023-02-19T15:23:43.945096Z","iopub.status.idle":"2023-02-19T15:23:44.482369Z","shell.execute_reply.started":"2023-02-19T15:23:43.945063Z","shell.execute_reply":"2023-02-19T15:23:44.481126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('submission.csv')\nprint(sub.shape, sub.correct.mean())\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T15:23:44.483542Z","iopub.execute_input":"2023-02-19T15:23:44.483866Z","iopub.status.idle":"2023-02-19T15:23:44.506302Z","shell.execute_reply.started":"2023-02-19T15:23:44.483835Z","shell.execute_reply":"2023-02-19T15:23:44.505298Z"},"trusted":true},"execution_count":null,"outputs":[]}]}