{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# The goal\n\nJust to implement Radek's last 20 aids notebook https://www.kaggle.com/code/radek1/last-20-aids in polars\n\nInstead of using Radek's otto-full-optimized-memory-footprint dataset, I am using otto-radek-style-polars dataset. The only difference is that I have included milliseconds datast with seconds dataset. I am using milliseconds dataset here.","metadata":{}},{"cell_type":"markdown","source":"Grabbing the last 20 aid for a session and using them as prediction.","metadata":{}},{"cell_type":"code","source":"!pip install polars\n\nimport polars as pl\nimport pandas as pd\nimport random\nfrom polars.testing import assert_frame_equal, assert_series_equal\nfrom datetime import datetime\n\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\npd.set_option('display.max_colwidth', None)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:37:02.497128Z","iopub.execute_input":"2023-01-15T03:37:02.498631Z","iopub.status.idle":"2023-01-15T03:37:15.022513Z","shell.execute_reply.started":"2023-01-15T03:37:02.498587Z","shell.execute_reply":"2023-01-15T03:37:15.021114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg = pl.Config.restore_defaults()  \npl.Config.set_tbl_rows(10)  \npl.Config.set_fmt_str_lengths(1000)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:37:15.025995Z","iopub.execute_input":"2023-01-15T03:37:15.026568Z","iopub.status.idle":"2023-01-15T03:37:15.040754Z","shell.execute_reply.started":"2023-01-15T03:37:15.026514Z","shell.execute_reply":"2023-01-15T03:37:15.039085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ms = pl.scan_parquet('/kaggle/input/otto-radek-style-polars/train_ms.parquet')\ntest_ms = pl.scan_parquet('/kaggle/input/otto-radek-style-polars/test_ms.parquet')\nsample_sub = pl.read_csv('../input/otto-recommender-system/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:37:15.042718Z","iopub.execute_input":"2023-01-15T03:37:15.043367Z","iopub.status.idle":"2023-01-15T03:37:15.768890Z","shell.execute_reply.started":"2023-01-15T03:37:15.043314Z","shell.execute_reply":"2023-01-15T03:37:15.767100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# grab the last 20 aids for each test session","metadata":{}},{"cell_type":"code","source":"joined = (\n    test_ms\n    .with_columns([\n        pl.col('aid').cumcount().over('session').alias('idx_inside_each_sess'),\n        pl.col('aid').count().over('session').alias('tot_rows_each_sess')\n    ])\n    .sort(['session', 'ts'], reverse=[False, False]) # corrected\n    .groupby('session')\n    .tail(20)\n    .select(pl.exclude(['idx_inside_each_sess', 'tot_rows_each_sess']))\n    .groupby('session')\n    .agg([\n        pl.col('aid'),\n    ])\n    .with_columns([\n        pl.col('aid').arr.eval(pl.element().cast(pl.Utf8)).arr.join(' ').alias('labels')\n    ])\n    .select(pl.exclude('aid'))\n    .with_columns([\n        (pl.col('session').cast(pl.Utf8) + '_clicks').alias('sess_click'),\n        (pl.col('session').cast(pl.Utf8) + '_carts').alias('sess_cart'),\n        (pl.col('session').cast(pl.Utf8) + '_orders').alias('sess_order'),        \n    ])\n    .collect()\n)\njoined.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:37:15.770989Z","iopub.execute_input":"2023-01-15T03:37:15.771765Z","iopub.status.idle":"2023-01-15T03:37:23.026511Z","shell.execute_reply.started":"2023-01-15T03:37:15.771719Z","shell.execute_reply":"2023-01-15T03:37:23.025324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\"141736 199008 57315 194067 199008 199008 199008 199008 199008 199008 918667\" \"12899781_clicks\"\n\n","metadata":{}},{"cell_type":"code","source":"part1 = (\n    joined\n    .select([\n        pl.col('sess_click').alias('session_type'),\n        'labels',\n        'session'\n    ])\n)\npart1.head()\npart1.height*3","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:37:23.030704Z","iopub.execute_input":"2023-01-15T03:37:23.031601Z","iopub.status.idle":"2023-01-15T03:37:23.044722Z","shell.execute_reply.started":"2023-01-15T03:37:23.031557Z","shell.execute_reply":"2023-01-15T03:37:23.043492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"part2 = (\n    joined\n    .select([\n        pl.col('sess_cart').alias('session_type'),\n        'labels',\n        'session'\n    ])\n)\npart2.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:37:23.046800Z","iopub.execute_input":"2023-01-15T03:37:23.047792Z","iopub.status.idle":"2023-01-15T03:37:23.057656Z","shell.execute_reply.started":"2023-01-15T03:37:23.047748Z","shell.execute_reply":"2023-01-15T03:37:23.056499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"part3 = (\n    joined\n    .select([\n        pl.col('sess_order').alias('session_type'),\n        'labels',\n        'session'\n    ])\n)\npart3.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:37:23.059110Z","iopub.execute_input":"2023-01-15T03:37:23.059832Z","iopub.status.idle":"2023-01-15T03:37:23.076191Z","shell.execute_reply.started":"2023-01-15T03:37:23.059786Z","shell.execute_reply":"2023-01-15T03:37:23.074348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = (\n    pl.concat([part1, part2, part3])\n    .sort('session')\n    .select(pl.exclude('session'))\n\n)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:37:23.078278Z","iopub.execute_input":"2023-01-15T03:37:23.079487Z","iopub.status.idle":"2023-01-15T03:37:24.490965Z","shell.execute_reply.started":"2023-01-15T03:37:23.079431Z","shell.execute_reply":"2023-01-15T03:37:24.489485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:37:24.493029Z","iopub.execute_input":"2023-01-15T03:37:24.493490Z","iopub.status.idle":"2023-01-15T03:37:24.501953Z","shell.execute_reply.started":"2023-01-15T03:37:24.493453Z","shell.execute_reply":"2023-01-15T03:37:24.500377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.write_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:37:24.503799Z","iopub.execute_input":"2023-01-15T03:37:24.504860Z","iopub.status.idle":"2023-01-15T03:37:25.897424Z","shell.execute_reply.started":"2023-01-15T03:37:24.504815Z","shell.execute_reply":"2023-01-15T03:37:25.895809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pl.read_csv('submission.csv').head()","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:37:25.899296Z","iopub.execute_input":"2023-01-15T03:37:25.899766Z","iopub.status.idle":"2023-01-15T03:37:26.753164Z","shell.execute_reply.started":"2023-01-15T03:37:25.899726Z","shell.execute_reply":"2023-01-15T03:37:26.751654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Verify the difference between Radek's and mine","metadata":{}},{"cell_type":"code","source":"sub_radek = pl.read_csv('/kaggle/input/radek-pandas-last20aids-submission/submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:37:26.755070Z","iopub.execute_input":"2023-01-15T03:37:26.755841Z","iopub.status.idle":"2023-01-15T03:37:27.594572Z","shell.execute_reply.started":"2023-01-15T03:37:26.755805Z","shell.execute_reply":"2023-01-15T03:37:27.593351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_radek.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:37:27.596473Z","iopub.execute_input":"2023-01-15T03:37:27.597117Z","iopub.status.idle":"2023-01-15T03:37:27.614177Z","shell.execute_reply.started":"2023-01-15T03:37:27.597078Z","shell.execute_reply":"2023-01-15T03:37:27.612445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert_frame_equal(submission, sub_radek)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:37:27.618193Z","iopub.execute_input":"2023-01-15T03:37:27.619084Z","iopub.status.idle":"2023-01-15T03:37:27.740198Z","shell.execute_reply.started":"2023-01-15T03:37:27.619015Z","shell.execute_reply":"2023-01-15T03:37:27.738491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, they are the same.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}