{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# A cleaner version","metadata":{}},{"cell_type":"markdown","source":"In this notebook I would like to reimplement `test_sessions_full` as it is in the organizer's [scripts](https://github.com/otto-de/recsys-dataset/blob/main/src/testset.py).","metadata":{}},{"cell_type":"code","source":"!pip install polars\n\nimport polars as pl\nimport pandas as pd\nimport random\nfrom polars.testing import assert_frame_equal, assert_series_equal\n\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\npd.set_option('display.max_colwidth', None)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:21:10.382981Z","iopub.execute_input":"2023-01-12T15:21:10.386349Z","iopub.status.idle":"2023-01-12T15:21:27.520154Z","shell.execute_reply.started":"2023-01-12T15:21:10.385873Z","shell.execute_reply":"2023-01-12T15:21:27.518548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Radek's validation set is using the older version of organizer's script (there is a little cold start problem on aid)\ntrain_v = pl.scan_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/train.parquet')\ntest_v = pl.scan_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/test.parquet')\n\n# I created this validation set on paperspace and optimized on Kaggle\ntrain_v7 = pl.scan_parquet('/kaggle/input/ottovalidation7days/train_sessions.parquet')\ntest_v7 = pl.scan_parquet('/kaggle/input/ottovalidation7days/test_sessions.parquet')\ntest_v7_full = pl.scan_parquet('/kaggle/input/ottovalidation7days/test_sessions_full.parquet')\n\n# I created this validation set on paperspace and optimized on Kaggle too\ntrain_v7_2nd = pl.scan_parquet('/kaggle/input/ottovalidation7days2nd/train_sessions.parquet')\ntest_v7_2nd = pl.scan_parquet('/kaggle/input/ottovalidation7days2nd/test_sessions.parquet')\ntest_v7_full_2nd = pl.scan_parquet('/kaggle/input/ottovalidation7days2nd/test_sessions_full.parquet')\n\n# I created this validation set on Kaggle and optimized on Kaggle too\ntrain_v7_3rd = pl.scan_parquet('/kaggle/input/otto-validation-optimized-parquet/train_sessions.parquet')\ntest_v7_3rd = pl.scan_parquet('/kaggle/input/otto-validation-optimized-parquet/test_sessions.parquet')\ntest_v7_full_3rd = pl.scan_parquet('/kaggle/input/otto-validation-optimized-parquet/test_sessions_full.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:21:27.524420Z","iopub.execute_input":"2023-01-12T15:21:27.524758Z","iopub.status.idle":"2023-01-12T15:21:27.606288Z","shell.execute_reply.started":"2023-01-12T15:21:27.524725Z","shell.execute_reply":"2023-01-12T15:21:27.605318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ms = pl.scan_parquet('/kaggle/input/otto-radek-style-polars/train_ms.parquet')\ntest_ms = pl.scan_parquet('/kaggle/input/otto-radek-style-polars/test_ms.parquet')\n\nsplit_ts_ms = train_ms.select([\n    (pl.col('ts').max() - 7*24*60*60*1000).alias('split_ts')\n]).collect().to_series().to_list()[0]\n\n# building `train_items` as it is in organizer's script https://github.com/otto-de/recsys-dataset/blob/main/src/testset.py#L100\nunique_aids_train_valid = (\n    train_ms\n    .filter(~(pl.col('ts').first() > split_ts_ms).over('session'))\n    .filter(pl.col('ts') < split_ts_ms)\n    .filter((pl.col('aid').count()>=2).over('session'))\n    .select([\n        pl.col('aid').unique().alias('unique_aids')\n    ]).collect().to_series().to_list()\n)\nlen(unique_aids_train_valid) # 1825325\n\ntest_sessions_full = (\n    train_ms\n    .filter((pl.col('ts').first() > split_ts_ms).over('session')).collect() # stop being lazy early can avoid error and wrong answers\n    .filter(pl.col('aid').is_in(unique_aids_train_valid)) # step 2-2: https://github.com/otto-de/recsys-dataset/blob/main/src/testset.py#L75\n    .filter((pl.col('aid').count()>=2).over('session')) # step 3: https://github.com/otto-de/recsys-dataset/blob/main/src/testset.py#L76\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:21:27.610752Z","iopub.execute_input":"2023-01-12T15:21:27.611158Z","iopub.status.idle":"2023-01-12T15:23:05.438169Z","shell.execute_reply.started":"2023-01-12T15:21:27.611113Z","shell.execute_reply":"2023-01-12T15:23:05.436815Z"},"trusted":true},"execution_count":null,"outputs":[]}]}