{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# cleaned version","metadata":{}},{"cell_type":"markdown","source":"In this notebook I would like to reimplement train_valid as it is in the organizer's [scripts](https://github.com/otto-de/recsys-dataset/blob/main/src/testset.py).","metadata":{}},{"cell_type":"code","source":"!pip install polars","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:04:14.560840Z","iopub.execute_input":"2023-01-12T15:04:14.561365Z","iopub.status.idle":"2023-01-12T15:04:30.158534Z","shell.execute_reply.started":"2023-01-12T15:04:14.561267Z","shell.execute_reply":"2023-01-12T15:04:30.157106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"pipenv run python -m src.testset --train-set train.jsonl --days 2 --output-path 'out/' --seed 42 ","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport pandas as pd\nimport random","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:04:30.160755Z","iopub.execute_input":"2023-01-12T15:04:30.161119Z","iopub.status.idle":"2023-01-12T15:04:30.224856Z","shell.execute_reply.started":"2023-01-12T15:04:30.161085Z","shell.execute_reply":"2023-01-12T15:04:30.223696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\npd.set_option('display.max_colwidth', None)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:04:30.226518Z","iopub.execute_input":"2023-01-12T15:04:30.227235Z","iopub.status.idle":"2023-01-12T15:04:30.233797Z","shell.execute_reply.started":"2023-01-12T15:04:30.227170Z","shell.execute_reply":"2023-01-12T15:04:30.232655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Radek's validation set is using the older version of organizer's script (there is a little cold start problem on aid)\ntrain_v = pl.scan_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/train.parquet')\ntest_v = pl.scan_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/test.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:04:30.236857Z","iopub.execute_input":"2023-01-12T15:04:30.237619Z","iopub.status.idle":"2023-01-12T15:04:30.264287Z","shell.execute_reply.started":"2023-01-12T15:04:30.237586Z","shell.execute_reply":"2023-01-12T15:04:30.263020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I created this validation set on paperspace and optimized on Kaggle\ntrain_v7 = pl.scan_parquet('/kaggle/input/ottovalidation7days/train_sessions.parquet')\ntest_v7 = pl.scan_parquet('/kaggle/input/ottovalidation7days/test_sessions.parquet')\ntest_v7_full = pl.scan_parquet('/kaggle/input/ottovalidation7days/test_sessions_full.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:04:30.266148Z","iopub.execute_input":"2023-01-12T15:04:30.266559Z","iopub.status.idle":"2023-01-12T15:04:30.287431Z","shell.execute_reply.started":"2023-01-12T15:04:30.266525Z","shell.execute_reply":"2023-01-12T15:04:30.286286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I created this validation set on paperspace and optimized on Kaggle too\ntrain_v7_2nd = pl.scan_parquet('/kaggle/input/ottovalidation7days2nd/train_sessions.parquet')\ntest_v7_2nd = pl.scan_parquet('/kaggle/input/ottovalidation7days2nd/test_sessions.parquet')\ntest_v7_full_2nd = pl.scan_parquet('/kaggle/input/ottovalidation7days2nd/test_sessions_full.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:04:30.288752Z","iopub.execute_input":"2023-01-12T15:04:30.289085Z","iopub.status.idle":"2023-01-12T15:04:30.310807Z","shell.execute_reply.started":"2023-01-12T15:04:30.289054Z","shell.execute_reply":"2023-01-12T15:04:30.309552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I created this validation set on Kaggle and optimized on Kaggle too\ntrain_v7_3rd = pl.scan_parquet('/kaggle/input/otto-validation-optimized-parquet/train_sessions.parquet')\ntest_v7_3rd = pl.scan_parquet('/kaggle/input/otto-validation-optimized-parquet/test_sessions.parquet')\ntest_v7_full_3rd = pl.scan_parquet('/kaggle/input/otto-validation-optimized-parquet/test_sessions_full.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:04:30.312307Z","iopub.execute_input":"2023-01-12T15:04:30.312904Z","iopub.status.idle":"2023-01-12T15:04:30.340903Z","shell.execute_reply.started":"2023-01-12T15:04:30.312869Z","shell.execute_reply":"2023-01-12T15:04:30.339649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# create validation 7 days from train set (4 weeks)","metadata":{}},{"cell_type":"code","source":"train_ms = pl.scan_parquet('/kaggle/input/otto-radek-style-polars/train_ms.parquet')\ntest_ms = pl.scan_parquet('/kaggle/input/otto-radek-style-polars/test_ms.parquet')\n\nsplit_ts_ms = train_ms.select([\n    (pl.col('ts').max() - 7*24*60*60*1000).alias('split_ts')\n]).collect().to_series().to_list()[0]\n\ntrain_valid = (\n    train_ms\n    .filter(~(pl.col('ts').first() > split_ts_ms).over('session'))\n    .filter(pl.col('ts') < split_ts_ms)\n    .filter((pl.col('aid').count()>=2).over('session'))\n    .collect()\n)    ","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:04:40.020598Z","iopub.execute_input":"2023-01-12T15:04:40.021051Z","iopub.status.idle":"2023-01-12T15:04:54.152925Z","shell.execute_reply.started":"2023-01-12T15:04:40.021015Z","shell.execute_reply":"2023-01-12T15:04:54.151840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n(\n    train_ms # step 1: https://github.com/otto-de/recsys-dataset/blob/main/src/testset.py#L95-L97\n    .filter(~(pl.col('ts').first() > split_ts_ms).over('session'))\n    .filter(pl.col('ts') < split_ts_ms) # step 2: https://github.com/otto-de/recsys-dataset/blob/main/src/testset.py#L98\n    .filter((pl.col('aid').count()>=2).over('session')) # step 3: https://github.com/otto-de/recsys-dataset/blob/main/src/testset.py#L99\n    .select([\n        pl.col('session').n_unique().alias('total_sessions'),\n        pl.col('aid').n_unique().alias('total_aid'),    \n        pl.col('session').count().alias('total_rows'),       \n    ]).collect()\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:06:06.432419Z","iopub.execute_input":"2023-01-12T15:06:06.432934Z","iopub.status.idle":"2023-01-12T15:06:49.722852Z","shell.execute_reply.started":"2023-01-12T15:06:06.432885Z","shell.execute_reply":"2023-01-12T15:06:49.721608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:07:30.546353Z","iopub.execute_input":"2023-01-12T15:07:30.546784Z","iopub.status.idle":"2023-01-12T15:08:00.702924Z","shell.execute_reply.started":"2023-01-12T15:07:30.546753Z","shell.execute_reply":"2023-01-12T15:08:00.701602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}