{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"According to organizer's script, there should be no cold start aids in the test set. For more details please see discussion [here](https://www.kaggle.com/datasets/radek1/otto-train-and-test-data-for-local-validation/discussion/374405#2077900)","metadata":{}},{"cell_type":"markdown","source":"In this notebook, I will compare different versions of validation sets to find the correct version","metadata":{}},{"cell_type":"code","source":"!pip install polars","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:30:34.503819Z","iopub.execute_input":"2022-12-28T03:30:34.504510Z","iopub.status.idle":"2022-12-28T03:30:51.488011Z","shell.execute_reply.started":"2022-12-28T03:30:34.504376Z","shell.execute_reply":"2022-12-28T03:30:51.486662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"pipenv run python -m src.testset --train-set train.jsonl --days 2 --output-path 'out/' --seed 42 ","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport pandas as pd\nimport random","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:30:51.490659Z","iopub.execute_input":"2022-12-28T03:30:51.491107Z","iopub.status.idle":"2022-12-28T03:30:51.576205Z","shell.execute_reply.started":"2022-12-28T03:30:51.491064Z","shell.execute_reply":"2022-12-28T03:30:51.574261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:30:51.578275Z","iopub.execute_input":"2022-12-28T03:30:51.578713Z","iopub.status.idle":"2022-12-28T03:30:51.584398Z","shell.execute_reply.started":"2022-12-28T03:30:51.578678Z","shell.execute_reply":"2022-12-28T03:30:51.582633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # train = pl.scan_parquet('/kaggle/input/otto-radek-style-polars/train.parquet')\n\n\ntrain_v = pl.scan_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/train.parquet')\ntest_v = pl.scan_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/test.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:30:51.589551Z","iopub.execute_input":"2022-12-28T03:30:51.590164Z","iopub.status.idle":"2022-12-28T03:30:51.612083Z","shell.execute_reply.started":"2022-12-28T03:30:51.590115Z","shell.execute_reply":"2022-12-28T03:30:51.610249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_v7 = pl.scan_parquet('/kaggle/input/ottovalidation7days/train_sessions.parquet')\ntest_v7 = pl.scan_parquet('/kaggle/input/ottovalidation7days/test_sessions.parquet')\ntest_v7_full = pl.scan_parquet('/kaggle/input/ottovalidation7days/test_sessions_full.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:30:51.614879Z","iopub.execute_input":"2022-12-28T03:30:51.615643Z","iopub.status.idle":"2022-12-28T03:30:51.651261Z","shell.execute_reply.started":"2022-12-28T03:30:51.615577Z","shell.execute_reply":"2022-12-28T03:30:51.649795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_v7_2nd = pl.scan_parquet('/kaggle/input/ottovalidation7days2nd/train_sessions.parquet')\ntest_v7_2nd = pl.scan_parquet('/kaggle/input/ottovalidation7days2nd/test_sessions.parquet')\ntest_v7_full_2nd = pl.scan_parquet('/kaggle/input/ottovalidation7days2nd/test_sessions_full.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:30:51.652975Z","iopub.execute_input":"2022-12-28T03:30:51.653891Z","iopub.status.idle":"2022-12-28T03:30:51.678271Z","shell.execute_reply.started":"2022-12-28T03:30:51.653843Z","shell.execute_reply":"2022-12-28T03:30:51.677281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_v7_3rd = pl.scan_parquet('/kaggle/input/otto-validation-optimized-parquet/train_sessions.parquet')\ntest_v7_3rd = pl.scan_parquet('/kaggle/input/otto-validation-optimized-parquet/test_sessions.parquet')\ntest_v7_full_3rd = pl.scan_parquet('/kaggle/input/otto-validation-optimized-parquet/test_sessions_full.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:31:34.244048Z","iopub.execute_input":"2022-12-28T03:31:34.244515Z","iopub.status.idle":"2022-12-28T03:31:34.268466Z","shell.execute_reply.started":"2022-12-28T03:31:34.244479Z","shell.execute_reply":"2022-12-28T03:31:34.266669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_v.select([\n    pl.col('session').n_unique().alias('total_sessions'),\n    pl.col('aid').n_unique().alias('total_aid'),    \n    pl.col('session').count().alias('total_rows'),    \n    (pl.col('ts').cast(pl.Int64)*1000).min().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"min_datetime\"), \n    (pl.col('ts').cast(pl.Int64)*1000).max().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"max_datetime\"), \n    (pl.col('ts').cast(pl.Int64)*1000).first().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"first_datetime\"),     \n    (pl.col('ts').cast(pl.Int64)*1000).last().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"last_datetime\"),     \n]).collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:31:36.491131Z","iopub.execute_input":"2022-12-28T03:31:36.491600Z","iopub.status.idle":"2022-12-28T03:32:07.670642Z","shell.execute_reply.started":"2022-12-28T03:31:36.491565Z","shell.execute_reply":"2022-12-28T03:32:07.668966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_v7.select([\n    pl.col('session').n_unique().alias('total_sessions'),\n    pl.col('aid').n_unique().alias('total_aid'),    \n    pl.col('session').count().alias('total_rows'),      \n    pl.col('ts').min().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"min_datetime\"), \n    pl.col('ts').max().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"max_datetime\"), \n    pl.col('ts').first().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"first_datetime\"),     \n    pl.col('ts').last().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"last_datetime\"),     \n]).collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:32:07.672933Z","iopub.execute_input":"2022-12-28T03:32:07.674482Z","iopub.status.idle":"2022-12-28T03:32:36.013170Z","shell.execute_reply.started":"2022-12-28T03:32:07.674414Z","shell.execute_reply":"2022-12-28T03:32:36.011779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_v7_2nd.select([\n    pl.col('session').n_unique().alias('total_sessions'),\n    pl.col('aid').n_unique().alias('total_aid'),    \n    pl.col('session').count().alias('total_rows'),      \n    pl.col('ts').min().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"min_datetime\"), \n    pl.col('ts').max().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"max_datetime\"), \n    pl.col('ts').first().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"first_datetime\"),     \n    pl.col('ts').last().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"last_datetime\"),     \n]).collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:32:36.017895Z","iopub.execute_input":"2022-12-28T03:32:36.018331Z","iopub.status.idle":"2022-12-28T03:33:07.686764Z","shell.execute_reply.started":"2022-12-28T03:32:36.018280Z","shell.execute_reply":"2022-12-28T03:33:07.684364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_v7_3rd.select([\n    pl.col('session').n_unique().alias('total_sessions'),\n    pl.col('aid').n_unique().alias('total_aid'),    \n    pl.col('session').count().alias('total_rows'),      \n    pl.col('ts').min().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"min_datetime\"), \n    pl.col('ts').max().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"max_datetime\"), \n    pl.col('ts').first().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"first_datetime\"),     \n    pl.col('ts').last().cast(pl.Datetime).dt.with_time_unit('ms').alias(\"last_datetime\"),     \n]).collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:33:07.690331Z","iopub.execute_input":"2022-12-28T03:33:07.691911Z","iopub.status.idle":"2022-12-28T03:33:53.389826Z","shell.execute_reply.started":"2022-12-28T03:33:07.691856Z","shell.execute_reply":"2022-12-28T03:33:53.387490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# which validation set has cold start problem on aid","metadata":{}},{"cell_type":"markdown","source":"## validation set provided by Radek has cold start problem on 2% of aids","metadata":{}},{"cell_type":"code","source":"unique_aids_train = train_v.select([\n    pl.col('aid').unique().alias('unique_aids')\n]).collect().to_series().to_list()\nlen(unique_aids_train)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:43:07.514886Z","iopub.execute_input":"2022-12-28T03:43:07.515951Z","iopub.status.idle":"2022-12-28T03:43:16.787860Z","shell.execute_reply.started":"2022-12-28T03:43:07.515883Z","shell.execute_reply":"2022-12-28T03:43:16.786950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_aids_test = test_v.select([\n    pl.col('aid').unique().alias('unique_aids')\n]).collect().to_series().to_list()\nlen(unique_aids_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:43:29.031169Z","iopub.execute_input":"2022-12-28T03:43:29.031629Z","iopub.status.idle":"2022-12-28T03:43:29.904737Z","shell.execute_reply.started":"2022-12-28T03:43:29.031594Z","shell.execute_reply":"2022-12-28T03:43:29.899187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set(unique_aids_test).intersection(set(unique_aids_train)))","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:44:19.863319Z","iopub.execute_input":"2022-12-28T03:44:19.863742Z","iopub.status.idle":"2022-12-28T03:44:20.231224Z","shell.execute_reply.started":"2022-12-28T03:44:19.863707Z","shell.execute_reply":"2022-12-28T03:44:20.230130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set(unique_aids_test).intersection(set(unique_aids_train)))/len(unique_aids_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:45:01.055346Z","iopub.execute_input":"2022-12-28T03:45:01.055807Z","iopub.status.idle":"2022-12-28T03:45:01.490562Z","shell.execute_reply.started":"2022-12-28T03:45:01.055769Z","shell.execute_reply":"2022-12-28T03:45:01.489312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## my validation set (3rd) has no cold start problem on aids","metadata":{}},{"cell_type":"code","source":"unique_aids_train = train_v7_3rd.select([\n    pl.col('aid').unique().alias('unique_aids')\n]).collect().to_series().to_list()\nlen(unique_aids_train)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:46:35.594864Z","iopub.execute_input":"2022-12-28T03:46:35.595360Z","iopub.status.idle":"2022-12-28T03:46:49.294128Z","shell.execute_reply.started":"2022-12-28T03:46:35.595319Z","shell.execute_reply":"2022-12-28T03:46:49.292622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_aids_test = test_v7_3rd.select([\n    pl.col('aid').unique().alias('unique_aids')\n]).collect().to_series().to_list()\nlen(unique_aids_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:47:10.470656Z","iopub.execute_input":"2022-12-28T03:47:10.471081Z","iopub.status.idle":"2022-12-28T03:47:11.287981Z","shell.execute_reply.started":"2022-12-28T03:47:10.471047Z","shell.execute_reply":"2022-12-28T03:47:11.286626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set(unique_aids_test).intersection(set(unique_aids_train)))","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:47:15.990200Z","iopub.execute_input":"2022-12-28T03:47:15.990670Z","iopub.status.idle":"2022-12-28T03:47:16.375482Z","shell.execute_reply.started":"2022-12-28T03:47:15.990634Z","shell.execute_reply":"2022-12-28T03:47:16.374110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set(unique_aids_test).intersection(set(unique_aids_train)))/len(unique_aids_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:47:19.194839Z","iopub.execute_input":"2022-12-28T03:47:19.195255Z","iopub.status.idle":"2022-12-28T03:47:19.549594Z","shell.execute_reply.started":"2022-12-28T03:47:19.195209Z","shell.execute_reply":"2022-12-28T03:47:19.547600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_aids_test = test_v7_full_3rd.select([\n    pl.col('aid').unique().alias('unique_aids')\n]).collect().to_series().to_list()\nlen(unique_aids_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:49:57.540273Z","iopub.execute_input":"2022-12-28T03:49:57.540941Z","iopub.status.idle":"2022-12-28T03:49:59.210396Z","shell.execute_reply.started":"2022-12-28T03:49:57.540894Z","shell.execute_reply":"2022-12-28T03:49:59.208801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set(unique_aids_test).intersection(set(unique_aids_train)))","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:50:20.927437Z","iopub.execute_input":"2022-12-28T03:50:20.927938Z","iopub.status.idle":"2022-12-28T03:50:21.304728Z","shell.execute_reply.started":"2022-12-28T03:50:20.927899Z","shell.execute_reply":"2022-12-28T03:50:21.303438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set(unique_aids_test).intersection(set(unique_aids_train)))/len(unique_aids_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:50:22.723604Z","iopub.execute_input":"2022-12-28T03:50:22.724084Z","iopub.status.idle":"2022-12-28T03:50:23.086681Z","shell.execute_reply.started":"2022-12-28T03:50:22.724047Z","shell.execute_reply":"2022-12-28T03:50:23.085253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## my validation set (2nd) has no cold start problem on aids","metadata":{}},{"cell_type":"code","source":"unique_aids_train = train_v7_2nd.select([\n    pl.col('aid').unique().alias('unique_aids')\n]).collect().to_series().to_list()\nlen(unique_aids_train)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:50:57.145875Z","iopub.execute_input":"2022-12-28T03:50:57.146416Z","iopub.status.idle":"2022-12-28T03:51:09.195330Z","shell.execute_reply.started":"2022-12-28T03:50:57.146334Z","shell.execute_reply":"2022-12-28T03:51:09.194148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_aids_test = test_v7_2nd.select([\n    pl.col('aid').unique().alias('unique_aids')\n]).collect().to_series().to_list()\nlen(unique_aids_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:51:09.197502Z","iopub.execute_input":"2022-12-28T03:51:09.197846Z","iopub.status.idle":"2022-12-28T03:51:09.824094Z","shell.execute_reply.started":"2022-12-28T03:51:09.197816Z","shell.execute_reply":"2022-12-28T03:51:09.822419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set(unique_aids_test).intersection(set(unique_aids_train)))","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:51:09.826335Z","iopub.execute_input":"2022-12-28T03:51:09.826734Z","iopub.status.idle":"2022-12-28T03:51:10.208787Z","shell.execute_reply.started":"2022-12-28T03:51:09.826701Z","shell.execute_reply":"2022-12-28T03:51:10.207271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set(unique_aids_test).intersection(set(unique_aids_train)))/len(unique_aids_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:51:10.212010Z","iopub.execute_input":"2022-12-28T03:51:10.213348Z","iopub.status.idle":"2022-12-28T03:51:10.587837Z","shell.execute_reply.started":"2022-12-28T03:51:10.213200Z","shell.execute_reply":"2022-12-28T03:51:10.586502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_aids_test = test_v7_full_2nd.select([\n    pl.col('aid').unique().alias('unique_aids')\n]).collect().to_series().to_list()\nlen(unique_aids_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:51:10.589427Z","iopub.execute_input":"2022-12-28T03:51:10.589819Z","iopub.status.idle":"2022-12-28T03:51:12.225028Z","shell.execute_reply.started":"2022-12-28T03:51:10.589786Z","shell.execute_reply":"2022-12-28T03:51:12.222840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set(unique_aids_test).intersection(set(unique_aids_train)))","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:51:12.227368Z","iopub.execute_input":"2022-12-28T03:51:12.227909Z","iopub.status.idle":"2022-12-28T03:51:12.604216Z","shell.execute_reply.started":"2022-12-28T03:51:12.227864Z","shell.execute_reply":"2022-12-28T03:51:12.601994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set(unique_aids_test).intersection(set(unique_aids_train)))/len(unique_aids_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:51:12.606218Z","iopub.execute_input":"2022-12-28T03:51:12.606728Z","iopub.status.idle":"2022-12-28T03:51:12.982116Z","shell.execute_reply.started":"2022-12-28T03:51:12.606688Z","shell.execute_reply":"2022-12-28T03:51:12.980776Z"},"trusted":true},"execution_count":null,"outputs":[]}]}