{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install polars","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-28T01:57:41.293341Z","iopub.execute_input":"2022-12-28T01:57:41.293763Z","iopub.status.idle":"2022-12-28T01:57:58.132824Z","shell.execute_reply.started":"2022-12-28T01:57:41.293678Z","shell.execute_reply":"2022-12-28T01:57:58.131734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"","metadata":{"execution":{"iopub.status.busy":"2022-12-28T01:57:58.134981Z","iopub.execute_input":"2022-12-28T01:57:58.135374Z","iopub.status.idle":"2022-12-28T01:57:58.141478Z","shell.execute_reply.started":"2022-12-28T01:57:58.135338Z","shell.execute_reply":"2022-12-28T01:57:58.140235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\nimport pandas as pd\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-28T01:57:58.142940Z","iopub.execute_input":"2022-12-28T01:57:58.143285Z","iopub.status.idle":"2022-12-28T01:57:58.207988Z","shell.execute_reply.started":"2022-12-28T01:57:58.143235Z","shell.execute_reply":"2022-12-28T01:57:58.206722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_colwidth', None)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T01:57:58.210745Z","iopub.execute_input":"2022-12-28T01:57:58.211176Z","iopub.status.idle":"2022-12-28T01:57:58.216415Z","shell.execute_reply.started":"2022-12-28T01:57:58.211142Z","shell.execute_reply":"2022-12-28T01:57:58.215313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pl = pl.scan_ndjson('/kaggle/input/otto-recommender-system/train.jsonl', n_rows=30)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-28T01:57:58.218049Z","iopub.execute_input":"2022-12-28T01:57:58.218568Z","iopub.status.idle":"2022-12-28T01:57:58.248405Z","shell.execute_reply.started":"2022-12-28T01:57:58.218526Z","shell.execute_reply":"2022-12-28T01:57:58.247449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pl.collect().head() # column names like `ts`, `aid`, `type` are gone, let's try pandas below.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-28T01:57:58.249918Z","iopub.execute_input":"2022-12-28T01:57:58.250609Z","iopub.status.idle":"2022-12-28T01:57:58.296739Z","shell.execute_reply.started":"2022-12-28T01:57:58.250574Z","shell.execute_reply":"2022-12-28T01:57:58.295894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_json('/kaggle/input/otto-recommender-system/train.jsonl', lines=True, nrows=5)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-28T01:57:58.297914Z","iopub.execute_input":"2022-12-28T01:57:58.298405Z","iopub.status.idle":"2022-12-28T01:57:58.320944Z","shell.execute_reply.started":"2022-12-28T01:57:58.298374Z","shell.execute_reply":"2022-12-28T01:57:58.319778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-28T01:57:58.322390Z","iopub.execute_input":"2022-12-28T01:57:58.322841Z","iopub.status.idle":"2022-12-28T01:57:58.424549Z","shell.execute_reply.started":"2022-12-28T01:57:58.322799Z","shell.execute_reply":"2022-12-28T01:57:58.423459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.events[0] # can we explode the list?","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-28T01:57:58.426202Z","iopub.execute_input":"2022-12-28T01:57:58.426565Z","iopub.status.idle":"2022-12-28T01:57:58.466435Z","shell.execute_reply.started":"2022-12-28T01:57:58.426533Z","shell.execute_reply":"2022-12-28T01:57:58.465227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.explode('events') # then can we split the dictionary into multiple columns?","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-28T01:57:58.469872Z","iopub.execute_input":"2022-12-28T01:57:58.470194Z","iopub.status.idle":"2022-12-28T01:57:58.502450Z","shell.execute_reply.started":"2022-12-28T01:57:58.470165Z","shell.execute_reply":"2022-12-28T01:57:58.501306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pl.explode('events').collect() # I can use polars to split the struct into multiple columns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-28T01:57:58.503778Z","iopub.execute_input":"2022-12-28T01:57:58.504081Z","iopub.status.idle":"2022-12-28T01:57:58.517587Z","shell.execute_reply.started":"2022-12-28T01:57:58.504052Z","shell.execute_reply":"2022-12-28T01:57:58.516468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pl.explode('events').select([\n    pl.col('session'),\n    pl.col('events').struct.field('aid').alias('aid'),\n    pl.col('events').struct.field('ts').alias('ts'),    \n    pl.col('events').struct.field('type').alias('type'),# can we turn clicks, carts, orders to 0, 1, 2?    \n]).collect()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-28T01:57:58.519026Z","iopub.execute_input":"2022-12-28T01:57:58.519412Z","iopub.status.idle":"2022-12-28T01:57:58.536741Z","shell.execute_reply.started":"2022-12-28T01:57:58.519374Z","shell.execute_reply":"2022-12-28T01:57:58.535884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The dataset is huge, it is easier to deal the dataframe in the format above.","metadata":{}},{"cell_type":"code","source":"train_pl.explode('events').select([\n    pl.col('session'),\n    pl.col('events').struct.field('aid').alias('aid'),\n    pl.col('events').struct.field('ts').alias('ts'),    \n    pl.col('events').struct.field('type').alias('type'),# can we turn clicks, carts, orders to 0, 1, 2?    \n]).select([\n    pl.col('session').cast(pl.Int32).alias('session'),\n    pl.col('aid').cast(pl.Int32).alias('aid'),\n    pl.col('ts').cast(pl.Int64).alias('ts'), # using it as milliseconds must use Int64\n    pl.col('type').alias('ori_type'),\n    (pl.when(pl.col('type') == 'clicks')\n        .then(pl.lit(0))\n        .when(pl.col('type') == 'carts')\n        .then(pl.lit(1))\n        .otherwise(pl.lit(2))).cast(pl.Int8).alias('type'),\n]).collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T01:57:58.538002Z","iopub.execute_input":"2022-12-28T01:57:58.538530Z","iopub.status.idle":"2022-12-28T01:57:58.560539Z","shell.execute_reply.started":"2022-12-28T01:57:58.538497Z","shell.execute_reply":"2022-12-28T01:57:58.559386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's peek at test_labels.jsonl","metadata":{}},{"cell_type":"markdown","source":"## Radek's test_labels look like","metadata":{}},{"cell_type":"code","source":"test_labels_radek = pd.read_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/test_labels.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T01:57:58.561880Z","iopub.execute_input":"2022-12-28T01:57:58.562247Z","iopub.status.idle":"2022-12-28T01:58:00.224687Z","shell.execute_reply.started":"2022-12-28T01:57:58.562212Z","shell.execute_reply":"2022-12-28T01:58:00.223697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_radek.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T01:58:00.225650Z","iopub.execute_input":"2022-12-28T01:58:00.225943Z","iopub.status.idle":"2022-12-28T01:58:00.239761Z","shell.execute_reply.started":"2022-12-28T01:58:00.225916Z","shell.execute_reply":"2022-12-28T01:58:00.238959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Original test_labels.jsonl look like","metadata":{}},{"cell_type":"code","source":"test_labels = pd.read_json('/kaggle/input/otto-validation-7days-jsonl-from-script-on-kaggle/test_labels.jsonl', lines=True, nrows=5)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T01:58:00.240866Z","iopub.execute_input":"2022-12-28T01:58:00.241754Z","iopub.status.idle":"2022-12-28T01:58:00.259729Z","shell.execute_reply.started":"2022-12-28T01:58:00.241721Z","shell.execute_reply":"2022-12-28T01:58:00.258956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T01:58:00.260787Z","iopub.execute_input":"2022-12-28T01:58:00.261490Z","iopub.status.idle":"2022-12-28T01:58:00.275546Z","shell.execute_reply.started":"2022-12-28T01:58:00.261455Z","shell.execute_reply":"2022-12-28T01:58:00.274371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_pl = pl.from_pandas(test_labels.head())","metadata":{"execution":{"iopub.status.busy":"2022-12-28T01:58:00.276929Z","iopub.execute_input":"2022-12-28T01:58:00.277320Z","iopub.status.idle":"2022-12-28T01:58:00.287927Z","shell.execute_reply.started":"2022-12-28T01:58:00.277288Z","shell.execute_reply":"2022-12-28T01:58:00.287061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_pl.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T01:58:00.289137Z","iopub.execute_input":"2022-12-28T01:58:00.289658Z","iopub.status.idle":"2022-12-28T01:58:00.301350Z","shell.execute_reply.started":"2022-12-28T01:58:00.289625Z","shell.execute_reply":"2022-12-28T01:58:00.300408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_pl.select([\n    pl.col('session'),\n    pl.when(pl.col('labels').struct.field('clicks').is_not_null()).then(pl.col('labels').struct.field('clicks')).alias('label_clicks')\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-28T01:58:00.302769Z","iopub.execute_input":"2022-12-28T01:58:00.303945Z","iopub.status.idle":"2022-12-28T01:58:00.315561Z","shell.execute_reply.started":"2022-12-28T01:58:00.303899Z","shell.execute_reply":"2022-12-28T01:58:00.314300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_pl.select([\n    pl.col('session'),\n    pl.col('labels').struct.field('clicks').alias('label_clicks'),\n    pl.col('labels').struct.field('carts').alias('label_carts'),\n    pl.col('labels').struct.field('orders').alias('label_orders'),    \n])","metadata":{"execution":{"iopub.status.busy":"2022-12-28T01:58:00.317634Z","iopub.execute_input":"2022-12-28T01:58:00.318355Z","iopub.status.idle":"2022-12-28T01:58:00.327789Z","shell.execute_reply.started":"2022-12-28T01:58:00.318311Z","shell.execute_reply":"2022-12-28T01:58:00.326684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_clicks = test_labels_pl.select([\n    pl.col('session'),\n    pl.lit('clicks').alias('type'),\n    pl.col('labels').struct.field('clicks').alias('ground_truth_nonlist'),\n    pl.concat_list(pl.col('labels').struct.field('clicks')).alias('ground_truth')    \n])\nlabel_clicks","metadata":{"execution":{"iopub.status.busy":"2022-12-28T02:47:17.620491Z","iopub.execute_input":"2022-12-28T02:47:17.621479Z","iopub.status.idle":"2022-12-28T02:47:17.632737Z","shell.execute_reply.started":"2022-12-28T02:47:17.621439Z","shell.execute_reply":"2022-12-28T02:47:17.631197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_clicks = test_labels_pl.select([\n    pl.col('session'),\n    pl.lit('clicks').alias('type'),\n    pl.col('labels').struct.field('clicks').alias('ground_truth_nonlist'),\n]).select([\n    pl.all().exclude('ground_truth_nonlist'),\n    pl.when(pl.col('ground_truth_nonlist').is_not_null()).then(pl.concat_list(['ground_truth_nonlist'])).alias('ground_truth')        \n])\nlabel_clicks","metadata":{"execution":{"iopub.status.busy":"2022-12-28T02:51:14.209439Z","iopub.execute_input":"2022-12-28T02:51:14.209844Z","iopub.status.idle":"2022-12-28T02:51:14.221607Z","shell.execute_reply.started":"2022-12-28T02:51:14.209813Z","shell.execute_reply":"2022-12-28T02:51:14.220195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_carts = test_labels_pl.select([\n    pl.col('session'),\n    pl.lit('carts').alias('type'),\n    pl.col('labels').struct.field('carts').alias('ground_truth')    \n])\nlabel_carts","metadata":{"execution":{"iopub.status.busy":"2022-12-28T01:58:00.340735Z","iopub.execute_input":"2022-12-28T01:58:00.341661Z","iopub.status.idle":"2022-12-28T01:58:00.351296Z","shell.execute_reply.started":"2022-12-28T01:58:00.341626Z","shell.execute_reply":"2022-12-28T01:58:00.350393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_orders = test_labels_pl.select([\n    pl.col('session'),\n    pl.lit('orders').alias('type'),\n    pl.col('labels').struct.field('orders').alias('ground_truth')    \n])\nlabel_orders","metadata":{"execution":{"iopub.status.busy":"2022-12-28T01:58:00.352726Z","iopub.execute_input":"2022-12-28T01:58:00.353306Z","iopub.status.idle":"2022-12-28T01:58:00.365074Z","shell.execute_reply.started":"2022-12-28T01:58:00.353272Z","shell.execute_reply":"2022-12-28T01:58:00.363764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pl.concat([label_clicks, label_carts, label_orders]).sort('session').drop_nulls().select([\n    pl.col('session').cast(pl.Int32),\n    pl.col('type'),\n    pl.col('ground_truth').arr.eval(pl.element().cast(pl.Int32)) # cast into Int32 for each element of a list in a column\n \n])","metadata":{"execution":{"iopub.status.busy":"2022-12-28T03:03:52.389188Z","iopub.execute_input":"2022-12-28T03:03:52.389620Z","iopub.status.idle":"2022-12-28T03:03:52.403632Z","shell.execute_reply.started":"2022-12-28T03:03:52.389587Z","shell.execute_reply":"2022-12-28T03:03:52.402034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_radek.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T02:52:51.660841Z","iopub.execute_input":"2022-12-28T02:52:51.661287Z","iopub.status.idle":"2022-12-28T02:52:51.675457Z","shell.execute_reply.started":"2022-12-28T02:52:51.661235Z","shell.execute_reply":"2022-12-28T02:52:51.674130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}