{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install polars","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-22T13:32:39.477829Z","iopub.execute_input":"2022-12-22T13:32:39.478281Z","iopub.status.idle":"2022-12-22T13:32:53.140119Z","shell.execute_reply.started":"2022-12-22T13:32:39.478197Z","shell.execute_reply":"2022-12-22T13:32:53.139398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\nimport gc\nimport sys","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:32:53.142051Z","iopub.execute_input":"2022-12-22T13:32:53.14238Z","iopub.status.idle":"2022-12-22T13:32:53.197147Z","shell.execute_reply.started":"2022-12-22T13:32:53.142354Z","shell.execute_reply":"2022-12-22T13:32:53.195753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:32:53.199476Z","iopub.execute_input":"2022-12-22T13:32:53.19987Z","iopub.status.idle":"2022-12-22T13:32:53.2047Z","shell.execute_reply.started":"2022-12-22T13:32:53.199846Z","shell.execute_reply":"2022-12-22T13:32:53.203618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Further check on my dataset\n\nPrevious [checks](https://www.kaggle.com/code/danielliao/recreate-otto-full-optimized-memory-footprint?scriptVersionId=114180650&cellId=4) on my dataset against radek's","metadata":{}},{"cell_type":"code","source":"train = pl.scan_parquet('/kaggle/input/otto-radek-style-polars/train.parquet')\ntest = pl.scan_parquet('/kaggle/input/otto-radek-style-polars/test.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:32:53.207308Z","iopub.execute_input":"2022-12-22T13:32:53.2076Z","iopub.status.idle":"2022-12-22T13:32:53.234347Z","shell.execute_reply.started":"2022-12-22T13:32:53.207577Z","shell.execute_reply":"2022-12-22T13:32:53.233657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(3).collect()\ntrain.tail(3).collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:32:53.235262Z","iopub.execute_input":"2022-12-22T13:32:53.235486Z","iopub.status.idle":"2022-12-22T13:33:17.458352Z","shell.execute_reply.started":"2022-12-22T13:32:53.235464Z","shell.execute_reply":"2022-12-22T13:33:17.457648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# A simple pipeline\n\nthe original author [approach](https://www.kaggle.com/code/danielliao/otto-getting-started-eda-baseline?scriptVersionId=113569269&cellId=115)\n\n**on the individual user/session level:** \n- the more occurrences on an aid in early part of a session of test set, the more likely it will be clicked, carted or ordered in the later part of the session\n\n\n**on the universal user/session or on aid level:**\n- if no enough aids in a test session used for prediction, use the most common clicks or carts or orders from train set","metadata":{}},{"cell_type":"markdown","source":"# Step 1: order aid in each session by num of occurs from high to low","metadata":{}},{"cell_type":"code","source":"test.groupby(['session', 'aid']).agg([\n    pl.col('aid').count().alias('num_occurs_each_aid_each_session')\n]).sort(['session', 'aid']).collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:33:17.45942Z","iopub.execute_input":"2022-12-22T13:33:17.459653Z","iopub.status.idle":"2022-12-22T13:33:19.711332Z","shell.execute_reply.started":"2022-12-22T13:33:17.459631Z","shell.execute_reply":"2022-12-22T13:33:19.710702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.groupby(['session', 'aid']).agg([\n    pl.col('aid').count().alias('num_occurs_each_aid_each_session')\n]).sort(['session', 'aid']).collect().select([\n    pl.col('session'),\n    pl.col('aid').sort_by('num_occurs_each_aid_each_session', reverse=True).over('session').alias('aid_1'),\n    pl.col('num_occurs_each_aid_each_session').sort(reverse=True).over('session').alias('num_occurs'),\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:33:19.712305Z","iopub.execute_input":"2022-12-22T13:33:19.712811Z","iopub.status.idle":"2022-12-22T13:33:22.763869Z","shell.execute_reply.started":"2022-12-22T13:33:19.712787Z","shell.execute_reply":"2022-12-22T13:33:22.763053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.groupby(['session', 'aid']).agg([\n    pl.col('aid').count().alias('num_occurs_each_aid_each_session')\n]).sort(['session', 'aid']).collect().select([\n    pl.col('session'),\n    pl.col('aid').sort_by('num_occurs_each_aid_each_session', reverse=True).over('session').alias('aid_1'),\n    pl.col('num_occurs_each_aid_each_session').sort(reverse=True).over('session').alias('num_occurs'),\n]).groupby('session').agg([\n    pl.all()\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:33:22.765343Z","iopub.execute_input":"2022-12-22T13:33:22.765717Z","iopub.status.idle":"2022-12-22T13:33:25.837433Z","shell.execute_reply.started":"2022-12-22T13:33:22.765667Z","shell.execute_reply":"2022-12-22T13:33:25.835847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.groupby(['session', 'aid']).agg([\n    pl.col('aid').count().alias('num_occurs_each_aid_each_session')\n]).sort(['session', 'aid']).collect().select([\n    pl.col('session'),\n    pl.col('aid').sort_by('num_occurs_each_aid_each_session', reverse=True).over('session').alias('aid_1'),\n    pl.col('num_occurs_each_aid_each_session').sort(reverse=True).over('session').alias('num_occurs'),\n]).groupby('session').agg([\n    pl.col('aid_1').alias('aid_order_each_session')\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:33:25.838862Z","iopub.execute_input":"2022-12-22T13:33:25.839133Z","iopub.status.idle":"2022-12-22T13:33:28.964378Z","shell.execute_reply.started":"2022-12-22T13:33:25.839109Z","shell.execute_reply":"2022-12-22T13:33:28.962102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The above shows the predictions for each test session only using aids of the test session","metadata":{}},{"cell_type":"markdown","source":"## Duplicate predictions 3 times for clicks, carts and orders","metadata":{}},{"cell_type":"code","source":"test.select('session').collect().n_unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:33:28.968623Z","iopub.execute_input":"2022-12-22T13:33:28.968968Z","iopub.status.idle":"2022-12-22T13:33:29.279027Z","shell.execute_reply.started":"2022-12-22T13:33:28.968945Z","shell.execute_reply":"2022-12-22T13:33:29.278239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_clicks = test.groupby(['session', 'aid']).agg([\n    pl.col('aid').count().alias('num_occurs_each_aid_each_session')\n]).sort(['session', 'aid']).collect().select([\n    pl.col('session'),\n    pl.col('aid').sort_by('num_occurs_each_aid_each_session', reverse=True).over('session').alias('aid_1'),\n    pl.col('num_occurs_each_aid_each_session').sort(reverse=True).over('session').alias('num_occurs'),\n]).groupby('session').agg([\n    pl.col('aid_1').alias('aid_order_each_session')\n]).with_columns([\n    (pl.col('session') + \"_clicks\").alias('session_type')\n])\n\npreds_clicks.head()\npreds_clicks.tail()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:33:29.280269Z","iopub.execute_input":"2022-12-22T13:33:29.282355Z","iopub.status.idle":"2022-12-22T13:33:32.626286Z","shell.execute_reply.started":"2022-12-22T13:33:29.282295Z","shell.execute_reply":"2022-12-22T13:33:32.625312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_carts = test.groupby(['session', 'aid']).agg([\n    pl.col('aid').count().alias('num_occurs_each_aid_each_session')\n]).sort(['session', 'aid']).collect().select([\n    pl.col('session'),\n    pl.col('aid').sort_by('num_occurs_each_aid_each_session', reverse=True).over('session').alias('aid_1'),\n    pl.col('num_occurs_each_aid_each_session').sort(reverse=True).over('session').alias('num_occurs'),\n]).groupby('session').agg([\n    pl.col('aid_1').alias('aid_order_each_session')\n]).with_columns([\n    (pl.col('session') + \"_carts\").alias('session_type')\n])\n\npreds_carts.head()\npreds_carts.tail()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:33:32.627519Z","iopub.execute_input":"2022-12-22T13:33:32.627918Z","iopub.status.idle":"2022-12-22T13:33:36.120671Z","shell.execute_reply.started":"2022-12-22T13:33:32.627893Z","shell.execute_reply":"2022-12-22T13:33:36.11975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_orders = test.groupby(['session', 'aid']).agg([\n    pl.col('aid').count().alias('num_occurs_each_aid_each_session')\n]).sort(['session', 'aid']).collect().select([\n    pl.col('session'),\n    pl.col('aid').sort_by('num_occurs_each_aid_each_session', reverse=True).over('session').alias('aid_1'),\n    pl.col('num_occurs_each_aid_each_session').sort(reverse=True).over('session').alias('num_occurs'),\n]).groupby('session').agg([\n    pl.col('aid_1').alias('aid_order_each_session')\n]).with_columns([\n    (pl.col('session') + \"_orders\").alias('session_type')\n])\n\npreds_orders.head()\npreds_orders.tail()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:33:36.122046Z","iopub.execute_input":"2022-12-22T13:33:36.122368Z","iopub.status.idle":"2022-12-22T13:33:39.663939Z","shell.execute_reply.started":"2022-12-22T13:33:36.122337Z","shell.execute_reply":"2022-12-22T13:33:39.663095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joined = pl.concat([preds_clicks,preds_carts,preds_orders]).sort('session')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:33:39.665158Z","iopub.execute_input":"2022-12-22T13:33:39.665969Z","iopub.status.idle":"2022-12-22T13:33:40.020861Z","shell.execute_reply.started":"2022-12-22T13:33:39.665935Z","shell.execute_reply":"2022-12-22T13:33:40.019856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joined.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:33:40.02195Z","iopub.execute_input":"2022-12-22T13:33:40.022328Z","iopub.status.idle":"2022-12-22T13:33:40.02766Z","shell.execute_reply.started":"2022-12-22T13:33:40.022304Z","shell.execute_reply":"2022-12-22T13:33:40.027038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## create 3 lists (20 most common aids) for clicks, carts and orders in train set and add them to predictions above","metadata":{}},{"cell_type":"code","source":"clicks_lst = train.filter(pl.col('type') == 0).groupby('aid').agg([\n    pl.col('aid').count().alias('total_occurs_of_aid_as_clicks')\n]).sort('total_occurs_of_aid_as_clicks', reverse=True).limit(20).collect().select('aid').to_series().to_list()\nclicks_lst","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:33:40.028793Z","iopub.execute_input":"2022-12-22T13:33:40.029151Z","iopub.status.idle":"2022-12-22T13:33:53.721964Z","shell.execute_reply.started":"2022-12-22T13:33:40.029129Z","shell.execute_reply":"2022-12-22T13:33:53.721127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"carts_lst = train.filter(pl.col('type') == 1).groupby('aid').agg([\n    pl.col('aid').count().alias('total_occurs_of_aid_as_carts')\n]).sort('total_occurs_of_aid_as_carts', reverse=True).limit(20).collect().select('aid').to_series().to_list()\ncarts_lst","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:33:53.723228Z","iopub.execute_input":"2022-12-22T13:33:53.723706Z","iopub.status.idle":"2022-12-22T13:33:58.955871Z","shell.execute_reply.started":"2022-12-22T13:33:53.723661Z","shell.execute_reply":"2022-12-22T13:33:58.955218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orders_lst = train.filter(pl.col('type') == 2).groupby('aid').agg([\n    pl.col('aid').count().alias('total_occurs_of_aid_as_orders')\n]).sort('total_occurs_of_aid_as_orders', reverse=True).limit(20).collect().select('aid').to_series().to_list()\norders_lst","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:33:58.958807Z","iopub.execute_input":"2022-12-22T13:33:58.960313Z","iopub.status.idle":"2022-12-22T13:34:03.73339Z","shell.execute_reply.started":"2022-12-22T13:33:58.960285Z","shell.execute_reply":"2022-12-22T13:34:03.732309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joined.with_columns([\n    pl.col('aid_order_each_session').alias('predictions'),\n    pl.Series('top_clicks', [clicks_lst]),\n    pl.Series('top_carts', [carts_lst]),\n    pl.Series('top_orders', [orders_lst]),    \n]).head()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:34:03.734543Z","iopub.execute_input":"2022-12-22T13:34:03.734973Z","iopub.status.idle":"2022-12-22T13:34:05.19025Z","shell.execute_reply.started":"2022-12-22T13:34:03.734945Z","shell.execute_reply":"2022-12-22T13:34:05.188539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# add the most common clicks to the predictions for clicks","metadata":{}},{"cell_type":"code","source":"%%timeit -r 1\njoined.with_columns([\n    pl.col('aid_order_each_session').alias('predictions'),    \n    pl.Series('top_clicks', [clicks_lst]),\n]).filter(pl.col('session_type').str.contains('clicks')).with_columns([\n    pl.struct(['predictions', 'top_clicks']).apply(lambda x: x['predictions'] + x['top_clicks'][:max(0,20-len(x['predictions']))]).alias('full_clicks'),\n]).select([\n    'session',\n    'session_type',\n    'full_clicks',\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:34:05.192045Z","iopub.execute_input":"2022-12-22T13:34:05.19242Z","iopub.status.idle":"2022-12-22T13:35:03.767915Z","shell.execute_reply.started":"2022-12-22T13:34:05.192395Z","shell.execute_reply":"2022-12-22T13:35:03.766089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit -r 1\njoined.with_columns([\n    pl.col('aid_order_each_session').alias('predictions'),    \n    pl.Series('top_clicks', [clicks_lst]),\n]).filter(pl.col('session_type').str.contains('clicks')).with_columns([\n#     pl.struct(['predictions', 'top_clicks']).apply(lambda x: x['predictions'][:20] + x['top_clicks'][:max(0,20-len(x['predictions']))]).alias('full_clicks'),\n#         pl.col('predictions').arr.head(20) + pl.col('top_clicks').arr.head(np.max(0, 20-))\n    pl.col('predictions').arr.lengths().alias('pred_lengths')\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:35:03.770137Z","iopub.execute_input":"2022-12-22T13:35:03.770484Z","iopub.status.idle":"2022-12-22T13:35:06.014601Z","shell.execute_reply.started":"2022-12-22T13:35:03.770459Z","shell.execute_reply":"2022-12-22T13:35:06.013256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:35:06.017476Z","iopub.execute_input":"2022-12-22T13:35:06.017839Z","iopub.status.idle":"2022-12-22T13:35:06.022905Z","shell.execute_reply.started":"2022-12-22T13:35:06.017814Z","shell.execute_reply":"2022-12-22T13:35:06.021797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joined.with_columns([\n    pl.col('aid_order_each_session').alias('predictions'),    \n    pl.Series('top_clicks', [clicks_lst]),\n]).filter(pl.col('session_type').str.contains('clicks')).with_columns([\n#     pl.struct(['predictions', 'top_clicks']).apply(lambda x: x['predictions'][:20] + x['top_clicks'][:max(0,20-len(x['predictions']))]).alias('full_clicks'),\n    pl.col('predictions').arr.lengths().alias('pred_lengths')\n]).with_columns([\n    pl.col('predictions').arr.head(20).alias('preds'),# + pl.col('top_clicks').arr.head(np.max(0, 20-x))).alias('preds')  \n    pl.col('predictions').arr.head(20).arr.lengths().alias('preds_count'),    \n#     pl.col('pred_lengths').apply(lambda x: pl.col('predictions').arr.head(20) + pl.col('top_clicks').arr.head(np.max(0, 20-x))).alias('preds')\n#     pl.col('predictions').arr.head(20) + pl.col('top_clicks').arr.head(np.max(0, 20-pl.col('predictions').arr.lengths())),    \n]).with_columns([\n    pl.col('top_clicks').arr.head((20 - pl.col('preds_count'))).alias('added_preds')\n]).head()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:35:06.024392Z","iopub.execute_input":"2022-12-22T13:35:06.024732Z","iopub.status.idle":"2022-12-22T13:35:08.557909Z","shell.execute_reply.started":"2022-12-22T13:35:06.024675Z","shell.execute_reply":"2022-12-22T13:35:08.556883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joined.with_columns([\n    pl.col('aid_order_each_session').alias('predictions'),    \n    pl.Series('top_clicks', [clicks_lst]),\n]).filter(pl.col('session_type').str.contains('clicks')).with_columns([\n#     pl.struct(['predictions', 'top_clicks']).apply(lambda x: x['predictions'][:20] + x['top_clicks'][:max(0,20-len(x['predictions']))]).alias('full_clicks'),\n    pl.col('predictions').arr.lengths().alias('pred_lengths')\n]).with_columns([\n    pl.col('predictions').arr.head(20).alias('preds'),# + pl.col('top_clicks').arr.head(np.max(0, 20-x))).alias('preds')  \n    pl.col('predictions').arr.head(20).arr.lengths().alias('preds_count'),    \n#     pl.col('pred_lengths').apply(lambda x: pl.col('predictions').arr.head(20) + pl.col('top_clicks').arr.head(np.max(0, 20-x))).alias('preds')\n#     pl.col('predictions').arr.head(20) + pl.col('top_clicks').arr.head(np.max(0, 20-pl.col('predictions').arr.lengths())),    \n]).with_columns([\n    pl.col('top_clicks').arr.head((20 - pl.col('preds_count'))).alias('added_preds')\n]).with_columns([\n    pl.col('added_preds').arr.eval(pl.element().cast(pl.Int32))\n#     (pl.col('preds') + pl.col('added_preds').cast(pl.Li)).alias('new')\n#     pl.struct(['preds', 'added_preds']).apply(lambda x: x['preds'] + x['added_preds']).alias('new')\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:35:08.559238Z","iopub.execute_input":"2022-12-22T13:35:08.55949Z","iopub.status.idle":"2022-12-22T13:35:12.67961Z","shell.execute_reply.started":"2022-12-22T13:35:08.559466Z","shell.execute_reply":"2022-12-22T13:35:12.678612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit -r 1\njoined.with_columns([\n    pl.col('aid_order_each_session').alias('predictions'),    \n    pl.Series('top_clicks', [clicks_lst]),\n]).filter(pl.col('session_type').str.contains('clicks')).with_columns([\n#     pl.struct(['predictions', 'top_clicks']).apply(lambda x: x['predictions'][:20] + x['top_clicks'][:max(0,20-len(x['predictions']))]).alias('full_clicks'),\n    pl.col('predictions').arr.lengths().alias('pred_lengths')\n]).with_columns([\n    pl.col('predictions').arr.head(20).alias('preds'),# + pl.col('top_clicks').arr.head(np.max(0, 20-x))).alias('preds')  \n    pl.col('predictions').arr.head(20).arr.lengths().alias('preds_count'),    \n#     pl.col('pred_lengths').apply(lambda x: pl.col('predictions').arr.head(20) + pl.col('top_clicks').arr.head(np.max(0, 20-x))).alias('preds')\n#     pl.col('predictions').arr.head(20) + pl.col('top_clicks').arr.head(np.max(0, 20-pl.col('predictions').arr.lengths())),    \n]).with_columns([\n    pl.col('top_clicks').arr.head((20 - pl.col('preds_count'))).alias('added_preds')\n]).with_columns([\n    pl.col('added_preds').arr.eval(pl.element().cast(pl.Int32))\n#     (pl.col('preds') + pl.col('added_preds').cast(pl.Li)).alias('new')\n#     pl.struct(['preds', 'added_preds']).apply(lambda x: x['preds'] + x['added_preds']).alias('new')\n]).with_columns([\n#     (pl.col('preds') + pl.col('added_preds')).alias('full_preds'),\n    pl.struct(['preds', 'added_preds']).apply(lambda x: x['preds'] + x['added_preds']).alias('full_preds')\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:35:12.680798Z","iopub.execute_input":"2022-12-22T13:35:12.682072Z","iopub.status.idle":"2022-12-22T13:36:13.338639Z","shell.execute_reply.started":"2022-12-22T13:35:12.682022Z","shell.execute_reply":"2022-12-22T13:36:13.337408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit -r 1\njoined.with_columns([\n    pl.col('aid_order_each_session').alias('predictions'),    \n    pl.Series('top_clicks', [clicks_lst]),\n]).filter(pl.col('session_type').str.contains('clicks')).with_columns([\n    pl.col('predictions').arr.lengths().alias('pred_lengths')\n]).with_columns([\n    pl.col('predictions').arr.head(20).alias('preds'),# + pl.col('top_clicks').arr.head(np.max(0, 20-x))).alias('preds')  \n    pl.col('predictions').arr.head(20).arr.lengths().alias('preds_count'),     \n]).with_columns([\n    pl.col('top_clicks').arr.head((20 - pl.col('preds_count'))).alias('added_preds')\n]).with_columns([\n    pl.col('added_preds').arr.eval(pl.element().cast(pl.Int32))\n\n]).with_columns([\n    pl.concat_list(['preds', 'added_preds']).alias('full_preds')\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:36:13.339989Z","iopub.execute_input":"2022-12-22T13:36:13.340343Z","iopub.status.idle":"2022-12-22T13:36:23.225165Z","shell.execute_reply.started":"2022-12-22T13:36:13.34031Z","shell.execute_reply":"2022-12-22T13:36:23.223732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\njoined.with_columns([\n    pl.col('aid_order_each_session').alias('predictions'),    \n    pl.Series('top_clicks', [clicks_lst]),\n]).filter(pl.col('session_type').str.contains('clicks')).with_columns([\n    pl.col('predictions').arr.lengths().alias('pred_lengths')\n]).with_columns([\n    pl.col('predictions').arr.head(20).alias('preds'),# + pl.col('top_clicks').arr.head(np.max(0, 20-x))).alias('preds')  \n    pl.col('predictions').arr.head(20).arr.lengths().alias('preds_count'),     \n]).with_columns([\n    pl.col('top_clicks').arr.head((20 - pl.col('preds_count'))).alias('added_preds')\n]).with_columns([\n    pl.col('added_preds').arr.eval(pl.element().cast(pl.Int32))\n\n]).with_columns([\n    pl.concat_list(['preds', 'added_preds']).alias('full_preds')\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T13:36:23.226406Z","iopub.execute_input":"2022-12-22T13:36:23.226766Z","iopub.status.idle":"2022-12-22T13:36:28.046441Z","shell.execute_reply.started":"2022-12-22T13:36:23.226735Z","shell.execute_reply":"2022-12-22T13:36:28.045443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit -r 1\njoined.with_columns([\n    pl.col('aid_order_each_session').alias('predictions'),    \n    pl.Series('top_clicks', [clicks_lst]),\n]).filter(pl.col('session_type').str.contains('clicks')).with_columns([\n    pl.col('predictions').arr.lengths().alias('pred_lengths')\n]).with_columns([\n    pl.col('predictions').arr.head(20).alias('preds'),# + pl.col('top_clicks').arr.head(np.max(0, 20-x))).alias('preds')  \n    pl.col('predictions').arr.head(20).arr.lengths().alias('preds_count'),     \n]).with_columns([\n    pl.col('top_clicks').arr.head((20 - pl.col('preds_count'))).alias('added_preds')\n]).with_columns([\n    pl.col('added_preds').arr.eval(pl.element().cast(pl.Int32))\n\n]).with_columns([\n    pl.col('preds').arr.concat(pl.col('added_preds')).alias('full_preds') # this could be the one\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T14:03:13.675256Z","iopub.execute_input":"2022-12-22T14:03:13.675774Z","iopub.status.idle":"2022-12-22T14:03:24.50042Z","shell.execute_reply.started":"2022-12-22T14:03:13.675739Z","shell.execute_reply":"2022-12-22T14:03:24.498512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joined.with_columns([\n    pl.col('aid_order_each_session').alias('predictions'),    \n    pl.Series('top_clicks', [clicks_lst]),\n]).filter(pl.col('session_type').str.contains('clicks')).with_columns([\n    pl.col('predictions').arr.lengths().alias('pred_lengths')\n]).with_columns([\n    pl.col('predictions').arr.head(20).alias('preds'),# + pl.col('top_clicks').arr.head(np.max(0, 20-x))).alias('preds')  \n    pl.col('predictions').arr.head(20).arr.lengths().alias('preds_count'),     \n]).with_columns([\n    pl.col('top_clicks').arr.head((20 - pl.col('preds_count'))).alias('added_preds')\n]).with_columns([\n    pl.col('added_preds').arr.eval(pl.element().cast(pl.Int32))\n\n]).with_columns([\n    pl.col('preds').arr.concat(pl.col('added_preds')).alias('full_preds') # this could be the one\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T14:07:44.262928Z","iopub.execute_input":"2022-12-22T14:07:44.263263Z","iopub.status.idle":"2022-12-22T14:07:49.681769Z","shell.execute_reply.started":"2022-12-22T14:07:44.263239Z","shell.execute_reply":"2022-12-22T14:07:49.680832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Add predictions for clicks, carts and orders together","metadata":{}},{"cell_type":"code","source":"clicks = joined.with_columns([\n    pl.col('aid_order_each_session').alias('predictions'),    \n    pl.Series('top_clicks', [clicks_lst]),\n]).filter(pl.col('session_type').str.contains('clicks')).with_columns([\n    pl.col('predictions').arr.lengths().alias('pred_lengths')\n]).with_columns([\n    pl.col('predictions').arr.head(20).alias('preds'),# + pl.col('top_clicks').arr.head(np.max(0, 20-x))).alias('preds')  \n    pl.col('predictions').arr.head(20).arr.lengths().alias('preds_count'),     \n]).with_columns([\n    pl.col('top_clicks').arr.head((20 - pl.col('preds_count'))).alias('added_preds')\n]).with_columns([\n    pl.col('added_preds').arr.eval(pl.element().cast(pl.Int32))\n\n]).with_columns([\n    pl.col('preds').arr.concat(pl.col('added_preds')).alias('full_preds') # this could be the one\n]).select([\n    'session',\n    'session_type',\n    'full_preds'\n])\nclicks.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T14:12:05.681196Z","iopub.execute_input":"2022-12-22T14:12:05.681617Z","iopub.status.idle":"2022-12-22T14:12:10.537426Z","shell.execute_reply.started":"2022-12-22T14:12:05.681589Z","shell.execute_reply":"2022-12-22T14:12:10.536868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"carts = joined.with_columns([\n    pl.col('aid_order_each_session').alias('predictions'),    \n    pl.Series('top_carts', [carts_lst]),\n]).filter(pl.col('session_type').str.contains('carts')).with_columns([\n    pl.col('predictions').arr.lengths().alias('pred_lengths')\n]).with_columns([\n    pl.col('predictions').arr.head(20).alias('preds'),\n    pl.col('predictions').arr.head(20).arr.lengths().alias('preds_count'),     \n]).with_columns([\n    pl.col('top_carts').arr.head((20 - pl.col('preds_count'))).alias('added_preds')\n]).with_columns([\n    pl.col('added_preds').arr.eval(pl.element().cast(pl.Int32))\n\n]).with_columns([\n    pl.col('preds').arr.concat(pl.col('added_preds')).alias('full_preds') # this could be the one\n]).select([\n    'session',\n    'session_type',\n    'full_preds'\n])\ncarts.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T14:12:15.447124Z","iopub.execute_input":"2022-12-22T14:12:15.447702Z","iopub.status.idle":"2022-12-22T14:12:20.065205Z","shell.execute_reply.started":"2022-12-22T14:12:15.44764Z","shell.execute_reply":"2022-12-22T14:12:20.06435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orders = joined.with_columns([\n    pl.col('aid_order_each_session').alias('predictions'),    \n    pl.Series('top_orders', [orders_lst]),\n]).filter(pl.col('session_type').str.contains('orders')).with_columns([\n    pl.col('predictions').arr.lengths().alias('pred_lengths')\n]).with_columns([\n    pl.col('predictions').arr.head(20).alias('preds'),\n    pl.col('predictions').arr.head(20).arr.lengths().alias('preds_count'),     \n]).with_columns([\n    pl.col('top_orders').arr.head((20 - pl.col('preds_count'))).alias('added_preds')\n]).with_columns([\n    pl.col('added_preds').arr.eval(pl.element().cast(pl.Int32))\n\n]).with_columns([\n    pl.col('preds').arr.concat(pl.col('added_preds')).alias('full_preds') # this could be the one\n]).select([\n    'session',\n    'session_type',\n    'full_preds'\n])\norders.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T14:12:25.419721Z","iopub.execute_input":"2022-12-22T14:12:25.420336Z","iopub.status.idle":"2022-12-22T14:12:30.102776Z","shell.execute_reply.started":"2022-12-22T14:12:25.420302Z","shell.execute_reply":"2022-12-22T14:12:30.101754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pl.concat([clicks, carts, orders]).sort('session').head()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T14:13:52.747574Z","iopub.execute_input":"2022-12-22T14:13:52.747932Z","iopub.status.idle":"2022-12-22T14:13:53.826203Z","shell.execute_reply.started":"2022-12-22T14:13:52.747907Z","shell.execute_reply":"2022-12-22T14:13:53.8252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# submission format","metadata":{}},{"cell_type":"code","source":"out  = pl.concat([clicks, carts, orders]).sort('session')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T14:17:02.717603Z","iopub.execute_input":"2022-12-22T14:17:02.717979Z","iopub.status.idle":"2022-12-22T14:17:03.956583Z","shell.execute_reply.started":"2022-12-22T14:17:02.717947Z","shell.execute_reply":"2022-12-22T14:17:03.955181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out.with_columns([\n    pl.col('full_preds').arr.eval(pl.element().cast(pl.Utf8)).alias(\"str\"),     \n]).with_columns([\n     pl.col('str').arr.join(\" \").alias('labels')\n ]).select([\n    'session_type',\n    'labels'\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T14:17:03.958738Z","iopub.execute_input":"2022-12-22T14:17:03.959106Z","iopub.status.idle":"2022-12-22T14:17:16.456098Z","shell.execute_reply.started":"2022-12-22T14:17:03.95908Z","shell.execute_reply":"2022-12-22T14:17:16.454658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit = out.with_columns([\n    pl.col('full_preds').arr.eval(pl.element().cast(pl.Utf8)).alias(\"str\"),     \n]).with_columns([\n     pl.col('str').arr.join(\" \").alias('labels')\n ]).select([\n    'session_type',\n    'labels'\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T14:18:07.321807Z","iopub.execute_input":"2022-12-22T14:18:07.32211Z","iopub.status.idle":"2022-12-22T14:18:19.744939Z","shell.execute_reply.started":"2022-12-22T14:18:07.322087Z","shell.execute_reply":"2022-12-22T14:18:19.743614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit.write_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T14:18:19.746902Z","iopub.execute_input":"2022-12-22T14:18:19.747168Z","iopub.status.idle":"2022-12-22T14:18:21.138403Z","shell.execute_reply.started":"2022-12-22T14:18:19.747143Z","shell.execute_reply":"2022-12-22T14:18:21.137737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pl.read_csv('/kaggle/input/otto-recommender-system/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T14:18:21.139832Z","iopub.execute_input":"2022-12-22T14:18:21.141103Z","iopub.status.idle":"2022-12-22T14:18:21.956986Z","shell.execute_reply.started":"2022-12-22T14:18:21.141068Z","shell.execute_reply":"2022-12-22T14:18:21.955866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-22T14:18:21.95865Z","iopub.execute_input":"2022-12-22T14:18:21.959135Z","iopub.status.idle":"2022-12-22T14:18:21.965862Z","shell.execute_reply.started":"2022-12-22T14:18:21.959109Z","shell.execute_reply":"2022-12-22T14:18:21.963868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T14:18:21.967408Z","iopub.execute_input":"2022-12-22T14:18:21.967799Z","iopub.status.idle":"2022-12-22T14:18:21.979852Z","shell.execute_reply.started":"2022-12-22T14:18:21.967773Z","shell.execute_reply":"2022-12-22T14:18:21.978972Z"},"trusted":true},"execution_count":null,"outputs":[]}]}