{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# OTTO – Multi-Objective Recommender System","metadata":{}},{"cell_type":"markdown","source":"Based on idea from [that](https://www.kaggle.com/code/zyhchasel/otto-fast-baseline) notebook.<br>\nLB 0.573<br>\nScore can be imroved to 0.575 by adding counters with shift -2 and 1","metadata":{}},{"cell_type":"code","source":"!pip install polars","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:13:02.597701Z","iopub.execute_input":"2023-02-15T12:13:02.598344Z","iopub.status.idle":"2023-02-15T12:13:19.468621Z","shell.execute_reply.started":"2023-02-15T12:13:02.598219Z","shell.execute_reply":"2023-02-15T12:13:19.467213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport polars as pl\nfrom datetime import datetime\nfrom tqdm import tqdm\nimport gc\nfrom collections import defaultdict\nimport math\nimport numpy as np\n\nfrom collections import Counter\nfrom sklearn.preprocessing import minmax_scale, normalize\nfrom multiprocess import Pool","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-15T12:13:19.470685Z","iopub.execute_input":"2023-02-15T12:13:19.471140Z","iopub.status.idle":"2023-02-15T12:13:20.231918Z","shell.execute_reply.started":"2023-02-15T12:13:19.471103Z","shell.execute_reply":"2023-02-15T12:13:20.230483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pl.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet').to_pandas()\ntest_df = pl.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet').to_pandas()","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:13:20.233819Z","iopub.execute_input":"2023-02-15T12:13:20.234884Z","iopub.status.idle":"2023-02-15T12:13:42.037766Z","shell.execute_reply.started":"2023-02-15T12:13:20.234831Z","shell.execute_reply":"2023-02-15T12:13:42.036341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask = train_df.groupby('session')['type'].count() != 1\ntrain_session = mask.loc[mask==True].index.values","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:13:42.040473Z","iopub.execute_input":"2023-02-15T12:13:42.041757Z","iopub.status.idle":"2023-02-15T12:13:52.419949Z","shell.execute_reply.started":"2023-02-15T12:13:42.041687Z","shell.execute_reply":"2023-02-15T12:13:52.418181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.query('session in @train_session').reset_index(drop=True)\ndel mask, train_session","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:13:52.421935Z","iopub.execute_input":"2023-02-15T12:13:52.422642Z","iopub.status.idle":"2023-02-15T12:14:12.770347Z","shell.execute_reply.started":"2023-02-15T12:13:52.422571Z","shell.execute_reply":"2023-02-15T12:14:12.768548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.sort_values(by=['session', 'ts'])\ntrain_df.drop(columns=['ts', 'type'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:14:12.772622Z","iopub.execute_input":"2023-02-15T12:14:12.773116Z","iopub.status.idle":"2023-02-15T12:18:25.191749Z","shell.execute_reply.started":"2023-02-15T12:14:12.773074Z","shell.execute_reply":"2023-02-15T12:18:25.189106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['session_next'] = train_df['session'].shift(-1).fillna(0).astype('int32')\ntrain_df['session_filter'] = train_df['session_next'] - train_df['session']\ntrain_df['aid_next'] = train_df['aid'].shift(-1).fillna(0).astype('int32')","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:18:25.195208Z","iopub.execute_input":"2023-02-15T12:18:25.195894Z","iopub.status.idle":"2023-02-15T12:18:32.346270Z","shell.execute_reply.started":"2023-02-15T12:18:25.195836Z","shell.execute_reply":"2023-02-15T12:18:32.344903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsim_aids = (train_df.loc[train_df['session_filter'] == 0, ['aid', 'aid_next']]\n            .groupby('aid')\n            .apply(lambda x: Counter(x.aid_next).most_common(20))\n            .to_dict()\n           )","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:18:32.348210Z","iopub.execute_input":"2023-02-15T12:18:32.348606Z","iopub.status.idle":"2023-02-15T12:23:09.269489Z","shell.execute_reply.started":"2023-02-15T12:18:32.348572Z","shell.execute_reply":"2023-02-15T12:23:09.267830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_df","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:23:09.271363Z","iopub.execute_input":"2023-02-15T12:23:09.271813Z","iopub.status.idle":"2023-02-15T12:23:09.299654Z","shell.execute_reply.started":"2023-02-15T12:23:09.271768Z","shell.execute_reply":"2023-02-15T12:23:09.298334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def divide(sim):\n    \n    a = {aid: top for aid, top in sim.items() if aid < len(sim) // 4}\n    b = {aid: top for aid, top in sim.items() if (aid >= len(sim) // 4) and (aid < len(sim) // 2)}\n    c = {aid: top for aid, top in sim.items() if (aid >= len(sim) // 2) and (aid < 3 * (len(sim) // 4))}\n    d = {aid: top for aid, top in sim.items() if (aid >= 3 * (len(sim) // 4))}\n   \n    return a, b, c, d","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:23:09.304838Z","iopub.execute_input":"2023-02-15T12:23:09.305419Z","iopub.status.idle":"2023-02-15T12:23:09.466909Z","shell.execute_reply.started":"2023-02-15T12:23:09.305367Z","shell.execute_reply":"2023-02-15T12:23:09.465523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sim(sim_a):\n    \n    from collections import Counter\n    import pandas as pd\n    from sklearn.preprocessing import minmax_scale\n    sim = {aid: Counter(pd.Series(minmax_scale(pd.Series(dict(top))), \n                                  index=dict(top).keys()).to_dict()) for aid, top in sim_a.items()}\n    \n    return sim","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:23:09.468631Z","iopub.execute_input":"2023-02-15T12:23:09.469174Z","iopub.status.idle":"2023-02-15T12:23:09.481987Z","shell.execute_reply.started":"2023-02-15T12:23:09.469128Z","shell.execute_reply":"2023-02-15T12:23:09.480568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a, b, c, d = divide(sim_aids)\ndel sim_aids","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:23:09.483889Z","iopub.execute_input":"2023-02-15T12:23:09.484402Z","iopub.status.idle":"2023-02-15T12:23:12.144744Z","shell.execute_reply.started":"2023-02-15T12:23:09.484355Z","shell.execute_reply":"2023-02-15T12:23:12.143325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwith Pool(4) as p:\n    some_res = p.map(sim, [a,b,c,d])","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:23:12.146243Z","iopub.execute_input":"2023-02-15T12:23:12.146615Z","iopub.status.idle":"2023-02-15T12:39:55.188025Z","shell.execute_reply.started":"2023-02-15T12:23:12.146583Z","shell.execute_reply":"2023-02-15T12:39:55.186347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sim_aids = {**some_res[0], **some_res[1], **some_res[2], **some_res[3]}\ndel a, b, c, d, some_res\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:39:55.191038Z","iopub.execute_input":"2023-02-15T12:39:55.191538Z","iopub.status.idle":"2023-02-15T12:39:59.511589Z","shell.execute_reply.started":"2023-02-15T12:39:55.191493Z","shell.execute_reply":"2023-02-15T12:39:59.509955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = test_df.sort_values([\"session\", \"type\", \"ts\"])","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:39:59.514033Z","iopub.execute_input":"2023-02-15T12:39:59.514492Z","iopub.status.idle":"2023-02-15T12:40:04.416438Z","shell.execute_reply.started":"2023-02-15T12:39:59.514442Z","shell.execute_reply":"2023-02-15T12:40:04.414798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsession_types = ['clicks', 'carts', 'orders']\ntest_session_AIDs = test_df.reset_index(drop=True).groupby('session')['aid'].apply(list)","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:40:04.419040Z","iopub.execute_input":"2023-02-15T12:40:04.419906Z","iopub.status.idle":"2023-02-15T12:40:37.551633Z","shell.execute_reply.started":"2023-02-15T12:40:04.419850Z","shell.execute_reply":"2023-02-15T12:40:37.550097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nlabels = []\n\nsession_num = len(test_session_AIDs)\n\nfor AIDs in tqdm(test_session_AIDs[:session_num]):\n                                    \n    if len(dict.fromkeys(AIDs)) >= 20:\n     \n        labels.append(list(dict.fromkeys(AIDs[::-1]))[:20])\n        \n    elif len(AIDs) == 1:\n   \n        new_aids = Counter()\n        new_aids.update(sim_aids.get(AIDs[0], Counter()))\n        temp_aids = [aid2 for aid2, cnt in new_aids.most_common(20) if (aid2 not in AIDs)]\n       \n        for aid in temp_aids:\n            new_aids.update(sim_aids.get(aid, Counter()))\n          \n        top_aids = [aid2 for aid2, cnt in new_aids.most_common(20) if (aid2 not in AIDs)]\n        final_rec_list = list(AIDs) + top_aids\n        \n        labels.append(final_rec_list[:20])\n            \n    else:\n       \n        AIDs = list(dict.fromkeys(AIDs[::-1]))\n        \n        recent_len = max(min(4,len(AIDs)),2)\n        new_aids = Counter()\n        \n        for aid in AIDs[:recent_len]:\n            new_aids.update(sim_aids.get(aid, Counter()))\n                    \n        top_aids = [aid2 for aid2, cnt in new_aids.most_common(20) if (aid2 not in AIDs)]\n       \n        final_rec_list = list(AIDs) + top_aids\n\n        labels.append(final_rec_list[:20])\n        ","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:40:37.553108Z","iopub.execute_input":"2023-02-15T12:40:37.553485Z","iopub.status.idle":"2023-02-15T12:45:36.423750Z","shell.execute_reply.started":"2023-02-15T12:40:37.553452Z","shell.execute_reply":"2023-02-15T12:45:36.422203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_as_strings = [' '.join([str(int(l)) for l in lls]) for lls in labels]","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:45:36.425832Z","iopub.execute_input":"2023-02-15T12:45:36.427103Z","iopub.status.idle":"2023-02-15T12:45:53.646376Z","shell.execute_reply.started":"2023-02-15T12:45:36.427048Z","shell.execute_reply":"2023-02-15T12:45:53.645168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = pd.DataFrame(data={'session_type': test_session_AIDs.index, 'labels': labels_as_strings})\n\nprediction_dfs = []\n\nfor st in tqdm(session_types):\n    modified_predictions = predictions.copy()\n    modified_predictions.session_type = modified_predictions.session_type.astype('str') + f'_{st}'\n    prediction_dfs.append(modified_predictions)\n\nsubmission = pd.concat(prediction_dfs).reset_index(drop=True)\nsubmission.to_csv('submission.csv', index=False)\n\ndel labels, labels_as_strings, predictions, prediction_dfs\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:45:53.647953Z","iopub.execute_input":"2023-02-15T12:45:53.648433Z","iopub.status.idle":"2023-02-15T12:46:22.912454Z","shell.execute_reply.started":"2023-02-15T12:45:53.648393Z","shell.execute_reply":"2023-02-15T12:46:22.910829Z"},"trusted":true},"execution_count":null,"outputs":[]}]}