{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install pickle5","metadata":{"execution":{"iopub.status.busy":"2022-12-03T23:53:42.364003Z","iopub.execute_input":"2022-12-03T23:53:42.364620Z","iopub.status.idle":"2022-12-03T23:53:55.541397Z","shell.execute_reply.started":"2022-12-03T23:53:42.364529Z","shell.execute_reply":"2022-12-03T23:53:55.539728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\n\nfrom datetime import datetime\nfrom tqdm import tqdm\n\nfrom collections import defaultdict\nimport math\nimport numpy as np\nimport random\nimport copy\nfrom collections import Counter","metadata":{"execution":{"iopub.status.busy":"2022-12-03T23:53:57.767557Z","iopub.execute_input":"2022-12-03T23:53:57.767946Z","iopub.status.idle":"2022-12-03T23:53:57.774541Z","shell.execute_reply.started":"2022-12-03T23:53:57.767912Z","shell.execute_reply":"2022-12-03T23:53:57.773193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')\ntest_df = pd.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')\n\nimport pickle5 as pickle\n\nwith open('../input/otto-full-optimized-memory-footprint/id2type.pkl', \"rb\") as fh:\n    id2type = pickle.load(fh)\nwith open('../input/otto-full-optimized-memory-footprint/type2id.pkl', \"rb\") as fh:\n    type2id = pickle.load(fh)\n    \nsample_sub_df = pd.read_csv('../input/otto-recommender-system/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-03T23:54:00.969129Z","iopub.execute_input":"2022-12-03T23:54:00.969563Z","iopub.status.idle":"2022-12-03T23:54:26.221653Z","shell.execute_reply.started":"2022-12-03T23:54:00.969528Z","shell.execute_reply":"2022-12-03T23:54:26.220516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = {\n    'train_session_num':3000000,\n}","metadata":{"execution":{"iopub.status.busy":"2022-12-03T23:54:28.151530Z","iopub.execute_input":"2022-12-03T23:54:28.151958Z","iopub.status.idle":"2022-12-03T23:54:28.157113Z","shell.execute_reply.started":"2022-12-03T23:54:28.151924Z","shell.execute_reply":"2022-12-03T23:54:28.156070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['aid'] = train_df['aid'].astype('int32').astype('str')\ntest_df['aid'] = test_df['aid'].astype('int32').astype('str')","metadata":{"execution":{"iopub.status.busy":"2022-12-03T23:54:30.452320Z","iopub.execute_input":"2022-12-03T23:54:30.452773Z","iopub.status.idle":"2022-12-03T23:56:55.223524Z","shell.execute_reply.started":"2022-12-03T23:54:30.452735Z","shell.execute_reply":"2022-12-03T23:56:55.222172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_session = random.sample(list(train_df['session'].unique()),config['train_session_num'])\ntrain_df = train_df.query('session in @train_session').reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-03T23:58:17.457174Z","iopub.execute_input":"2022-12-03T23:58:17.458520Z","iopub.status.idle":"2022-12-03T23:58:51.892246Z","shell.execute_reply.started":"2022-12-03T23:58:17.458472Z","shell.execute_reply":"2022-12-03T23:58:51.890984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['time_stamp'] = pd.to_datetime(train_df['ts'],unit='s').dt.strftime('%Y-%m-%d')\ntest_df['time_stamp'] = pd.to_datetime(test_df['ts'],unit='s').dt.strftime('%Y-%m-%d')","metadata":{"execution":{"iopub.status.busy":"2022-12-03T23:59:54.982329Z","iopub.execute_input":"2022-12-03T23:59:54.982803Z","iopub.status.idle":"2022-12-04T00:04:41.515042Z","shell.execute_reply.started":"2022-12-03T23:59:54.982770Z","shell.execute_reply":"2022-12-04T00:04:41.513292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_pairs(df):\n    df = df.sort_values(by=['session','ts'])\n    df['aid_next'] = df['aid'].shift(-1)\n    df['session_day'] = df['session'].astype('str')+'_'+df['time_stamp']\n    df['session_day_count'] = df['session_day'].map(df['session_day'].value_counts())\n    df['ranking'] = df.groupby(['session_day'])['ts'].rank(method='first', ascending=True)\n    df = df.query('session_day_count!=ranking').reset_index(drop=True)\n    \n    sim_aids = df.groupby('aid').apply(lambda df: Counter(df.aid_next).most_common(50)).to_dict()\n    sim_aids = {aid: Counter(dict(top)) for aid, top in sim_aids.items()}\n    return sim_aids","metadata":{"execution":{"iopub.status.busy":"2022-12-04T00:04:47.062641Z","iopub.execute_input":"2022-12-04T00:04:47.064187Z","iopub.status.idle":"2022-12-04T00:04:47.073784Z","shell.execute_reply.started":"2022-12-04T00:04:47.064142Z","shell.execute_reply":"2022-12-04T00:04:47.072622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sim_aids = generate_pairs(train_df)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T00:04:49.991299Z","iopub.execute_input":"2022-12-04T00:04:49.991701Z","iopub.status.idle":"2022-12-04T00:11:40.466976Z","shell.execute_reply.started":"2022-12-04T00:04:49.991670Z","shell.execute_reply":"2022-12-04T00:11:40.465555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-12-04T00:16:01.477274Z","iopub.execute_input":"2022-12-04T00:16:01.477762Z","iopub.status.idle":"2022-12-04T00:16:01.505545Z","shell.execute_reply.started":"2022-12-04T00:16:01.477726Z","shell.execute_reply":"2022-12-04T00:16:01.504424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sim_aids","metadata":{"execution":{"iopub.status.busy":"2022-12-04T00:16:04.136915Z","iopub.execute_input":"2022-12-04T00:16:04.137479Z","iopub.status.idle":"2022-12-04T00:16:04.504765Z","shell.execute_reply.started":"2022-12-04T00:16:04.137417Z","shell.execute_reply":"2022-12-04T00:16:04.500525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def recommend(aids,popular_items):\n    \n    if len(aids) >= 20:\n        return aids[-20:]\n\n    aids = set(aids)\n    new_aids = Counter()\n    for aid in aids:\n        new_aids.update(sim_aids.get(aid, Counter()))\n    \n    top_aids2 = [aid2 for aid2, cnt in new_aids.most_common(40) if aid2 not in aids] \n    final_rec_list = list(aids) + top_aids2[:20 - len(aids)]\n    \n    if len(final_rec_list)<20:\n        return final_rec_list + popular_items[:20-len(final_rec_list)]\n    else:\n        return final_rec_list","metadata":{"execution":{"iopub.status.busy":"2022-12-04T00:16:18.045051Z","iopub.execute_input":"2022-12-04T00:16:18.045449Z","iopub.status.idle":"2022-12-04T00:16:18.053722Z","shell.execute_reply.started":"2022-12-04T00:16:18.045417Z","shell.execute_reply":"2022-12-04T00:16:18.052551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = test_df.sort_values([\"session\", \"type\", \"ts\"])\ntest_session_dict = test_df.groupby('session')['aid'].agg(list).to_dict()\nsession_id_list = []\nitem_id_list = []\n\npopular_items = list(train_df['aid'].value_counts().index)\n\nfor session_id,session_item_list in tqdm(test_session_dict.items()):\n    item_list = recommend(session_item_list,popular_items)\n    \n    session_id_list.append(session_id)\n    item_id_list.append(list(item_list))\n\nres_df = pd.DataFrame()\nres_df['session_type'] = session_id_list\nres_df['labels'] = [' '.join([str(l) for l in lls]) for lls in item_id_list]","metadata":{"execution":{"iopub.status.busy":"2022-12-04T00:16:20.869038Z","iopub.execute_input":"2022-12-04T00:16:20.869435Z","iopub.status.idle":"2022-12-04T00:20:00.193564Z","shell.execute_reply.started":"2022-12-04T00:16:20.869404Z","shell.execute_reply":"2022-12-04T00:20:00.192396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_list = []\nfor type_ in [0,1,2]:\n    temp_df = copy.deepcopy(res_df)\n    temp_df['session_type'] = temp_df['session_type'].apply(lambda x:'{}_{}'.format(x,id2type[type_]))\n    res_list.append(temp_df)\nres_df = pd.concat(res_list,axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T00:21:13.467708Z","iopub.execute_input":"2022-12-04T00:21:13.468293Z","iopub.status.idle":"2022-12-04T00:21:16.592494Z","shell.execute_reply.started":"2022-12-04T00:21:13.468243Z","shell.execute_reply":"2022-12-04T00:21:16.591418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_df.to_csv('baseline.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T00:22:05.326099Z","iopub.execute_input":"2022-12-04T00:22:05.326546Z","iopub.status.idle":"2022-12-04T00:22:26.087018Z","shell.execute_reply.started":"2022-12-04T00:22:05.326507Z","shell.execute_reply":"2022-12-04T00:22:26.085792Z"},"trusted":true},"execution_count":null,"outputs":[]}]}