{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install pickle5","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-11T15:41:16.489184Z","iopub.execute_input":"2022-12-11T15:41:16.489827Z","iopub.status.idle":"2022-12-11T15:41:30.188276Z","shell.execute_reply.started":"2022-12-11T15:41:16.489690Z","shell.execute_reply":"2022-12-11T15:41:30.187227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom datetime import datetime\nfrom tqdm import tqdm\nfrom collections import defaultdict\nimport math\nimport numpy as np\nimport random\nimport copy\nfrom collections import Counter","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:41:30.191242Z","iopub.execute_input":"2022-12-11T15:41:30.191808Z","iopub.status.idle":"2022-12-11T15:41:30.199549Z","shell.execute_reply.started":"2022-12-11T15:41:30.191753Z","shell.execute_reply":"2022-12-11T15:41:30.198015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')\n# test_df = pd.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')\n\nimport pickle5 as pickle\n\nwith open('../input/otto-full-optimized-memory-footprint/id2type.pkl', \"rb\") as fh:\n    id2type = pickle.load(fh)\nwith open('../input/otto-full-optimized-memory-footprint/type2id.pkl', \"rb\") as fh:\n    type2id = pickle.load(fh)\n    \nsample_sub_df = pd.read_csv('../input/otto-recommender-system/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:41:30.201722Z","iopub.execute_input":"2022-12-11T15:41:30.202182Z","iopub.status.idle":"2022-12-11T15:41:55.091945Z","shell.execute_reply.started":"2022-12-11T15:41:30.202146Z","shell.execute_reply":"2022-12-11T15:41:55.090711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = {\n    'train_session_num':12899779,\n}","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:41:55.093812Z","iopub.execute_input":"2022-12-11T15:41:55.094194Z","iopub.status.idle":"2022-12-11T15:41:55.101155Z","shell.execute_reply.started":"2022-12-11T15:41:55.094162Z","shell.execute_reply":"2022-12-11T15:41:55.099675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['aid'] = train_df['aid'].astype('int32').astype('str')\n# test_df['aid'] = test_df['aid'].astype('int32').astype('str')","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:41:55.105604Z","iopub.execute_input":"2022-12-11T15:41:55.106020Z","iopub.status.idle":"2022-12-11T15:44:13.302534Z","shell.execute_reply.started":"2022-12-11T15:41:55.105986Z","shell.execute_reply":"2022-12-11T15:44:13.301032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_session = random.sample(list(train_df['session'].unique()),config['train_session_num'])\ntrain_df = train_df.query('session in @train_session').reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:44:13.304249Z","iopub.execute_input":"2022-12-11T15:44:13.305494Z","iopub.status.idle":"2022-12-11T15:45:07.381800Z","shell.execute_reply.started":"2022-12-11T15:44:13.305448Z","shell.execute_reply":"2022-12-11T15:45:07.378874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#清理内存\nimport gc\ndel train_session\ngc.collect()\n\ntrain_df['time_stamp'] = pd.to_datetime(train_df['ts'],unit='s').dt.strftime('%Y-%m-%d')\n# test_df['time_stamp'] = pd.to_datetime(test_df['ts'],unit='s').dt.strftime('%Y-%m-%d')","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:45:07.387131Z","iopub.execute_input":"2022-12-11T15:45:07.387805Z","iopub.status.idle":"2022-12-11T15:59:48.931917Z","shell.execute_reply.started":"2022-12-11T15:45:07.387725Z","shell.execute_reply":"2022-12-11T15:59:48.930194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_pairs(df):\n    df = df.sort_values(by=['session','ts'])\n    df['aid_next'] = df['aid'].shift(-1)\n    df['session_day'] = df['session'].astype('str')+'_'+df['time_stamp']\n    df['session_day_count'] = df['session_day'].map(df['session_day'].value_counts())\n    df['ranking'] = df.groupby(['session_day'])['ts'].rank(method='first', ascending=True)\n    df = df.query('session_day_count!=ranking').reset_index(drop=True)\n    \n    sim_aids = df.groupby('aid').apply(lambda df: Counter(df.aid_next).most_common(50)).to_dict()\n    sim_aids = {aid: Counter(dict(top)) for aid, top in sim_aids.items()}\n    return sim_aids","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:59:48.933832Z","iopub.execute_input":"2022-12-11T15:59:48.935191Z","iopub.status.idle":"2022-12-11T15:59:48.956241Z","shell.execute_reply.started":"2022-12-11T15:59:48.935137Z","shell.execute_reply":"2022-12-11T15:59:48.955086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sim_aids = generate_pairs(train_df)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:59:48.957674Z","iopub.execute_input":"2022-12-11T15:59:48.958765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def recommend(aids,popular_items):\n    \n    if len(aids) >= 20:\n        return aids[-20:]\n\n    aids = set(aids)\n    new_aids = Counter()\n    for aid in aids:\n        new_aids.update(sim_aids.get(aid, Counter()))\n    \n    top_aids2 = [aid2 for aid2, cnt in new_aids.most_common(40) if aid2 not in aids] \n    final_rec_list = list(aids) + top_aids2[:20 - len(aids)]\n    \n    if len(final_rec_list)<20:\n        return final_rec_list + popular_items[:20-len(final_rec_list)]\n    else:\n        return final_rec_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#清理内存\ndel train_df\ngc.collect()\n\ntest_df = pd.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')\ntest_df['aid'] = test_df['aid'].astype('int32').astype('str')\ntest_df['time_stamp'] = pd.to_datetime(test_df['ts'],unit='s').dt.strftime('%Y-%m-%d')\n\ntest_df = test_df.sort_values([\"session\", \"type\", \"ts\"])\ntest_session_dict = test_df.groupby('session')['aid'].agg(list).to_dict()\nsession_id_list = []\nitem_id_list = []\n\npopular_items = list(train_df['aid'].value_counts().index)\n\nfor session_id,session_item_list in tqdm(test_session_dict.items()):\n    item_list = recommend(session_item_list,popular_items)\n    \n    session_id_list.append(session_id)\n    item_id_list.append(list(item_list))\n\nres_df = pd.DataFrame()\nres_df['session_type'] = session_id_list\nres_df['labels'] = [' '.join([str(l) for l in lls]) for lls in item_id_list]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_list = []\nfor type_ in [0,1,2]:\n    temp_df = copy.deepcopy(res_df)\n    temp_df['session_type'] = temp_df['session_type'].apply(lambda x:'{}_{}'.format(x,id2type[type_]))\n    res_list.append(temp_df)\nres_df = pd.concat(res_list,axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_df.to_csv('submission.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}