{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom collections import Counter\nfrom heapq import nlargest\nfrom tqdm.notebook import tqdm\nimport gc\nimport json\nfrom itertools import chain","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-10T10:13:48.963722Z","iopub.execute_input":"2022-12-10T10:13:48.964413Z","iopub.status.idle":"2022-12-10T10:13:48.972463Z","shell.execute_reply.started":"2022-12-10T10:13:48.964356Z","shell.execute_reply":"2022-12-10T10:13:48.971187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunks = pd.read_json('/kaggle/input/otto-recommender-system/test.jsonl', lines=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:13:48.975038Z","iopub.execute_input":"2022-12-10T10:13:48.976853Z","iopub.status.idle":"2022-12-10T10:14:27.011834Z","shell.execute_reply.started":"2022-12-10T10:13:48.976793Z","shell.execute_reply":"2022-12-10T10:14:27.009960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunks_exploded = chunks.explode('events')","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:14:27.014221Z","iopub.execute_input":"2022-12-10T10:14:27.014845Z","iopub.status.idle":"2022-12-10T10:14:33.916814Z","shell.execute_reply.started":"2022-12-10T10:14:27.014796Z","shell.execute_reply":"2022-12-10T10:14:33.914934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_aid(input):\n    return input['aid']\n\ndef get_type(input):\n    return input['type']\n\ndef get_ts(input):\n    return input['ts']\n\nchunks_exploded.loc[:, 'aid'] = chunks_exploded.loc[:, 'events'].map(get_aid)\nchunks_exploded.loc[:, 'type'] = chunks_exploded.loc[:,'events'].map(get_type)\nchunks_exploded.loc[:, 'ts'] = chunks_exploded.loc[:,'events'].map(get_ts)","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:14:33.919453Z","iopub.execute_input":"2022-12-10T10:14:33.920064Z","iopub.status.idle":"2022-12-10T10:14:47.781221Z","shell.execute_reply.started":"2022-12-10T10:14:33.920010Z","shell.execute_reply":"2022-12-10T10:14:47.779497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunks_exploded.drop(columns=['events'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:14:47.786498Z","iopub.execute_input":"2022-12-10T10:14:47.787006Z","iopub.status.idle":"2022-12-10T10:14:49.649549Z","shell.execute_reply.started":"2022-12-10T10:14:47.786953Z","shell.execute_reply":"2022-12-10T10:14:49.648080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunks_resettet = chunks_exploded.groupby(['aid', 'type']).count().reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:14:49.651453Z","iopub.execute_input":"2022-12-10T10:14:49.651999Z","iopub.status.idle":"2022-12-10T10:14:52.803824Z","shell.execute_reply.started":"2022-12-10T10:14:49.651921Z","shell.execute_reply":"2022-12-10T10:14:52.802432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_common_types = {}","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:14:52.805997Z","iopub.execute_input":"2022-12-10T10:14:52.806455Z","iopub.status.idle":"2022-12-10T10:14:52.813632Z","shell.execute_reply.started":"2022-12-10T10:14:52.806416Z","shell.execute_reply":"2022-12-10T10:14:52.812209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_common_clicks = chunks_resettet.loc[chunks_resettet['type'] == 'clicks'].sort_values(by=['session'], ascending=False)['session'].values[:20]","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:14:52.816396Z","iopub.execute_input":"2022-12-10T10:14:52.816919Z","iopub.status.idle":"2022-12-10T10:14:53.062813Z","shell.execute_reply.started":"2022-12-10T10:14:52.816868Z","shell.execute_reply":"2022-12-10T10:14:53.061550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_common_clicks = chunks_resettet.loc[chunks_resettet['type'] == 'clicks'].sort_values(by=['session'], ascending=False)['session'].values[:20]\nmost_common_types['clicks'] = [str(el) for el in list(most_common_clicks)]","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:14:53.065090Z","iopub.execute_input":"2022-12-10T10:14:53.065456Z","iopub.status.idle":"2022-12-10T10:14:53.288503Z","shell.execute_reply.started":"2022-12-10T10:14:53.065425Z","shell.execute_reply":"2022-12-10T10:14:53.287195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_common_orders = chunks_resettet.loc[chunks_resettet['type'] == 'orders'].sort_values(by=['session'], ascending=False)['session'].values[:20]\nmost_common_types['orders'] = [str(el) for el in list(most_common_orders)]","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:14:53.290182Z","iopub.execute_input":"2022-12-10T10:14:53.290567Z","iopub.status.idle":"2022-12-10T10:14:53.382647Z","shell.execute_reply.started":"2022-12-10T10:14:53.290533Z","shell.execute_reply":"2022-12-10T10:14:53.381270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_common_carts =chunks_resettet.loc[chunks_resettet['type'] == 'carts'].sort_values(by=['session'], ascending=False)['session'].values[:20]\nmost_common_types['carts'] = [str(el) for el in list(most_common_carts)]","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:14:53.384369Z","iopub.execute_input":"2022-12-10T10:14:53.384847Z","iopub.status.idle":"2022-12-10T10:14:53.499010Z","shell.execute_reply.started":"2022-12-10T10:14:53.384800Z","shell.execute_reply":"2022-12-10T10:14:53.497843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunks_exploded['type'] = ' ' + chunks_exploded['type']","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:14:53.500554Z","iopub.execute_input":"2022-12-10T10:14:53.501045Z","iopub.status.idle":"2022-12-10T10:14:54.951970Z","shell.execute_reply.started":"2022-12-10T10:14:53.501001Z","shell.execute_reply":"2022-12-10T10:14:54.950093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunks_exploded = chunks_exploded.astype({'aid': 'str'})","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:14:54.953856Z","iopub.execute_input":"2022-12-10T10:14:54.955331Z","iopub.status.idle":"2022-12-10T10:14:59.966995Z","shell.execute_reply.started":"2022-12-10T10:14:54.955268Z","shell.execute_reply":"2022-12-10T10:14:59.965414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunks_exploded['aid'] = ' ' + chunks_exploded['aid']","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:14:59.973461Z","iopub.execute_input":"2022-12-10T10:14:59.973967Z","iopub.status.idle":"2022-12-10T10:15:01.324519Z","shell.execute_reply.started":"2022-12-10T10:14:59.973930Z","shell.execute_reply":"2022-12-10T10:15:01.323085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aid = chunks_exploded.groupby('session')['aid'].agg('sum').reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:15:01.326261Z","iopub.execute_input":"2022-12-10T10:15:01.326678Z","iopub.status.idle":"2022-12-10T10:15:13.417282Z","shell.execute_reply.started":"2022-12-10T10:15:01.326640Z","shell.execute_reply":"2022-12-10T10:15:13.415929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_aid(x):\n    x=x.strip()\n    cnt = Counter()\n    x_as_array = x.split(' ')\n    for aid in x_as_array:\n        cnt[aid] += 1\n    return cnt","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:15:13.419740Z","iopub.execute_input":"2022-12-10T10:15:13.420147Z","iopub.status.idle":"2022-12-10T10:15:13.426673Z","shell.execute_reply.started":"2022-12-10T10:15:13.420112Z","shell.execute_reply":"2022-12-10T10:15:13.425231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aid['aid_counter'] = session_aid['aid'].map(count_aid)","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:15:13.428657Z","iopub.execute_input":"2022-12-10T10:15:13.429085Z","iopub.status.idle":"2022-12-10T10:15:21.280532Z","shell.execute_reply.started":"2022-12-10T10:15:13.429012Z","shell.execute_reply":"2022-12-10T10:15:21.279494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aid['largest_aids'] = session_aid['aid_counter'].map(lambda x: nlargest(20, x, key=x.get))","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:15:21.282088Z","iopub.execute_input":"2022-12-10T10:15:21.282672Z","iopub.status.idle":"2022-12-10T10:15:24.875098Z","shell.execute_reply.started":"2022-12-10T10:15:21.282634Z","shell.execute_reply":"2022-12-10T10:15:24.873847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aid_pruned = session_aid[['session', 'largest_aids']]","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:15:24.877102Z","iopub.execute_input":"2022-12-10T10:15:24.877797Z","iopub.status.idle":"2022-12-10T10:15:25.350909Z","shell.execute_reply.started":"2022-12-10T10:15:24.877738Z","shell.execute_reply":"2022-12-10T10:15:25.349841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_clicks_carts_orders(x):\n    counters = []\n    for act in ['clicks', 'carts', 'orders']:\n        counters.append(most_common_types[act])\n    return counters\n\nsession_aid_pruned.loc[: ,'aids_added'] = session_aid.loc[:, 'largest_aids'].map(map_clicks_carts_orders)","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:17:09.834073Z","iopub.execute_input":"2022-12-10T10:17:09.835773Z","iopub.status.idle":"2022-12-10T10:17:11.581266Z","shell.execute_reply.started":"2022-12-10T10:17:09.835693Z","shell.execute_reply":"2022-12-10T10:17:11.579312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aid_pruned = session_aid_pruned.explode('aids_added') ","metadata":{"execution":{"iopub.status.busy":"2022-12-10T09:52:45.163274Z","iopub.execute_input":"2022-12-10T09:52:45.163676Z","iopub.status.idle":"2022-12-10T09:52:48.213180Z","shell.execute_reply.started":"2022-12-10T09:52:45.163642Z","shell.execute_reply":"2022-12-10T09:52:48.211521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aid_pruned['all_aids'] = session_aid_pruned['largest_aids'] + session_aid_pruned['aids_added']","metadata":{"execution":{"iopub.status.busy":"2022-12-10T09:54:37.075005Z","iopub.execute_input":"2022-12-10T09:54:37.075539Z","iopub.status.idle":"2022-12-10T09:54:55.600882Z","shell.execute_reply.started":"2022-12-10T09:54:37.075501Z","shell.execute_reply":"2022-12-10T09:54:55.599382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aid_pruned['all_aids'] = session_aid_pruned['all_aids'].map(lambda x: x[:20])","metadata":{"execution":{"iopub.status.busy":"2022-12-10T09:55:11.010435Z","iopub.execute_input":"2022-12-10T09:55:11.011234Z","iopub.status.idle":"2022-12-10T09:55:31.660309Z","shell.execute_reply.started":"2022-12-10T09:55:11.011194Z","shell.execute_reply":"2022-12-10T09:55:31.658838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aid_pruned['len_all_aids'] = session_aid_pruned['all_aids'].map(lambda x: len(x))","metadata":{"execution":{"iopub.status.busy":"2022-12-10T09:55:57.820571Z","iopub.execute_input":"2022-12-10T09:55:57.821157Z","iopub.status.idle":"2022-12-10T09:56:00.191224Z","shell.execute_reply.started":"2022-12-10T09:55:57.821111Z","shell.execute_reply":"2022-12-10T09:56:00.189786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aid_pruned['labels'] = session_aid_pruned['all_aids'].map(lambda x: \" \".join(x))","metadata":{"execution":{"iopub.status.busy":"2022-12-10T09:56:44.306883Z","iopub.execute_input":"2022-12-10T09:56:44.307466Z","iopub.status.idle":"2022-12-10T09:56:47.579027Z","shell.execute_reply.started":"2022-12-10T09:56:44.307424Z","shell.execute_reply":"2022-12-10T09:56:47.577605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aid_pruned.drop(columns=['largest_aids','aids_added','all_aids','len_all_aids'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-10T09:57:53.785412Z","iopub.execute_input":"2022-12-10T09:57:53.786042Z","iopub.status.idle":"2022-12-10T09:57:57.351143Z","shell.execute_reply.started":"2022-12-10T09:57:53.785991Z","shell.execute_reply":"2022-12-10T09:57:57.349356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/otto-recommender-system/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-10T09:58:10.833278Z","iopub.execute_input":"2022-12-10T09:58:10.833764Z","iopub.status.idle":"2022-12-10T09:58:17.579093Z","shell.execute_reply.started":"2022-12-10T09:58:10.833716Z","shell.execute_reply":"2022-12-10T09:58:17.577474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['labels'] = session_aid_pruned['labels'].values","metadata":{"execution":{"iopub.status.busy":"2022-12-10T09:59:21.999312Z","iopub.execute_input":"2022-12-10T09:59:21.999844Z","iopub.status.idle":"2022-12-10T09:59:22.391114Z","shell.execute_reply.started":"2022-12-10T09:59:21.999807Z","shell.execute_reply":"2022-12-10T09:59:22.389663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}