{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ### colab\n# from google.colab import drive\n# drive.mount('/content/drive')\n\n# %cd /content/drive/MyDrive/Projects/OTTO/working","metadata":{"id":"1MkDCevzqYq7","executionInfo":{"status":"ok","timestamp":1667854332487,"user_tz":-420,"elapsed":5360,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"e21e5086-91eb-4633-e131-a6ff4fbb8071","execution":{"iopub.status.busy":"2022-11-15T08:01:10.079662Z","iopub.execute_input":"2022-11-15T08:01:10.080430Z","iopub.status.idle":"2022-11-15T08:01:10.111957Z","shell.execute_reply.started":"2022-11-15T08:01:10.080271Z","shell.execute_reply":"2022-11-15T08:01:10.110871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data","metadata":{"id":"l_CQeHNCsWQt"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_theme()\n\nfrom datetime import timedelta\nfrom tqdm.notebook import tqdm\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"id":"_t-MkHnOqpo7","executionInfo":{"status":"ok","timestamp":1667854334124,"user_tz":-420,"elapsed":1638,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"execution":{"iopub.status.busy":"2022-11-15T08:01:10.113753Z","iopub.execute_input":"2022-11-15T08:01:10.114288Z","iopub.status.idle":"2022-11-15T08:01:10.774872Z","shell.execute_reply.started":"2022-11-15T08:01:10.114248Z","shell.execute_reply":"2022-11-15T08:01:10.773690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Path\n\nDATA_PATH = '../input/otto-recommender-system'\nTRAIN_PATH = f'{DATA_PATH}/train.jsonl'\nTEST_PATH = f'{DATA_PATH}/test.jsonl'\nSAMPLE_SUBMISSION_PATH = f'{DATA_PATH}/sample_submission.csv'","metadata":{"id":"nVODjvqUquE8","executionInfo":{"status":"ok","timestamp":1667854334124,"user_tz":-420,"elapsed":2,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"execution":{"iopub.status.busy":"2022-11-15T08:01:10.776231Z","iopub.execute_input":"2022-11-15T08:01:10.776979Z","iopub.status.idle":"2022-11-15T08:01:10.785114Z","shell.execute_reply.started":"2022-11-15T08:01:10.776942Z","shell.execute_reply":"2022-11-15T08:01:10.784248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Lets check how many lines the training data has!\n\n# with open(TRAIN_PATH, 'r') as f:\n#     print(f\"We have {len(f.readlines()):,} lines in the training data\")\n    ","metadata":{"id":"zqIWWsMKqwGC","executionInfo":{"status":"ok","timestamp":1667854418531,"user_tz":-420,"elapsed":84409,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"b9427442-f06c-4719-bb87-60ac484e849c","execution":{"iopub.status.busy":"2022-11-15T08:01:10.786557Z","iopub.execute_input":"2022-11-15T08:01:10.787698Z","iopub.status.idle":"2022-11-15T08:01:10.801678Z","shell.execute_reply.started":"2022-11-15T08:01:10.787661Z","shell.execute_reply":"2022-11-15T08:01:10.800259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Load in a sample to a pandas df\n\n# sample_size = 150000\n\n# chunks = pd.read_json(TRAIN_PATH, lines=True, chunksize = sample_size)\n\n# for c in chunks:\n#     sample_train_df = c\n#     break","metadata":{"id":"3nKLm4moqxbn","executionInfo":{"status":"ok","timestamp":1667854430780,"user_tz":-420,"elapsed":12273,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"execution":{"iopub.status.busy":"2022-11-15T08:01:10.805529Z","iopub.execute_input":"2022-11-15T08:01:10.805968Z","iopub.status.idle":"2022-11-15T08:01:10.816369Z","shell.execute_reply.started":"2022-11-15T08:01:10.805930Z","shell.execute_reply":"2022-11-15T08:01:10.815012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_train_df.set_index('session', drop=True, inplace=True)\n# sample_train_df.head()","metadata":{"id":"UzzmIGiBqzA9","executionInfo":{"status":"ok","timestamp":1667854430781,"user_tz":-420,"elapsed":51,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"14cc9de2-b304-4901-f9a4-5753bdd51b00","execution":{"iopub.status.busy":"2022-11-15T08:01:10.817729Z","iopub.execute_input":"2022-11-15T08:01:10.818422Z","iopub.status.idle":"2022-11-15T08:01:10.829969Z","shell.execute_reply.started":"2022-11-15T08:01:10.818384Z","shell.execute_reply":"2022-11-15T08:01:10.828784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_train_df.iloc[0]['events']","metadata":{"id":"3p_DFDt3q1Yt","executionInfo":{"status":"ok","timestamp":1667854430781,"user_tz":-420,"elapsed":30,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"8678bb1d-245c-40c2-9666-aa7905e25e99","execution":{"iopub.status.busy":"2022-11-15T08:01:10.831547Z","iopub.execute_input":"2022-11-15T08:01:10.832169Z","iopub.status.idle":"2022-11-15T08:01:10.844331Z","shell.execute_reply.started":"2022-11-15T08:01:10.832134Z","shell.execute_reply":"2022-11-15T08:01:10.843248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### EDA","metadata":{"id":"5nHpLxeUsIFF"}},{"cell_type":"code","source":"# idx_example_session = 0\n# example_session = sample_train_df.iloc[idx_example_session]['events']","metadata":{"id":"IjNE8Wlar9Be","executionInfo":{"status":"ok","timestamp":1667854430782,"user_tz":-420,"elapsed":27,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"execution":{"iopub.status.busy":"2022-11-15T08:01:10.846115Z","iopub.execute_input":"2022-11-15T08:01:10.847505Z","iopub.status.idle":"2022-11-15T08:01:10.859494Z","shell.execute_reply.started":"2022-11-15T08:01:10.847459Z","shell.execute_reply":"2022-11-15T08:01:10.858362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(f'example_session: {idx_example_session}')\n# print(f'This session was {len(example_session)} actions long \\n')\n# print(f'The first action in the session: \\n {example_session[0]} \\n')\n\n# # Time of session\n# time_elapsed = example_session[-1][\"ts\"] - example_session[0][\"ts\"]\n# # The timestamp is in microseconds since 00:00:00 UTC on 1 January 1970\n# print(f'The first session elapsed: {str(timedelta(microseconds=time_elapsed))} \\n')\n\n# # Count the frequency of actions within the session\n# action_counts = {}\n# for action in example_session:\n#     action_counts[action['type']] = action_counts.get(action['type'], 0) + 1  \n# print(f'This session contains the following frequency of actions: \\n{action_counts}')","metadata":{"id":"NRtzRreUs6DD","executionInfo":{"status":"ok","timestamp":1667854430782,"user_tz":-420,"elapsed":27,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"38d2cbb0-7159-43c3-b666-6b382e90f89d","execution":{"iopub.status.busy":"2022-11-15T08:01:10.861404Z","iopub.execute_input":"2022-11-15T08:01:10.862728Z","iopub.status.idle":"2022-11-15T08:01:10.875203Z","shell.execute_reply.started":"2022-11-15T08:01:10.862650Z","shell.execute_reply":"2022-11-15T08:01:10.873880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# action_counts_list, article_id_counts_list, \\\n# session_length_time_list, session_length_action_list = ([] for i in range(4))\n# overall_action_counts = {}\n# overall_article_id_counts = {}\n\n# for i, row in tqdm(sample_train_df.iterrows(), total=len(sample_train_df)):\n#   actions = row['events']\n\n#   action_count = {}\n#   article_id_counts = {}\n#   for action in actions:\n#     action_counts[action['type']] = action_counts.get(action['type'], 0) + 1\n#     article_id_counts[action['aid']] = article_id_counts.get(action['aid'], 0) + 1\n\n#     overall_action_counts[action['type']] = overall_action_counts.get(action['type'], 0) + 1\n#     overall_article_id_counts[action['aid']] = overall_article_id_counts.get(action['aid'], 0) + 1\n  \n#   # Get the length of the session\n#   session_length_time = actions[-1]['ts'] - actions[0]['ts']\n\n#   action_counts_list.append(action_counts)\n  \n#   article_id_counts_list.append(article_id_counts)\n#   session_length_time_list.append(session_length_time)\n#   session_length_action_list.append(len(actions))\n\n# sample_train_df['action_counts'] = action_counts_list\n# sample_train_df['article_id_counts'] = article_id_counts_list\n# sample_train_df['session_length_unix'] = session_length_time_list\n# sample_train_df['session_length_minutes'] = sample_train_df['session_length_unix'] / (60 * 1e6)  # Convert to minutes\n# sample_train_df['session_length_action'] = session_length_action_list","metadata":{"id":"aAi_xEU3uEhO","executionInfo":{"status":"ok","timestamp":1667854458692,"user_tz":-420,"elapsed":27929,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"2b3c0d39-eb2d-40c7-a461-d90dd4e4c018","execution":{"iopub.status.busy":"2022-11-15T08:01:10.877678Z","iopub.execute_input":"2022-11-15T08:01:10.878508Z","iopub.status.idle":"2022-11-15T08:01:10.892245Z","shell.execute_reply.started":"2022-11-15T08:01:10.878456Z","shell.execute_reply":"2022-11-15T08:01:10.891075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_train_df.head()","metadata":{"id":"fCZZuu2h3rZT","executionInfo":{"status":"ok","timestamp":1667854458693,"user_tz":-420,"elapsed":29,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"e322a52e-83a1-4144-f927-a8f6ffece079","execution":{"iopub.status.busy":"2022-11-15T08:01:10.894087Z","iopub.execute_input":"2022-11-15T08:01:10.894946Z","iopub.status.idle":"2022-11-15T08:01:10.912767Z","shell.execute_reply.started":"2022-11-15T08:01:10.894896Z","shell.execute_reply":"2022-11-15T08:01:10.910786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# total_actions = sum(overall_action_counts.values())\n# print(f'Total actions: {total_actions}\\n')\n\n# plt.figure(figsize= (8, 6))\n# sns.barplot(x=list(overall_action_counts.keys()), y=[i/total_actions for i in overall_action_counts.values()]);\n# plt.title(f'Action frequency', fontsize=12)\n# plt.ylabel('Count', fontsize=12)\n# plt.xlabel('Category', fontsize=12)\n# plt.show()","metadata":{"id":"NdlucIyyhBU7","executionInfo":{"status":"ok","timestamp":1667854458693,"user_tz":-420,"elapsed":27,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"e4bcf1c3-7695-4674-a561-8db1300b3de6","execution":{"iopub.status.busy":"2022-11-15T08:01:10.914298Z","iopub.execute_input":"2022-11-15T08:01:10.915052Z","iopub.status.idle":"2022-11-15T08:01:10.927555Z","shell.execute_reply.started":"2022-11-15T08:01:10.915007Z","shell.execute_reply":"2022-11-15T08:01:10.926497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, axes = plt.subplots(1, 2, figsize=(16, 6))\n\n# sns.distplot(sample_train_df['session_length_action'], color='y', bins= 70, ax=axes[0], kde= False)\n# axes[0].set_title(\"distribution of the number of actions each session\")\n# axes[0].set_xlabel('Number of actions')\n# axes[0].set_ylabel('Density')\n# axes[0].axvline(sample_train_df['session_length_action'].mean(), color= 'r', linestyle='--', label=\"Mean\")\n\n# sns.distplot(sample_train_df['session_length_minutes'], color='b', bins= 70, ax= axes[1], kde= False)\n# axes[1].set_title('Length of each session')\n# axes[1].set_xlabel(\"Minutes\")\n# axes[1].set_ylabel('Density')\n# axes[1].axvline(sample_train_df['session_length_minutes'].mean(), color= 'r', linestyle='--', label=\"Mean\")\n# print()","metadata":{"id":"p27DtDIl7zdX","executionInfo":{"status":"ok","timestamp":1667854459670,"user_tz":-420,"elapsed":1000,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"bee5ab66-9f1e-4a1a-9fdc-0381cc9b1d17","execution":{"iopub.status.busy":"2022-11-15T08:01:10.929540Z","iopub.execute_input":"2022-11-15T08:01:10.930443Z","iopub.status.idle":"2022-11-15T08:01:10.941612Z","shell.execute_reply.started":"2022-11-15T08:01:10.930395Z","shell.execute_reply":"2022-11-15T08:01:10.940202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(f'{round(len(sample_train_df[sample_train_df[\"session_length_action\"]<10])/len(sample_train_df),3)*100}% of the sessions had less than 10 actions')","metadata":{"id":"UgjZz7bMbuka","executionInfo":{"status":"ok","timestamp":1667854459670,"user_tz":-420,"elapsed":7,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"8aa4e55a-0b61-4b02-f7bb-841cc96960fd","execution":{"iopub.status.busy":"2022-11-15T08:01:10.947479Z","iopub.execute_input":"2022-11-15T08:01:10.947873Z","iopub.status.idle":"2022-11-15T08:01:10.961134Z","shell.execute_reply.started":"2022-11-15T08:01:10.947843Z","shell.execute_reply":"2022-11-15T08:01:10.960018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# article_id_freq = list(overall_article_id_counts.values())\n# cut_off = [i for i in article_id_freq if i<30]\n\n# plt.figure(figsize=(8,6))\n# sns.distplot(cut_off, bins=30, kde=False);\n# plt.title(f'Article ID frequency', fontsize=12)\n# plt.ylabel('Count', fontsize=12)\n# plt.xlabel('Article', fontsize=12);","metadata":{"id":"iFMCQIZrjNa9","executionInfo":{"status":"ok","timestamp":1667854460975,"user_tz":-420,"elapsed":1309,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"43d84f45-c2ba-4924-893d-1ae0a4532e29","execution":{"iopub.status.busy":"2022-11-15T08:01:10.962596Z","iopub.execute_input":"2022-11-15T08:01:10.963257Z","iopub.status.idle":"2022-11-15T08:01:10.977740Z","shell.execute_reply.started":"2022-11-15T08:01:10.963220Z","shell.execute_reply":"2022-11-15T08:01:10.976502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ### Look at the most interacted with articles ###\n# idx_most_common_aid = np.argsort(list(overall_article_id_counts.values()))[-5:]\n# print(f'Frequency of most common articles: {np.array(list(overall_article_id_counts.values()))[idx_most_common_aid]} \\n')\n# print(f'IDs for those common articles: {idx_most_common_aid}')","metadata":{"id":"RGp_CLEZnSOW","executionInfo":{"status":"ok","timestamp":1667854460976,"user_tz":-420,"elapsed":7,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"66bb6d60-bfba-4e02-b0b6-f8b7d89aa01b","execution":{"iopub.status.busy":"2022-11-15T08:01:10.979372Z","iopub.execute_input":"2022-11-15T08:01:10.980738Z","iopub.status.idle":"2022-11-15T08:01:10.991106Z","shell.execute_reply.started":"2022-11-15T08:01:10.980683Z","shell.execute_reply":"2022-11-15T08:01:10.990187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Test & submission","metadata":{"id":"RrS2JY00pwpx"}},{"cell_type":"code","source":"with open(TEST_PATH, 'r') as f:\n    print(f\"We have {len(f.readlines()):,} lines in the test data\")","metadata":{"id":"_DlOLxPPpy7S","executionInfo":{"status":"ok","timestamp":1667854463128,"user_tz":-420,"elapsed":2156,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"2845fdea-7447-41d8-bc94-e0e2a033afa9","execution":{"iopub.status.busy":"2022-11-15T08:01:10.993071Z","iopub.execute_input":"2022-11-15T08:01:10.993974Z","iopub.status.idle":"2022-11-15T08:01:15.744858Z","shell.execute_reply.started":"2022-11-15T08:01:10.993935Z","shell.execute_reply":"2022-11-15T08:01:15.743621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load in a sample to a pandas df\n\nsample_size = 150\n\nchunks = pd.read_json(TEST_PATH, lines=True, chunksize = sample_size)\n\nfor c in chunks:\n    sample_test_df = c\n    break","metadata":{"id":"KtC2AWEGg1uK","executionInfo":{"status":"ok","timestamp":1667854463129,"user_tz":-420,"elapsed":11,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"execution":{"iopub.status.busy":"2022-11-15T08:01:15.746757Z","iopub.execute_input":"2022-11-15T08:01:15.747222Z","iopub.status.idle":"2022-11-15T08:01:15.775912Z","shell.execute_reply.started":"2022-11-15T08:01:15.747174Z","shell.execute_reply":"2022-11-15T08:01:15.774556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_test_df.head()","metadata":{"id":"3ax_qu34g5wK","executionInfo":{"status":"ok","timestamp":1667854463129,"user_tz":-420,"elapsed":10,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"c42eed82-e457-439a-b9ae-2cda5e6f0c1b","execution":{"iopub.status.busy":"2022-11-15T08:01:15.777513Z","iopub.execute_input":"2022-11-15T08:01:15.777948Z","iopub.status.idle":"2022-11-15T08:01:15.825706Z","shell.execute_reply.started":"2022-11-15T08:01:15.777902Z","shell.execute_reply":"2022-11-15T08:01:15.824476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH)\nsample_submission.head()","metadata":{"id":"mY0LJzbwg8BO","executionInfo":{"status":"ok","timestamp":1667854467732,"user_tz":-420,"elapsed":4610,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"7add9b59-358c-461c-d3d4-f9026e2c3896","execution":{"iopub.status.busy":"2022-11-15T08:01:15.827208Z","iopub.execute_input":"2022-11-15T08:01:15.827555Z","iopub.status.idle":"2022-11-15T08:01:22.219532Z","shell.execute_reply.started":"2022-11-15T08:01:15.827525Z","shell.execute_reply":"2022-11-15T08:01:22.218311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Baseline","metadata":{"id":"Rfaak3bKjZY-"}},{"cell_type":"code","source":"from collections import Counter","metadata":{"id":"UNdlu_8PmJOw","executionInfo":{"status":"ok","timestamp":1667854467732,"user_tz":-420,"elapsed":4,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"execution":{"iopub.status.busy":"2022-11-15T08:01:22.220932Z","iopub.execute_input":"2022-11-15T08:01:22.221305Z","iopub.status.idle":"2022-11-15T08:01:22.226963Z","shell.execute_reply.started":"2022-11-15T08:01:22.221274Z","shell.execute_reply":"2022-11-15T08:01:22.225721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_size = 150000\n\nchunks = pd.read_json(TRAIN_PATH, lines= True, chunksize= sample_size)\n\narticle_list = {\n    'clicks': [],\n    'carts': [],\n    'orders': []\n}\n\n\nfor e, c in enumerate(chunks):\n  if e > 2: break\n\n  sample_train_df = c\n\n  for i, row in c.iterrows():\n    actions = row['events']\n    for action in actions: \n      article_list[action['type']].append(action['aid'])","metadata":{"id":"DOFLJmY_hNhO","executionInfo":{"status":"ok","timestamp":1667854545036,"user_tz":-420,"elapsed":77308,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"execution":{"iopub.status.busy":"2022-11-15T08:01:22.228498Z","iopub.execute_input":"2022-11-15T08:01:22.228840Z","iopub.status.idle":"2022-11-15T08:02:56.372865Z","shell.execute_reply.started":"2022-11-15T08:01:22.228810Z","shell.execute_reply":"2022-11-15T08:02:56.371616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_counter = {}\nfor key in article_list.keys():\n  article_counter[key] = Counter(article_list[key])","metadata":{"id":"IDKU6TnUk2jZ","executionInfo":{"status":"ok","timestamp":1667854553340,"user_tz":-420,"elapsed":8324,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"execution":{"iopub.status.busy":"2022-11-15T08:02:56.374364Z","iopub.execute_input":"2022-11-15T08:02:56.374734Z","iopub.status.idle":"2022-11-15T08:03:04.836006Z","shell.execute_reply.started":"2022-11-15T08:02:56.374701Z","shell.execute_reply":"2022-11-15T08:03:04.834455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nb_predict = 30","metadata":{"execution":{"iopub.status.busy":"2022-11-15T08:03:04.837779Z","iopub.execute_input":"2022-11-15T08:03:04.838161Z","iopub.status.idle":"2022-11-15T08:03:04.852950Z","shell.execute_reply.started":"2022-11-15T08:03:04.838109Z","shell.execute_reply":"2022-11-15T08:03:04.851581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frequent_articles = {}\nfor key in article_counter.keys():\n  frequent_articles[key] = np.array(list(article_counter[key].keys()))[np.argsort([-1 * val for val in list(article_counter[key].values())])[:nb_predict]].tolist()","metadata":{"id":"AQL_yMkOmmA4","executionInfo":{"status":"ok","timestamp":1667854554486,"user_tz":-420,"elapsed":1164,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"execution":{"iopub.status.busy":"2022-11-15T08:03:04.854138Z","iopub.execute_input":"2022-11-15T08:03:04.854532Z","iopub.status.idle":"2022-11-15T08:03:06.159911Z","shell.execute_reply.started":"2022-11-15T08:03:04.854500Z","shell.execute_reply":"2022-11-15T08:03:06.158831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = pd.read_json(TEST_PATH, lines=True, chunksize=1000)\npreds = []\n\nfor chunk in tqdm(test_data, total=1672):\n\n    for i, row in chunk.iterrows():\n        actions = row['events']\n        article_id_list = []\n        for action in actions:\n            article_id_list.append(action['aid'])\n    \n        article_freg = Counter(article_id_list)\n\n        top_articles = np.array(list(article_freg.keys()))[np.argsort([-1 * val for val in list(article_freg.values())])[:nb_predict]].tolist()\n#     top_articles = []\n        padding_size = nb_predict - len(top_articles)\n    \n        for action in ['clicks', 'carts', 'orders']:\n            if action in ['clicks', 'carts']:\n                preds.append(\" \")\n                continue\n            top_articles = top_articles + frequent_articles[action][:padding_size]\n            preds.append(\" \".join([str(id) for id in top_articles]))","metadata":{"id":"6lTgSZQCynbU","executionInfo":{"status":"ok","timestamp":1667855021841,"user_tz":-420,"elapsed":323471,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"outputId":"0fd5876c-2c7d-474e-ff81-340aaf6884fd","execution":{"iopub.status.busy":"2022-11-15T08:22:00.721222Z","iopub.execute_input":"2022-11-15T08:22:00.721665Z","iopub.status.idle":"2022-11-15T08:26:06.411126Z","shell.execute_reply.started":"2022-11-15T08:22:00.721630Z","shell.execute_reply":"2022-11-15T08:26:06.409799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math","metadata":{"id":"JxHs0xXR1Skx","executionInfo":{"status":"ok","timestamp":1667855021842,"user_tz":-420,"elapsed":22,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"execution":{"iopub.status.busy":"2022-11-15T08:26:10.847922Z","iopub.execute_input":"2022-11-15T08:26:10.848885Z","iopub.status.idle":"2022-11-15T08:26:10.854493Z","shell.execute_reply.started":"2022-11-15T08:26:10.848839Z","shell.execute_reply":"2022-11-15T08:26:10.853111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['labels'] = preds\nsample_submission.to_csv('submission.csv', index=False)\n","metadata":{"id":"Q3s7c1OP3RSs","executionInfo":{"status":"ok","timestamp":1667855074528,"user_tz":-420,"elapsed":52110,"user":{"displayName":"Le Kien","userId":"14567157055187437308"}},"execution":{"iopub.status.busy":"2022-11-15T08:27:04.007804Z","iopub.execute_input":"2022-11-15T08:27:04.008248Z","iopub.status.idle":"2022-11-15T08:27:18.293004Z","shell.execute_reply.started":"2022-11-15T08:27:04.008214Z","shell.execute_reply":"2022-11-15T08:27:18.291809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"Gde8lqi74sei"},"execution_count":null,"outputs":[]}]}