{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":38760,"databundleVersionId":4493939,"sourceType":"competition"}],"dockerImageVersionId":30626,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-30T16:06:04.010045Z","iopub.execute_input":"2023-12-30T16:06:04.010429Z","iopub.status.idle":"2023-12-30T16:06:04.381887Z","shell.execute_reply.started":"2023-12-30T16:06:04.010396Z","shell.execute_reply":"2023-12-30T16:06:04.380860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom pathlib import Path\nimport os\nimport random\nimport numpy as np\nimport json\nfrom datetime import timedelta\nfrom collections import Counter\nfrom tqdm.notebook import tqdm\nfrom heapq import nlargest\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_theme()\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport math","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:06:04.384051Z","iopub.execute_input":"2023-12-30T16:06:04.384758Z","iopub.status.idle":"2023-12-30T16:06:05.796139Z","shell.execute_reply.started":"2023-12-30T16:06:04.384723Z","shell.execute_reply":"2023-12-30T16:06:05.795170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Data_Path = Path('../input/otto-recommender-system')\nTrain_Path = Data_Path/'train.jsonl'\nTest_Path = Data_Path/'test.jsonl'\nSample_sub_Path = Path('../input/otto-recommender-system/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:06:05.797407Z","iopub.execute_input":"2023-12-30T16:06:05.797695Z","iopub.status.idle":"2023-12-30T16:06:05.802892Z","shell.execute_reply.started":"2023-12-30T16:06:05.797669Z","shell.execute_reply":"2023-12-30T16:06:05.801778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(Train_Path, 'r') as f:\n    print(f\"We have {len(f.readlines()):,} lines in the training data\")","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:06:05.806586Z","iopub.execute_input":"2023-12-30T16:06:05.807100Z","iopub.status.idle":"2023-12-30T16:08:12.070386Z","shell.execute_reply.started":"2023-12-30T16:06:05.807069Z","shell.execute_reply":"2023-12-30T16:08:12.069185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_size = 100000\n\nchunks = pd.read_json(Train_Path, lines=True, chunksize = sample_size)\n\nfor c in chunks:\n    sample_train_df = c\n    break","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:08:12.071978Z","iopub.execute_input":"2023-12-30T16:08:12.072564Z","iopub.status.idle":"2023-12-30T16:08:19.357042Z","shell.execute_reply.started":"2023-12-30T16:08:12.072537Z","shell.execute_reply":"2023-12-30T16:08:19.355491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_train_df.set_index('session', drop=True, inplace=True)\nsample_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:08:19.358452Z","iopub.execute_input":"2023-12-30T16:08:19.358773Z","iopub.status.idle":"2023-12-30T16:08:19.452435Z","shell.execute_reply.started":"2023-12-30T16:08:19.358743Z","shell.execute_reply":"2023-12-30T16:08:19.450886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 提取函数：计算会话时长\ndef calculate_session_duration(session):\n    start_time = session[0]['ts']\n    end_time = session[-1]['ts']\n    duration = end_time - start_time\n    return duration\n\n# 提取函数：计算会话中每种操作的频率\ndef calculate_action_counts(session):\n    action_counts = {}\n    for action in session:\n        action_type = action['type']\n        action_counts[action_type] = action_counts.get(action_type, 0) + 1\n    return action_counts\n\n# 示例会话的统计分析\nexample_session = sample_train_df.iloc[0].item()\nsession_length = len(example_session)\nfirst_action = example_session[0]\nsession_duration = calculate_session_duration(example_session)\naction_counts = calculate_action_counts(example_session)\n\n# 输出统计结果\nprint(f'This session contains {session_length} actions')\nprint(f'The first action in the session: {first_action}')\nprint(f'The first session lasted for {str(timedelta(microseconds=session_duration))}')\nprint('The first session contains the following frequency of actions:')\nfor action, count in action_counts.items():\n    print(f'{action}: {count}')","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:08:19.454063Z","iopub.execute_input":"2023-12-30T16:08:19.454406Z","iopub.status.idle":"2023-12-30T16:08:19.461629Z","shell.execute_reply.started":"2023-12-30T16:08:19.454368Z","shell.execute_reply":"2023-12-30T16:08:19.460775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"action_counts_list, article_id_counts_list, session_length_time_list, session_length_action_list = ([] for i in range(4))\noverall_action_counts = {}\noverall_article_id_counts = {}\n\nfor i, row in tqdm(sample_train_df.iterrows(), total=len(sample_train_df)):\n    \n    actions = row['events']\n    \n    action_counts = {}\n    article_id_counts = {}\n    for action in actions:\n        action_counts[action['type']] = action_counts.get(action['type'], 0) + 1\n        article_id_counts[action['aid']] = article_id_counts.get(action['aid'], 0) + 1\n        overall_action_counts[action['type']] = overall_action_counts.get(action['type'], 0) + 1\n        overall_article_id_counts[action['aid']] = overall_article_id_counts.get(action['aid'], 0) + 1\n        \n    session_length_time = actions[-1]['ts'] - actions[0]['ts']\n    \n    action_counts_list.append(action_counts)\n    article_id_counts_list.append(article_id_counts)\n    session_length_time_list.append(session_length_time)\n    session_length_action_list.append(len(actions))\n    \nsample_train_df['action_counts'] = action_counts_list\nsample_train_df['article_id_counts'] = article_id_counts_list\nsample_train_df['session_length_unix'] = session_length_time_list\nsample_train_df['session_length_minutes'] = sample_train_df['session_length_unix']*1.66667e-8 \nsample_train_df['session_length_action'] = session_length_action_list","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:08:19.463065Z","iopub.execute_input":"2023-12-30T16:08:19.463350Z","iopub.status.idle":"2023-12-30T16:08:40.981319Z","shell.execute_reply.started":"2023-12-30T16:08:19.463325Z","shell.execute_reply":"2023-12-30T16:08:40.979872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_actions = sum(overall_action_counts.values())\n\nplt.figure(figsize=(8,6))\nsns.barplot(x=list(overall_action_counts.keys()), y=[i/total_actions for i in overall_action_counts.values()]);\nplt.title(f'Action frequency', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xlabel('Category', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:08:40.982996Z","iopub.execute_input":"2023-12-30T16:08:40.983347Z","iopub.status.idle":"2023-12-30T16:08:41.237161Z","shell.execute_reply.started":"2023-12-30T16:08:40.983315Z","shell.execute_reply":"2023-12-30T16:08:41.235590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 设置图形尺寸和布局\nfig, ax = plt.subplots(1, 2, figsize=(24, 10))\n\n# 绘制每个会话中操作数量的分布图\nsns.histplot(data=sample_train_df, x='session_length_action', color='y', bins=70, kde=False, ax=ax[0])\nax[0].set_xlabel(\"Number of actions\", fontsize=16)\nax[0].set_ylabel(\"Density\", fontsize=16)\nax[0].set_title(\"Distribution of the number of actions taken in each session\", fontsize=14)\nax[0].axvline(sample_train_df['session_length_action'].mean(), color='r', linestyle='--', label=\"Mean\")\nax[0].legend()\n\n# 绘制每个会话的时长分布图\nsns.histplot(data=sample_train_df, x='session_length_minutes', color='b', bins=70, kde=False, ax=ax[1])\nax[1].set_xlabel(\"Minutes\", fontsize=16)\nax[1].set_ylabel(\"Density\", fontsize=16)\nax[1].set_title(\"Length of each session\", fontsize=16)\n\n# 显示图形\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:08:41.241324Z","iopub.execute_input":"2023-12-30T16:08:41.241715Z","iopub.status.idle":"2023-12-30T16:08:42.167417Z","shell.execute_reply.started":"2023-12-30T16:08:41.241684Z","shell.execute_reply":"2023-12-30T16:08:42.166624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'{round(len(sample_train_df[sample_train_df[\"session_length_action\"]<10])/len(sample_train_df),3)*100}% of the sessions had less than 10 actions')","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:08:42.168324Z","iopub.execute_input":"2023-12-30T16:08:42.168623Z","iopub.status.idle":"2023-12-30T16:08:42.195005Z","shell.execute_reply.started":"2023-12-30T16:08:42.168596Z","shell.execute_reply":"2023-12-30T16:08:42.193820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_id_freq = list(overall_article_id_counts.values())\ncut_off = [i for i in article_id_freq if i<30]\n\nplt.figure(figsize=(8,6))\nsns.distplot(cut_off, bins=30, kde=False);\nplt.title(f'Article ID frequency', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xlabel('Article', fontsize=12);","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:08:42.196585Z","iopub.execute_input":"2023-12-30T16:08:42.196966Z","iopub.status.idle":"2023-12-30T16:08:42.640341Z","shell.execute_reply.started":"2023-12-30T16:08:42.196936Z","shell.execute_reply":"2023-12-30T16:08:42.638963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Frequency of most common articles: {sorted(list(overall_article_id_counts.values()))[-5:]} \\n')\nres = nlargest(5, overall_article_id_counts, key = overall_article_id_counts.get)\nprint(f'IDs for those common articles: {res}')","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:08:42.641729Z","iopub.execute_input":"2023-12-30T16:08:42.642039Z","iopub.status.idle":"2023-12-30T16:08:42.948245Z","shell.execute_reply.started":"2023-12-30T16:08:42.642015Z","shell.execute_reply":"2023-12-30T16:08:42.947332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(Test_Path, 'r') as f:\n    print(f\"We have {len(f.readlines()):,} lines in the test data\")","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:08:42.949569Z","iopub.execute_input":"2023-12-30T16:08:42.949972Z","iopub.status.idle":"2023-12-30T16:08:49.366847Z","shell.execute_reply.started":"2023-12-30T16:08:42.949940Z","shell.execute_reply":"2023-12-30T16:08:49.365381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_size = 150\n\nchunks = pd.read_json(Test_Path, lines=True, chunksize = sample_size)\n\nfor c in chunks:\n    sample_test_df = c\n    break","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:08:49.368417Z","iopub.execute_input":"2023-12-30T16:08:49.368724Z","iopub.status.idle":"2023-12-30T16:08:49.381563Z","shell.execute_reply.started":"2023-12-30T16:08:49.368696Z","shell.execute_reply":"2023-12-30T16:08:49.379936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:08:49.383433Z","iopub.execute_input":"2023-12-30T16:08:49.384527Z","iopub.status.idle":"2023-12-30T16:08:49.420149Z","shell.execute_reply.started":"2023-12-30T16:08:49.384490Z","shell.execute_reply":"2023-12-30T16:08:49.419194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv(Sample_sub_Path)\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:08:49.422886Z","iopub.execute_input":"2023-12-30T16:08:49.423252Z","iopub.status.idle":"2023-12-30T16:08:55.236533Z","shell.execute_reply.started":"2023-12-30T16:08:49.423227Z","shell.execute_reply":"2023-12-30T16:08:55.234841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_size = 100000\n\nchunks = pd.read_json(Train_Path, lines=True, chunksize = sample_size)\n\nclicks_article_list = []\ncarts_article_list = []\norders_article_list = []\n\nfor e, c in enumerate(chunks):\n    \n    if e > 2:\n        break\n    \n    sample_train_df = c\n    \n    for i, row in c.iterrows():\n        actions = row['events']\n        for action in actions:\n            if action['type'] == 'clicks':\n                clicks_article_list.append(action['aid'])\n            elif action['type'] == 'carts':\n                carts_article_list.append(action['aid'])\n            else:\n                orders_article_list.append(action['aid'])","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:08:55.240102Z","iopub.execute_input":"2023-12-30T16:08:55.240435Z","iopub.status.idle":"2023-12-30T16:10:15.545967Z","shell.execute_reply.started":"2023-12-30T16:08:55.240408Z","shell.execute_reply":"2023-12-30T16:10:15.544746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_click_freq = Counter(clicks_article_list)\narticle_carts_freq = Counter(carts_article_list)\narticle_order_freq = Counter(orders_article_list)","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:10:15.547978Z","iopub.execute_input":"2023-12-30T16:10:15.548305Z","iopub.status.idle":"2023-12-30T16:10:24.690922Z","shell.execute_reply.started":"2023-12-30T16:10:15.548276Z","shell.execute_reply":"2023-12-30T16:10:24.689762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_click_article = nlargest(20, article_click_freq, key = article_click_freq.get)\ntop_carts_article = nlargest(20, article_carts_freq, key = article_carts_freq.get)\ntop_order_article = nlargest(20, article_order_freq, key = article_order_freq.get) ","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:10:24.692336Z","iopub.execute_input":"2023-12-30T16:10:24.692668Z","iopub.status.idle":"2023-12-30T16:10:25.338514Z","shell.execute_reply.started":"2023-12-30T16:10:24.692637Z","shell.execute_reply":"2023-12-30T16:10:25.336867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frequent_articles = {'clicks': top_click_article, 'carts':top_carts_article, 'order':top_order_article}","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:10:25.339940Z","iopub.execute_input":"2023-12-30T16:10:25.340333Z","iopub.status.idle":"2023-12-30T16:10:25.345991Z","shell.execute_reply.started":"2023-12-30T16:10:25.340299Z","shell.execute_reply":"2023-12-30T16:10:25.344774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for action in ['clicks', 'carts', 'order']:\n    print(f'Most frequent articles for {action}: {frequent_articles[action][-5:]}')","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:10:25.347334Z","iopub.execute_input":"2023-12-30T16:10:25.347664Z","iopub.status.idle":"2023-12-30T16:10:25.358433Z","shell.execute_reply.started":"2023-12-30T16:10:25.347635Z","shell.execute_reply":"2023-12-30T16:10:25.357367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = pd.read_json(Test_Path, lines=True, chunksize=1000)\n\npreds = []\n\nfor chunk in tqdm(test_data, total=1671):\n    \n    for i, row in chunk.iterrows():\n        actions = row['events']\n        article_id_list = []\n        for action in actions:\n            article_id_list.append(action['aid'])\n            \n        article_freq = Counter(article_id_list)\n        top_articles = nlargest(20, article_freq, key = article_freq.get)\n        \n        padding_size = -(20 - len(top_articles))\n        for action in ['clicks', 'carts', 'order']:\n            top_articles = top_articles + frequent_articles[action][padding_size:]\n            preds.append(\" \".join([str(id) for id in top_articles]))","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:10:25.359665Z","iopub.execute_input":"2023-12-30T16:10:25.360194Z","iopub.status.idle":"2023-12-30T16:14:17.815913Z","shell.execute_reply.started":"2023-12-30T16:10:25.360167Z","shell.execute_reply":"2023-12-30T16:14:17.814721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['labels'] = preds","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:14:17.817533Z","iopub.execute_input":"2023-12-30T16:14:17.820744Z","iopub.status.idle":"2023-12-30T16:14:18.486452Z","shell.execute_reply.started":"2023-12-30T16:14:17.820677Z","shell.execute_reply":"2023-12-30T16:14:18.485736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-30T16:14:18.487846Z","iopub.execute_input":"2023-12-30T16:14:18.488371Z","iopub.status.idle":"2023-12-30T16:14:43.978146Z","shell.execute_reply.started":"2023-12-30T16:14:18.488345Z","shell.execute_reply":"2023-12-30T16:14:43.976763Z"},"trusted":true},"execution_count":null,"outputs":[]}]}