{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n

\n

📊 OTTO EDA Notebook 📊

\n <\\p>\n <\\div>","metadata":{}},{"cell_type":"markdown","source":"# 📝About data📝\n- **Training data structure**\n - **session** - the unique id of session\n - **events** - ordered events, that happened in this session. Each event consists of:\n - **aid**\n - the article id (product code) of the associated event\n - **ts**\n - the Unix timestamp of the event (in microseconds)\n - **type**\n - the event type (\"clicks\", \"carts\", \"orders\").\n - Whether a product was clicked, added to the user's cart, or ordered during the session\n- **Prediction**\n - **Session type** - \"{session id}_{type}\" like 12899779_clicks\n - **labels** - predicted first 20 aids corresponding to session type","metadata":{}},{"cell_type":"markdown","source":"# Import libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:08:52.055272Z","iopub.execute_input":"2022-12-04T02:08:52.056159Z","iopub.status.idle":"2022-12-04T02:08:52.592471Z","shell.execute_reply.started":"2022-12-04T02:08:52.056080Z","shell.execute_reply":"2022-12-04T02:08:52.591384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read data","metadata":{}},{"cell_type":"code","source":"data_root_path = Path('../input/otto-recommender-system')\n\nchunksize = 5000\ntrain_data = pd.read_json(data_root_path / 'train.jsonl', lines=True, chunksize=chunksize)\ntest_data = pd.read_json(data_root_path / 'test.jsonl', lines=True, chunksize=chunksize)\nsample_submission = pd.read_csv(data_root_path / 'sample_submission.csv', chunksize=chunksize)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:08:52.599157Z","iopub.execute_input":"2022-12-04T02:08:52.599476Z","iopub.status.idle":"2022-12-04T02:08:52.614969Z","shell.execute_reply.started":"2022-12-04T02:08:52.599446Z","shell.execute_reply":"2022-12-04T02:08:52.613639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.__next__()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:08:52.616739Z","iopub.execute_input":"2022-12-04T02:08:52.617099Z","iopub.status.idle":"2022-12-04T02:08:52.729235Z","shell.execute_reply.started":"2022-12-04T02:08:52.617058Z","shell.execute_reply":"2022-12-04T02:08:52.727988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.__next__()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:08:52.734088Z","iopub.execute_input":"2022-12-04T02:08:52.734744Z","iopub.status.idle":"2022-12-04T02:08:52.755260Z","shell.execute_reply.started":"2022-12-04T02:08:52.734700Z","shell.execute_reply":"2022-12-04T02:08:52.754379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NOTE: comment out to save processing time\n# with open(data_root_path / 'train.jsonl', 'r') as f:\n# nb_train_records = len(f.readlines())\n# print(f\"Train: {nb_train_records} lines\")\n# with open(data_root_path / 'test.jsonl', 'r') as f:\n# nb_test_records = len(f.readlines())\n# print(f\"Test: {nb_test_records} lines\")\n\nnb_train_records = 12899779\nnb_test_records = 1671803\nprint(f\"Train: {nb_train_records} lines\")\nprint(f\"Test: {nb_test_records} lines\")","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:08:52.756472Z","iopub.execute_input":"2022-12-04T02:08:52.757301Z","iopub.status.idle":"2022-12-04T02:08:52.763642Z","shell.execute_reply.started":"2022-12-04T02:08:52.757258Z","shell.execute_reply":"2022-12-04T02:08:52.762508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Random read","metadata":{}},{"cell_type":"code","source":"nb_chosen_chunk = 2\nindices = np.arange(50)\nsample_chunk_id = random.sample(list(indices), nb_chosen_chunk)\nchunks_of_train = []\n\n# limit times of reading to save processing time\nfor idx in range(max(sample_chunk_id)+1):\n chunk = train_data.__next__()\n if idx in sample_chunk_id:\n chunks_of_train.append(chunk)\n\ndf_train_chunk = pd.concat(chunks_of_train)\ndf_train_chunk","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:08:52.765161Z","iopub.execute_input":"2022-12-04T02:08:52.765563Z","iopub.status.idle":"2022-12-04T02:09:09.938601Z","shell.execute_reply.started":"2022-12-04T02:08:52.765529Z","shell.execute_reply":"2022-12-04T02:09:09.937099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_dict = {\n \"session\": [],\n \"aid\": [],\n \"ts\": [],\n \"type\": [],\n}\n\nfor _, row in df_train_chunk.iterrows():\n for event in row[\"events\"]:\n events_dict[\"session\"].append(row[\"session\"])\n events_dict[\"aid\"].append(event[\"aid\"])\n events_dict[\"ts\"].append(event[\"ts\"])\n events_dict[\"type\"].append(event[\"type\"])\n\ndf_train_part = pd.DataFrame(events_dict)\ndf_train_part","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:09:09.940354Z","iopub.execute_input":"2022-12-04T02:09:09.940808Z","iopub.status.idle":"2022-12-04T02:09:13.693424Z","shell.execute_reply.started":"2022-12-04T02:09:09.940772Z","shell.execute_reply":"2022-12-04T02:09:13.692124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📈Visualization📉","metadata":{}},{"cell_type":"markdown","source":"## Type counts","metadata":{}},{"cell_type":"code","source":"type_counts = df_train_part['type'].value_counts()\nlabel = list(type_counts.index)\n\nplt.figure(figsize=(6, 6))\nplt.pie(type_counts,\n labels=label, counterclock=False, startangle=90,\n autopct='%1.1f%%', pctdistance=0.7)\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:09:13.695112Z","iopub.execute_input":"2022-12-04T02:09:13.695546Z","iopub.status.idle":"2022-12-04T02:09:14.122333Z","shell.execute_reply.started":"2022-12-04T02:09:13.695510Z","shell.execute_reply":"2022-12-04T02:09:14.120990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n 📒High percentage of clicks
\n 📒clicksの割合が大きい。\n

\n
","metadata":{}},{"cell_type":"markdown","source":"## Session length","metadata":{}},{"cell_type":"code","source":"ts_max_min = df_train_part.groupby('session').agg([max, min])['ts']\nts_max_min['session_length'] = ts_max_min['max'] - ts_max_min['min']\n\nsns.histplot(\n ts_max_min,\n x=\"session_length\",\n palette=\"light:m_r\",\n edgecolor=\".3\",\n linewidth=.5,\n)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:09:14.124892Z","iopub.execute_input":"2022-12-04T02:09:14.125937Z","iopub.status.idle":"2022-12-04T02:09:16.632338Z","shell.execute_reply.started":"2022-12-04T02:09:14.125882Z","shell.execute_reply":"2022-12-04T02:09:16.629972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n 📒The degree of bin with short session time is large. Most leave the page immediately?
\n 📒session時間が短いbinの度数が大きい。ほとんどはすぐにページから離脱してしまう？\n

\n
","metadata":{}},{"cell_type":"markdown","source":"## Percentage of type per Session hour","metadata":{}},{"cell_type":"code","source":"df_counts_type = df_train_part.groupby(['session', 'type']).size()\ndf_counts_type = pd.DataFrame(df_counts_type).reset_index(level=1)\ndf_counts_type = df_counts_type.rename(columns={0: 'counts'})\ndf_counts_type = df_counts_type.pivot_table(['counts'], index='session', columns='type')\ndf_counts_type = df_counts_type['counts'].fillna(0)\n\nnew_columns = {col: f'{col}_counts' for col in df_counts_type.columns}\ndf_counts_type = df_counts_type.rename(columns=new_columns)\ndf_counts_type = pd.concat([df_counts_type, ts_max_min['session_length']], axis=1)\n\n# add type dummy\ndf_counts_type['order_dummy'] = df_counts_type['orders_counts'] > 0\ndf_counts_type['carts_dummy'] = df_counts_type['carts_counts'] > 0\ndf_counts_type['click_only'] = (df_counts_type['order_dummy'] == False) & (df_counts_type['carts_dummy'] == False)\ndf_counts_type['carts_not_order'] = (df_counts_type['order_dummy'] == False) & (df_counts_type['carts_dummy'] == True)\n\nseries = df_counts_type['order_dummy'] == True\nordered_index = series[series].index\n\nseries = df_counts_type['click_only'] == True\nclick_only_index = series[series].index\n\nseries = df_counts_type['carts_not_order'] == True\ncarts_not_order_index = series[series].index\n\ndf_counts_type['event'] = ''\ndf_counts_type['event'][ordered_index] = 'ordered'\ndf_counts_type['event'][click_only_index] = 'click_only'\ndf_counts_type['event'][carts_not_order_index] = 'carts_not_order'\ndf_counts_type","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:09:16.633763Z","iopub.execute_input":"2022-12-04T02:09:16.634108Z","iopub.status.idle":"2022-12-04T02:09:16.835632Z","shell.execute_reply.started":"2022-12-04T02:09:16.634070Z","shell.execute_reply":"2022-12-04T02:09:16.834283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_theme(style=\"whitegrid\")\nfig, ax = plt.subplots(figsize=(10, 5))\nsns.despine(bottom=True, left=True)\n\nsns.stripplot(\n data=df_counts_type, x=\"session_length\", y=\"event\",\n dodge=True, alpha=.25, zorder=1\n)\n\nsns.pointplot(\n data=df_counts_type, x=\"session_length\", y=\"event\",\n join=False, dodge=.8 - .8 / 3, palette=\"dark\",\n markers=\"d\", scale=.75, errorbar=None\n)\nplt.show()\n\nfig, ax = plt.subplots(figsize=(10, 5))\nsns.boxplot(data=df_counts_type, x=\"session_length\", y=\"event\",\n whis=[0, 100], width=.6, palette=\"vlag\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:09:16.839607Z","iopub.execute_input":"2022-12-04T02:09:16.839990Z","iopub.status.idle":"2022-12-04T02:09:17.671364Z","shell.execute_reply.started":"2022-12-04T02:09:16.839956Z","shell.execute_reply":"2022-12-04T02:09:17.669764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n 📒Click-only sessions include both short and long sessions.
\n Sessions with carts w/o order and with order tend to be long.\n
\n 📒clickのみのsessionは短時間も長時間も含む。
\n orderがないcarts、orderがあるsessionは時間が長い傾向にある。\n

\n
","metadata":{}},{"cell_type":"code","source":"session_time = {\n '0~10min': (0, 600*1e6),\n '10~20min': (600*1e6, 1200*1e6),\n '20~30min': (1200*1e6, 1800*1e6),\n '30~40min': (1800*1e6, 2400*1e6),\n '40~50min': (2400*1e6, 3000*1e6),\n '50~60min': (3000*1e6, 3600*1e6),\n '60~min': (3600*1e6, -1),\n }\n\ndf_session_times = []\n\nfor name, session_range in session_time.items():\n session_min, session_max = session_range\n \n if session_max == -1:\n df = df_counts_type[\n df_counts_type['session_length'] > session_min]\n else:\n df = df_counts_type[\n (df_counts_type['session_length'] > session_min) & \\\n (df_counts_type['session_length'] < session_max)]\n \n df = pd.DataFrame(df[['carts_counts', 'clicks_counts', 'orders_counts']].sum())\n df = df.rename(columns={0: name})\n df_session_times.append(df)\n \ndf_session_times = pd.concat(df_session_times, axis=1)\ndf_session_times","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:09:17.673147Z","iopub.execute_input":"2022-12-04T02:09:17.673571Z","iopub.status.idle":"2022-12-04T02:09:17.717943Z","shell.execute_reply.started":"2022-12-04T02:09:17.673538Z","shell.execute_reply":"2022-12-04T02:09:17.716586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_session_times = df_session_times / df_session_times.sum()\ndf_session_times = df_session_times.fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:09:17.723541Z","iopub.execute_input":"2022-12-04T02:09:17.723974Z","iopub.status.idle":"2022-12-04T02:09:17.734831Z","shell.execute_reply.started":"2022-12-04T02:09:17.723940Z","shell.execute_reply":"2022-12-04T02:09:17.733143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10, 6))\n\nfor i in range(len(df_session_times)):\n ax.bar(df_session_times.columns,\n df_session_times.iloc[i],\n bottom=df_session_times.iloc[:i].sum(),\n width=.5,\n alpha=.7)\n\nax.legend(df_session_times.index.tolist(), loc='lower right', borderaxespad=2)\nplt.grid(linestyle='dotted', linewidth=1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:09:17.744369Z","iopub.execute_input":"2022-12-04T02:09:17.744964Z","iopub.status.idle":"2022-12-04T02:09:18.052490Z","shell.execute_reply.started":"2022-12-04T02:09:17.744909Z","shell.execute_reply":"2022-12-04T02:09:18.051280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature of sessions including \"carts\" or \"orders\"","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(10, 5), sharey='all')\n\naxes[0].set_title('carts: False')\nsns.histplot(\n df_counts_type[df_counts_type['carts_dummy'] == False],\n x=\"session_length\",\n palette=\"light:m_r\",\n edgecolor=\".3\",\n linewidth=.5,\n ax=axes[0],\n bins=20\n)\n\naxes[1].set_title('carts: True')\nsns.histplot(\n df_counts_type[df_counts_type['carts_dummy'] == True],\n x=\"session_length\",\n palette=\"light:m_r\",\n edgecolor=\".3\",\n linewidth=.5,\n ax=axes[1],\n bins=20\n)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:09:18.056424Z","iopub.execute_input":"2022-12-04T02:09:18.056723Z","iopub.status.idle":"2022-12-04T02:09:18.582614Z","shell.execute_reply.started":"2022-12-04T02:09:18.056693Z","shell.execute_reply":"2022-12-04T02:09:18.581365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(10, 5), sharey='all')\n\naxes[0].set_title('order: False')\nsns.histplot(\n df_counts_type[df_counts_type['order_dummy'] == False],\n x=\"session_length\",\n palette=\"light:m_r\",\n edgecolor=\".3\",\n linewidth=.5,\n ax=axes[0],\n bins=20\n)\n\naxes[1].set_title('order: True')\nsns.histplot(\n df_counts_type[df_counts_type['order_dummy'] == True],\n x=\"session_length\",\n palette=\"light:m_r\",\n edgecolor=\".3\",\n linewidth=.5,\n ax=axes[1],\n bins=20\n)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:09:18.584692Z","iopub.execute_input":"2022-12-04T02:09:18.585540Z","iopub.status.idle":"2022-12-04T02:09:19.363784Z","shell.execute_reply.started":"2022-12-04T02:09:18.585491Z","shell.execute_reply":"2022-12-04T02:09:19.362149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n 📒the session in which carts or order was made has fewer bin with shorter session time.
\n 📒carts, orderが行われたsessionはsession時間が短いbinが少ない。\n

\n
","metadata":{}},{"cell_type":"markdown","source":"## Top N items","metadata":{}},{"cell_type":"code","source":"df_aid = pd.DataFrame(df_train_part.groupby(['aid']).size()).sort_values(by=0, ascending=False)\ndf_aid = df_aid.rename(columns={0: 'counts'})\nprint(f'unique aid: {len(df_aid)}')\n\nthres_counts = 100\ndf_thres_counts = df_aid[df_aid[\"counts\"] > thres_counts]\nprint(f'aid counts > {thres_counts}: {len(df_thres_counts)}')","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:09:19.365552Z","iopub.execute_input":"2022-12-04T02:09:19.366028Z","iopub.status.idle":"2022-12-04T02:09:19.456697Z","shell.execute_reply.started":"2022-12-04T02:09:19.365992Z","shell.execute_reply":"2022-12-04T02:09:19.454617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df_thres_counts.reset_index()\nfig = plt.figure(figsize=(20, 6))\nsns.barplot(x='aid', y='counts', data=df, order=df.sort_values('counts', ascending=False).aid)\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:09:19.458843Z","iopub.execute_input":"2022-12-04T02:09:19.459244Z","iopub.status.idle":"2022-12-04T02:09:22.485627Z","shell.execute_reply.started":"2022-12-04T02:09:19.459204Z","shell.execute_reply":"2022-12-04T02:09:22.483939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Features between items","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(20,20))\naid_cross = pd.crosstab(df_thres_counts['counts'], df_thres_counts['counts'])\ncmap = sns.diverging_palette(230, 20, as_cmap=True)\nmask = np.triu(np.ones_like(aid_cross, dtype=bool))\nsns.heatmap(aid_cross, mask=mask, cmap=cmap, vmax=.3, center=0,\n square=True, linewidths=.5, cbar_kws={\"shrink\": .5})\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:09:22.487754Z","iopub.execute_input":"2022-12-04T02:09:22.488218Z","iopub.status.idle":"2022-12-04T02:09:24.868124Z","shell.execute_reply.started":"2022-12-04T02:09:22.488181Z","shell.execute_reply":"2022-12-04T02:09:24.866293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n 📒Is there any tendency that when an action is taken on one item, another item is also acted on as a set?
\n 📒あるitemに対してactionしたら別のitemもセットでactionが起こされるという傾向は見られない？\n

\n
","metadata":{}},{"cell_type":"markdown","source":"## Features of items per session","metadata":{}},{"cell_type":"code","source":"df_type_aid = df_train_part.groupby(['session', 'type', 'aid']).size()\ndf_type_aid = pd.DataFrame(df_type_aid).reset_index(level=[1, 2])\ndf_type_aid = df_type_aid.rename(columns={0: 'counts'})\ndf_type_aid.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:09:24.869914Z","iopub.execute_input":"2022-12-04T02:09:24.870344Z","iopub.status.idle":"2022-12-04T02:09:25.201948Z","shell.execute_reply.started":"2022-12-04T02:09:24.870305Z","shell.execute_reply":"2022-12-04T02:09:25.200341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_session_id = df_type_aid[df_type_aid['type']=='orders'].index[0] # select session including 'order'\n\nfig, axes = plt.subplots(3, 1, figsize=(20, 15))\nfig.suptitle(f'session: {target_session_id} type counts per item')\ndf = df_type_aid[df_type_aid.index == target_session_id]\n\nfor i, type_name in enumerate(['clicks', 'carts', 'orders']):\n sns.barplot(x=\"aid\", y=\"counts\", data=df[df['type'] == type_name], ax=axes[i])\n axes[i].tick_params(labelrotation=45)\n axes[i].set_title(type_name)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:12:21.554334Z","iopub.execute_input":"2022-12-04T02:12:21.554801Z","iopub.status.idle":"2022-12-04T02:12:22.221638Z","shell.execute_reply.started":"2022-12-04T02:12:21.554765Z","shell.execute_reply":"2022-12-04T02:12:22.220193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# References\n- https://www.kaggle.com/code/mohdmuttalib/otto-eda","metadata":{}}]}