{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":38760,"databundleVersionId":4493939,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\n\ndata_root_path = Path('../input/otto-recommender-system')\n\nchunksize = 5000\ntrain_data = pd.read_json(data_root_path / 'train.jsonl', lines=True, chunksize=chunksize)\ntest_data = pd.read_json(data_root_path / 'test.jsonl', lines=True, chunksize=chunksize)\nsample_submission = pd.read_csv(data_root_path / 'sample_submission.csv', chunksize=chunksize)\n\ntest_data_iter = iter(test_data)\nnext(test_data_iter)\n\nsample_submission_iter = iter(sample_submission)\nnext(sample_submission_iter)\n\nnb_train_records = 12899779\nnb_test_records = 1671803\nprint(f\"Train: {nb_train_records} lines\")\nprint(f\"Test: {nb_test_records} lines\")\n\nnb_chosen_chunk = 2\nindices = np.arange(50)\nsample_chunk_id = random.sample(list(indices), nb_chosen_chunk)\nchunks_of_train = []\n\nfor idx in range(max(sample_chunk_id) + 1):\n    chunk = train_data.__next__()\n    if idx in sample_chunk_id:\n        chunks_of_train.append(chunk)\n\ndf_train_chunk = pd.concat(chunks_of_train)\n\nevents_dict = {\n    \"session\": [],\n    \"aid\": [],\n    \"ts\": [],\n    \"type\": [],\n}\n\nfor _, row in df_train_chunk.iterrows():\n    for event in row[\"events\"]:\n        events_dict[\"session\"].append(row[\"session\"])\n        events_dict[\"aid\"].append(event[\"aid\"])\n        events_dict[\"ts\"].append(event[\"ts\"])\n        events_dict[\"type\"].append(event[\"type\"])\n\ndf_train_part = pd.DataFrame(events_dict)\n\ntype_counts = df_train_part['type'].value_counts()\nlabel = list(type_counts.index)\n\nplt.figure(figsize=(6, 6))\nplt.pie(type_counts,\n        labels=label, counterclock=False, startangle=90,\n        autopct='%1.1f%%', pctdistance=0.7)\nplt.tight_layout()\nplt.show()\n\nts_max_min = df_train_part.groupby('session').agg(['max', 'min'])['ts']\nts_max_min['session_length'] = ts_max_min['max'] - ts_max_min['min']\n\nsns.histplot(\n    data=ts_max_min,\n    x=\"session_length\",\n    palette=\"light:m_r\",\n    edgecolor=\".3\",\n    linewidth=.5,\n)\nplt.show()\n\ndf_counts_type = df_train_part.groupby(['session', 'type']).size().reset_index(name='counts')\ndf_counts_type = df_counts_type.pivot(index='session', columns='type', values='counts').fillna(0)\n\n\nnew_columns = {col: f'{col}_counts' for col in df_counts_type.columns}\ndf_counts_type = df_counts_type.rename(columns=new_columns)\ndf_counts_type = pd.concat([df_counts_type, ts_max_min['session_length']], axis=1)\n\ndf_counts_type['order_dummy'] = df_counts_type['orders_counts'] > 0\ndf_counts_type['carts_dummy'] = df_counts_type['carts_counts'] > 0\ndf_counts_type['click_only'] = ~df_counts_type['order_dummy'] & ~df_counts_type['carts_dummy']\ndf_counts_type['carts_not_order'] = ~df_counts_type['order_dummy'] & df_counts_type['carts_dummy']\n\nseries = df_counts_type['order_dummy'] == True\nordered_index = series[series].index\n\nseries = df_counts_type['click_only'] == True\nclick_only_index = series[series].index\n\nseries = df_counts_type['carts_not_order'] == True\ncarts_not_order_index = series[series].index\n\nconditions = [\n    df_counts_type.index.isin(ordered_index),\n    df_counts_type.index.isin(click_only_index),\n    df_counts_type.index.isin(carts_not_order_index)\n]\n\nchoices = ['ordered', 'click_only', 'carts_not_order']\n\ndf_counts_type['event'] = np.select(conditions, choices, default='')\n\nsns.set_theme(style=\"whitegrid\")\nfig, ax = plt.subplots(figsize=(10, 5))\nsns.despine(bottom=True, left=True)\n\nsns.stripplot(\n    data=df_counts_type, x=\"session_length\", y=\"event\",\n    dodge=True, alpha=.25, zorder=1\n)\n\nsns.pointplot(\n    data=df_counts_type, x=\"session_length\", y=\"event\",\n    join=False, dodge=.8 - .8 / 3, palette=\"dark\",\n    markers=\"d\", scale=.75, errorbar=None\n)\nplt.show()\n\nfig, ax = plt.subplots(figsize=(10, 5))\nsns.boxplot(data=df_counts_type, x=\"session_length\", y=\"event\",\n            whis=[0, 100], width=.6, palette=\"vlag\")\nplt.show()\n\nsession_time = {\n    '0~10min': (0, 600 * 1e6),\n    '10~20min': (600 * 1e6, 1200 * 1e6),\n    '20~30min': (1200 * 1e6, 1800 * 1e6),\n    '30~40min': (1800 * 1e6, 2400 * 1e6),\n    '40~50min': (2400 * 1e6, 3000 * 1e6),\n    '50~60min': (3000 * 1e6, 3600 * 1e6),\n    '60~min': (3600 * 1e6, -1),\n}\n\ndf_session_times = []\nfor name, session_range in session_time.items():\n    session_min, session_max = session_range\n\n    if session_max == -1:\n        df = df_counts_type[\n            df_counts_type['session_length'] > session_min]\n    else:\n        df = df_counts_type[\n            (df_counts_type['session_length'] > session_min) & \\\n            (df_counts_type['session_length'] < session_max)]\n\n    df = pd.DataFrame(df[['carts_counts', 'clicks_counts', 'orders_counts']].sum())\n    df = df.rename(columns={0: name})\n    df_session_times.append(df)\n\ndf_session_times = pd.concat(df_session_times, axis=1)\ndf_session_times\ndf_session_times = df_session_times / df_session_times.sum()\ndf_session_times = df_session_times.fillna(0)\nfig, ax = plt.subplots(figsize=(10, 6))\n\nfor i in range(len(df_session_times)):\n    ax.bar(df_session_times.columns,\n           df_session_times.iloc[i],\n           bottom=df_session_times.iloc[:i].sum(),\n           width=.5,\n           alpha=.7)\n\nax.legend(df_session_times.index.tolist(), loc='lower right', borderaxespad=2)\nplt.grid(linestyle='dotted', linewidth=1)\nplt.show()\n\nfig, axes = plt.subplots(2, 2, figsize=(10, 10), sharey='all')\n\naxes[0, 0].set_title('carts: False')\nsns.histplot(\n    df_counts_type[df_counts_type['carts_dummy'] == False],\n    x=\"session_length\",\n    palette=\"light:m_r\",\n    edgecolor=\".3\",\n    linewidth=.5,\n    ax=axes[0, 0],\n    bins=20\n)\n\naxes[0, 1].set_title('carts: True')\nsns.histplot(\n    df_counts_type[df_counts_type['carts_dummy'] == True],\n    x=\"session_length\",\n    palette=\"light:m_r\",\n    edgecolor=\".3\",\n    linewidth=.5,\n    ax=axes[0, 1],\n    bins=20\n)\n\naxes[1, 0].set_title('order: False')\nsns.histplot(\n    df_counts_type[df_counts_type['order_dummy'] == False],\n    x=\"session_length\",\n    palette=\"light:m_r\",\n    edgecolor=\".3\",\n    linewidth=.5,\n    ax=axes[1, 0],\n    bins=20\n)\n\naxes[1, 1].set_title('order: True')\nsns.histplot(\n    df_counts_type[df_counts_type['order_dummy'] == True],\n    x=\"session_length\",\n    palette=\"light:m_r\",\n    edgecolor=\".3\",\n    linewidth=.5,\n    ax=axes[1, 1],\n    bins=20\n)\n\nplt.tight_layout()\nplt.show()\n\ndf_aid = df_train_part.groupby(['aid']).size().reset_index(name='counts').sort_values(by='counts', ascending=False)\nprint(f'unique aid: {len(df_aid)}')\n\n\nthres_counts = 100\ndf_thres_counts = df_aid[df_aid[\"counts\"] > thres_counts]\nprint(f'aid counts > {thres_counts}: {len(df_thres_counts)}')\ndf = df_thres_counts.reset_index()\nfig = plt.figure(figsize=(20, 6))\nsns.barplot(x='aid', y='counts', data=df, order=df.sort_values('counts', ascending=False).aid)\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()\n\n\nfig = plt.figure(figsize=(20,20))\naid_cross = pd.crosstab(df_thres_counts['counts'], df_thres_counts['counts'])\ncmap = sns.diverging_palette(230, 20, as_cmap=True)\nmask = np.triu(np.ones_like(aid_cross, dtype=bool))\nsns.heatmap(aid_cross, mask=mask, cmap=cmap, vmax=.3, center=0,\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5})\nplt.show()\ndf_type_aid = df_train_part.groupby(['session', 'type', 'aid']).size()\ndf_type_aid = pd.DataFrame(df_type_aid).reset_index(level=[1, 2])\ndf_type_aid = df_type_aid.rename(columns={0: 'counts'})\ndf_type_aid.head(5)\ntarget_session_id = df_type_aid[df_type_aid['type']=='orders'].index[0] # select session including 'order'\n\nfig, axes = plt.subplots(3, 1, figsize=(20, 15))\nfig.suptitle(f'session: {target_session_id}   type counts per item')\ndf = df_type_aid[df_type_aid.index == target_session_id]\n\nfor i, type_name in enumerate(['clicks', 'carts', 'orders']):\n    sns.barplot(x=\"aid\", y=\"counts\", data=df[df['type'] == type_name], ax=axes[i])\n    axes[i].tick_params(labelrotation=45)\n    axes[i].set_title(type_name)\n\nplt.tight_layout()\nplt.show()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-30T12:01:17.673654Z","iopub.execute_input":"2023-12-30T12:01:17.674145Z","iopub.status.idle":"2023-12-30T12:01:56.209879Z","shell.execute_reply.started":"2023-12-30T12:01:17.674108Z","shell.execute_reply":"2023-12-30T12:01:56.208903Z"},"trusted":true},"execution_count":null,"outputs":[]}]}