{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import random\nfrom collections import Counter\nfrom datetime import timedelta\nimport warnings\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-24T08:53:16.271560Z","iopub.execute_input":"2022-11-24T08:53:16.272031Z","iopub.status.idle":"2022-11-24T08:53:16.279257Z","shell.execute_reply.started":"2022-11-24T08:53:16.271991Z","shell.execute_reply":"2022-11-24T08:53:16.277807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style(\"darkgrid\")\nwarnings.simplefilter(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-11-24T08:53:20.836511Z","iopub.execute_input":"2022-11-24T08:53:20.836959Z","iopub.status.idle":"2022-11-24T08:53:20.843276Z","shell.execute_reply.started":"2022-11-24T08:53:20.836926Z","shell.execute_reply":"2022-11-24T08:53:20.841739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load Data**","metadata":{}},{"cell_type":"markdown","source":"Look like the big dataset, load in chunks","metadata":{}},{"cell_type":"code","source":"chunksize = 20_000\n\ntrain_data = pd.read_json(\"../input/otto-recommender-system/train.jsonl\", lines=True, chunksize=chunksize)\ntest_data = pd.read_json(\"../input/otto-recommender-system/test.jsonl\", lines=True, chunksize=chunksize)\nsample_submission = pd.read_csv(\"../input/otto-recommender-system/sample_submission.csv\", chunksize=chunksize)","metadata":{"execution":{"iopub.status.busy":"2022-11-24T08:53:27.776813Z","iopub.execute_input":"2022-11-24T08:53:27.777320Z","iopub.status.idle":"2022-11-24T08:53:27.802149Z","shell.execute_reply.started":"2022-11-24T08:53:27.777282Z","shell.execute_reply":"2022-11-24T08:53:27.800944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train and Test Data Examine","metadata":{}},{"cell_type":"code","source":"with open('../input/otto-recommender-system/train.jsonl', 'r') as f:\n    print(f\"Train: {len(f.readlines()):,} lines\")\nwith open('../input/otto-recommender-system/test.jsonl', 'r') as f:\n    print(f\"Test {len(f.readlines()):,} lines\")","metadata":{"execution":{"iopub.status.busy":"2022-11-24T08:53:40.662586Z","iopub.execute_input":"2022-11-24T08:53:40.663006Z","iopub.status.idle":"2022-11-24T08:56:04.890780Z","shell.execute_reply.started":"2022-11-24T08:53:40.662971Z","shell.execute_reply":"2022-11-24T08:56:04.889522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_chunk = train_data.__next__()\ntrain_data_chunk","metadata":{"execution":{"iopub.status.busy":"2022-11-24T08:57:46.969105Z","iopub.execute_input":"2022-11-24T08:57:46.969656Z","iopub.status.idle":"2022-11-24T08:57:48.664169Z","shell.execute_reply.started":"2022-11-24T08:57:46.969616Z","shell.execute_reply":"2022-11-24T08:57:48.662857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_chunk.iloc[112][\"events\"]","metadata":{"execution":{"iopub.status.busy":"2022-11-24T08:57:51.499202Z","iopub.execute_input":"2022-11-24T08:57:51.499641Z","iopub.status.idle":"2022-11-24T08:57:51.512657Z","shell.execute_reply.started":"2022-11-24T08:57:51.499605Z","shell.execute_reply":"2022-11-24T08:57:51.511016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Extract the data that we are working for","metadata":{}},{"cell_type":"code","source":"indices = [i for i in range(100)]\nrandom.shuffle(indices)\nindices = indices[:3]\nprint(f\"Chunks chosen: {indices}\")\n\nchunks_of_train = []\nfor idx, chunk in enumerate(train_data):\n    if idx in indices:\n        chunks_of_train.append(chunk)\n    if idx > max(indices):\n        break\n\nchunks_of_train = pd.concat(chunks_of_train)\nchunks_of_train","metadata":{"execution":{"iopub.status.busy":"2022-11-24T08:57:55.064342Z","iopub.execute_input":"2022-11-24T08:57:55.064787Z","iopub.status.idle":"2022-11-24T08:58:46.419544Z","shell.execute_reply.started":"2022-11-24T08:57:55.064754Z","shell.execute_reply":"2022-11-24T08:58:46.418344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_dict = {\n    \"session\": [],\n    \"aid\": [],\n    \"ts\": [],\n    \"type\": [],\n}\n\nfor _, row in chunks_of_train.iterrows():\n    for event in row[\"events\"]:\n        events_dict[\"session\"].append(row[\"session\"])\n        events_dict[\"aid\"].append(event[\"aid\"])\n        events_dict[\"ts\"].append(event[\"ts\"])\n        events_dict[\"type\"].append(event[\"type\"])\n\ntrain_part = pd.DataFrame(events_dict)\ntrain_part","metadata":{"execution":{"iopub.status.busy":"2022-11-24T08:58:50.633418Z","iopub.execute_input":"2022-11-24T08:58:50.633915Z","iopub.status.idle":"2022-11-24T08:59:08.087564Z","shell.execute_reply.started":"2022-11-24T08:58:50.633877Z","shell.execute_reply":"2022-11-24T08:59:08.086388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Calculate the number of sessions and events ","metadata":{}},{"cell_type":"code","source":"n_sessions = train_part[\"session\"].nunique()\nn_events = train_part.shape[0]\n\nprint(f\"Number of sessions: {n_sessions}\")\nprint(f\"Number of events: {n_events}\")\nprint(f\"Mean number of events in session: {n_events/n_sessions}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-24T08:59:13.264890Z","iopub.execute_input":"2022-11-24T08:59:13.265316Z","iopub.status.idle":"2022-11-24T08:59:13.293530Z","shell.execute_reply.started":"2022-11-24T08:59:13.265279Z","shell.execute_reply":"2022-11-24T08:59:13.292311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Relation of the number of sessions and events \n","metadata":{}},{"cell_type":"code","source":"events_in_session = train_part.groupby(['session'])['aid'].count().sort_values(ascending=False)\n\nfig, ax = plt.subplots(ncols=2, figsize=(16, 9))\nsns.distplot(x=events_in_session.values, ax=ax[0], bins=30, kde=False)\nax[0].set_title('Article frequency', fontsize=12)\nax[0].set_ylabel('N of article', fontsize=12)\nax[0].set_xlabel('Frequency', fontsize=12)\n\nsns.distplot(x=events_in_session.values, ax=ax[1], bins=30, kde=False)\nax[1].set_title('Article frequency (limited to 500)', fontsize=12)\nax[1].set_ylabel('N of articles', fontsize=12)\nax[1].set_xlabel('Frequency', fontsize=12)\nax[1].set_ylim(0, 500)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-24T08:59:18.159300Z","iopub.execute_input":"2022-11-24T08:59:18.159758Z","iopub.status.idle":"2022-11-24T08:59:19.402890Z","shell.execute_reply.started":"2022-11-24T08:59:18.159724Z","shell.execute_reply":"2022-11-24T08:59:19.401645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, compare the every type of events  ","metadata":{}},{"cell_type":"code","source":"events_types = train_part.groupby(['type'])['type'].count().sort_values(ascending=False)\nfig, ax = plt.subplots(figsize=(16, 9))\nsns.barplot(x=events_types.index, y=events_types.values, ax=ax)\nax.set_title('Event types frequency', fontsize=12)\nax.set_ylabel('Frequency', fontsize=12)\nax.set_xlabel('Types', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-24T08:59:51.061549Z","iopub.execute_input":"2022-11-24T08:59:51.063879Z","iopub.status.idle":"2022-11-24T08:59:52.454294Z","shell.execute_reply.started":"2022-11-24T08:59:51.063822Z","shell.execute_reply":"2022-11-24T08:59:52.453075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Clicks occured much more often, then carts and orders. But this events are more important to predict due to competition metric","metadata":{}},{"cell_type":"code","source":"def count_seconds(x):\n    max_value = int(x.max())\n    min_value = int(x.min())\n    session_time = timedelta(microseconds=max_value - min_value)\n    return session_time.total_seconds() / 60 ","metadata":{"execution":{"iopub.status.busy":"2022-11-24T08:59:55.381557Z","iopub.execute_input":"2022-11-24T08:59:55.382034Z","iopub.status.idle":"2022-11-24T08:59:55.389043Z","shell.execute_reply.started":"2022-11-24T08:59:55.381999Z","shell.execute_reply":"2022-11-24T08:59:55.388023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_counts = train_part.groupby(['session'])['ts'].apply(count_seconds)\nfig, ax = plt.subplots(figsize=(16, 9))\nsns.distplot(x=time_counts.values, ax=ax, bins=30, kde=False)\nax.set_title('Length of each session', fontsize=12)\nax.set_ylabel('Frequency', fontsize=12)\nax.set_xlabel('Minutes', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-24T08:59:56.498308Z","iopub.execute_input":"2022-11-24T08:59:56.499884Z","iopub.status.idle":"2022-11-24T09:00:03.595142Z","shell.execute_reply.started":"2022-11-24T08:59:56.499823Z","shell.execute_reply":"2022-11-24T09:00:03.594310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualize the actions perform in each session","metadata":{}},{"cell_type":"code","source":"event_types_in_session = train_part.groupby(['session']).count()\nfig, ax = plt.subplots(figsize=(16, 9))\nsns.distplot(x=event_types_in_session.values, ax=ax, bins=50, kde=False)\nax.set_title('Number of actions', fontsize=12)\nax.set_ylabel('Frequency', fontsize=12)\nax.set_xlabel('Actions', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-24T09:00:09.379670Z","iopub.execute_input":"2022-11-24T09:00:09.380746Z","iopub.status.idle":"2022-11-24T09:00:10.254812Z","shell.execute_reply.started":"2022-11-24T09:00:09.380707Z","shell.execute_reply":"2022-11-24T09:00:10.253629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"order_in_session = train_part.groupby(['session']).apply(lambda x: x['type'].eq(\"orders\").any())\norder_in_session","metadata":{"execution":{"iopub.status.busy":"2022-11-24T09:00:11.953844Z","iopub.execute_input":"2022-11-24T09:00:11.955511Z","iopub.status.idle":"2022-11-24T09:00:24.915924Z","shell.execute_reply.started":"2022-11-24T09:00:11.955433Z","shell.execute_reply":"2022-11-24T09:00:24.914744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_data = pd.concat([event_types_in_session[\"aid\"], time_counts, order_in_session], axis=1)\ncombined_data","metadata":{"execution":{"iopub.status.busy":"2022-11-24T09:00:40.838917Z","iopub.execute_input":"2022-11-24T09:00:40.839345Z","iopub.status.idle":"2022-11-24T09:00:40.863288Z","shell.execute_reply.started":"2022-11-24T09:00:40.839313Z","shell.execute_reply":"2022-11-24T09:00:40.862151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Mean time spend by users who order: {combined_data.loc[combined_data[0] == True]['ts'].mean()}\")\nprint(f\"Mean time spend by users who didn't order: {combined_data.loc[combined_data[0] == False]['ts'].mean()}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-24T09:00:42.121826Z","iopub.execute_input":"2022-11-24T09:00:42.122290Z","iopub.status.idle":"2022-11-24T09:00:42.137236Z","shell.execute_reply.started":"2022-11-24T09:00:42.122245Z","shell.execute_reply":"2022-11-24T09:00:42.135520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Mean time spend by users who order: {combined_data.loc[combined_data[0] == True]['aid'].mean()}\")\nprint(f\"Mean time spend by users who didn't order: {combined_data.loc[combined_data[0] == False]['aid'].mean()}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-24T09:00:43.470262Z","iopub.execute_input":"2022-11-24T09:00:43.470740Z","iopub.status.idle":"2022-11-24T09:00:43.482399Z","shell.execute_reply.started":"2022-11-24T09:00:43.470698Z","shell.execute_reply":"2022-11-24T09:00:43.481141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Seems like users who order spend more time and perform more actions","metadata":{}}]}