{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# OTTO EDA","metadata":{"execution":{"iopub.status.busy":"2022-11-02T10:20:44.429559Z","iopub.execute_input":"2022-11-02T10:20:44.430421Z","iopub.status.idle":"2022-11-02T10:20:44.435877Z","shell.execute_reply.started":"2022-11-02T10:20:44.430380Z","shell.execute_reply":"2022-11-02T10:20:44.434580Z"}}},{"cell_type":"markdown","source":"## Load libraries","metadata":{}},{"cell_type":"code","source":"import random\nfrom collections import Counter\nfrom datetime import timedelta\nimport warnings\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-11-03T22:25:08.396576Z","iopub.execute_input":"2022-11-03T22:25:08.397010Z","iopub.status.idle":"2022-11-03T22:25:08.404683Z","shell.execute_reply.started":"2022-11-03T22:25:08.396979Z","shell.execute_reply":"2022-11-03T22:25:08.403030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style(\"darkgrid\")\nwarnings.simplefilter(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-11-03T21:57:09.359669Z","iopub.execute_input":"2022-11-03T21:57:09.360089Z","iopub.status.idle":"2022-11-03T21:57:09.366258Z","shell.execute_reply.started":"2022-11-03T21:57:09.360055Z","shell.execute_reply":"2022-11-03T21:57:09.364976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{}},{"cell_type":"markdown","source":"It looks like pretty big dataset, so let's load it in chunks","metadata":{}},{"cell_type":"code","source":"chunksize = 20_000\n\ntrain_data = pd.read_json(\"../input/otto-recommender-system/train.jsonl\", lines=True, chunksize=chunksize)\ntest_data = pd.read_json(\"../input/otto-recommender-system/test.jsonl\", lines=True, chunksize=chunksize)\nsample_submission = pd.read_csv(\"../input/otto-recommender-system/sample_submission.csv\", chunksize=chunksize)","metadata":{"execution":{"iopub.status.busy":"2022-11-03T21:57:12.193097Z","iopub.execute_input":"2022-11-03T21:57:12.193554Z","iopub.status.idle":"2022-11-03T21:57:12.240510Z","shell.execute_reply.started":"2022-11-03T21:57:12.193521Z","shell.execute_reply":"2022-11-03T21:57:12.239262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's examine number of sessions in train and test","metadata":{}},{"cell_type":"code","source":"with open('../input/otto-recommender-system/train.jsonl', 'r') as f:\n    print(f\"Train: {len(f.readlines()):,} lines\")\nwith open('../input/otto-recommender-system/test.jsonl', 'r') as f:\n    print(f\"Test {len(f.readlines()):,} lines\")","metadata":{"execution":{"iopub.status.busy":"2022-11-03T21:57:13.735423Z","iopub.execute_input":"2022-11-03T21:57:13.736412Z","iopub.status.idle":"2022-11-03T21:59:41.883502Z","shell.execute_reply.started":"2022-11-03T21:57:13.736360Z","shell.execute_reply":"2022-11-03T21:59:41.880856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As it expected, dataset is realy big. So let's load the first chunk of data to examine structure of the datset","metadata":{}},{"cell_type":"code","source":"train_data_chunk = train_data.__next__()\ntrain_data_chunk","metadata":{"execution":{"iopub.status.busy":"2022-11-03T21:04:39.416858Z","iopub.execute_input":"2022-11-03T21:04:39.418615Z","iopub.status.idle":"2022-11-03T21:04:43.011576Z","shell.execute_reply.started":"2022-11-03T21:04:39.418519Z","shell.execute_reply":"2022-11-03T21:04:43.010052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's look at events of some session","metadata":{}},{"cell_type":"code","source":"train_data_chunk.iloc[112][\"events\"]","metadata":{"execution":{"iopub.status.busy":"2022-11-03T21:04:45.325342Z","iopub.execute_input":"2022-11-03T21:04:45.325853Z","iopub.status.idle":"2022-11-03T21:04:45.338480Z","shell.execute_reply.started":"2022-11-03T21:04:45.325814Z","shell.execute_reply":"2022-11-03T21:04:45.337525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we see, data has the next structure:\n- session - the unique id of session\n- events - ordered events, that happened in this session. Each event consists of:\n    - aid - the article id (product code) of the associated event\n    - ts - the Unix timestamp of the event (in microseconds)\n    - type - the event type (\"clicks\", \"carts\", \"orders\"). Whether a product was clicked, added to the user's cart, or ordered during the session","metadata":{}},{"cell_type":"markdown","source":"## Extract data to work with","metadata":{}},{"cell_type":"markdown","source":"Because full dataset is big, let's get a few chunks to perform analysis on","metadata":{}},{"cell_type":"code","source":"indices = [i for i in range(100)]\nrandom.shuffle(indices)\nindices = indices[:3]\nprint(f\"Chunks chosen: {indices}\")\n\nchunks_of_train = []\nfor idx, chunk in enumerate(train_data):\n    if idx in indices:\n        chunks_of_train.append(chunk)\n    if idx > max(indices):\n        break\n\nchunks_of_train = pd.concat(chunks_of_train)\nchunks_of_train","metadata":{"execution":{"iopub.status.busy":"2022-11-03T22:04:15.554832Z","iopub.execute_input":"2022-11-03T22:04:15.555299Z","iopub.status.idle":"2022-11-03T22:06:27.463911Z","shell.execute_reply.started":"2022-11-03T22:04:15.555265Z","shell.execute_reply":"2022-11-03T22:06:27.462440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we may transform dataset to csv format","metadata":{}},{"cell_type":"code","source":"events_dict = {\n    \"session\": [],\n    \"aid\": [],\n    \"ts\": [],\n    \"type\": [],\n}\n\nfor _, row in chunks_of_train.iterrows():\n    for event in row[\"events\"]:\n        events_dict[\"session\"].append(row[\"session\"])\n        events_dict[\"aid\"].append(event[\"aid\"])\n        events_dict[\"ts\"].append(event[\"ts\"])\n        events_dict[\"type\"].append(event[\"type\"])\n\ntrain_part = pd.DataFrame(events_dict)\ntrain_part","metadata":{"execution":{"iopub.status.busy":"2022-11-03T22:06:59.099159Z","iopub.execute_input":"2022-11-03T22:06:59.100499Z","iopub.status.idle":"2022-11-03T22:07:13.421991Z","shell.execute_reply.started":"2022-11-03T22:06:59.100453Z","shell.execute_reply":"2022-11-03T22:07:13.420424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Analysis","metadata":{}},{"cell_type":"markdown","source":"First of all, let's calculate number of sessions and events in this dataset","metadata":{}},{"cell_type":"code","source":"n_sessions = train_part[\"session\"].nunique()\nn_events = train_part.shape[0]\n\nprint(f\"Number of sessions: {n_sessions}\")\nprint(f\"Number of events: {n_events}\")\nprint(f\"Mean number of events in session: {n_events/n_sessions}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-03T22:07:21.960992Z","iopub.execute_input":"2022-11-03T22:07:21.961439Z","iopub.status.idle":"2022-11-03T22:07:21.986617Z","shell.execute_reply.started":"2022-11-03T22:07:21.961401Z","shell.execute_reply":"2022-11-03T22:07:21.985010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's dig into relation of numbers of events and sessions","metadata":{}},{"cell_type":"code","source":"events_in_session = train_part.groupby(['session'])['aid'].count().sort_values(ascending=False)\n\nfig, ax = plt.subplots(ncols=2, figsize=(16, 9))\nsns.distplot(x=events_in_session.values, ax=ax[0], bins=30, kde=False)\nax[0].set_title('Article frequency', fontsize=12)\nax[0].set_ylabel('N of article', fontsize=12)\nax[0].set_xlabel('Frequency', fontsize=12)\n\nsns.distplot(x=events_in_session.values, ax=ax[1], bins=30, kde=False)\nax[1].set_title('Article frequency (limited to 500)', fontsize=12)\nax[1].set_ylabel('N of articles', fontsize=12)\nax[1].set_xlabel('Frequency', fontsize=12)\nax[1].set_ylim(0, 500)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-03T22:09:31.090252Z","iopub.execute_input":"2022-11-03T22:09:31.090807Z","iopub.status.idle":"2022-11-03T22:09:31.593815Z","shell.execute_reply.started":"2022-11-03T22:09:31.090773Z","shell.execute_reply":"2022-11-03T22:09:31.592838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Seems like most of the articles has been opened just a few times","metadata":{}},{"cell_type":"markdown","source":"Now let's compare, how often every type of event occured","metadata":{}},{"cell_type":"code","source":"events_types = train_part.groupby(['type'])['type'].count().sort_values(ascending=False)\nfig, ax = plt.subplots(figsize=(16, 9))\nsns.barplot(x=events_types.index, y=events_types.values, ax=ax)\nax.set_title('Event types frequency', fontsize=12)\nax.set_ylabel('Frequency', fontsize=12)\nax.set_xlabel('Types', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-03T23:03:26.941651Z","iopub.execute_input":"2022-11-03T23:03:26.942131Z","iopub.status.idle":"2022-11-03T23:03:27.908085Z","shell.execute_reply.started":"2022-11-03T23:03:26.942097Z","shell.execute_reply":"2022-11-03T23:03:27.906690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Clicks occured much more often, then carts and orders. But this events are more important to predict due to competition metric","metadata":{}},{"cell_type":"code","source":"def count_seconds(x):\n    max_value = int(x.max())\n    min_value = int(x.min())\n    session_time = timedelta(microseconds=max_value - min_value)\n    return session_time.total_seconds() / 60 ","metadata":{"execution":{"iopub.status.busy":"2022-11-03T22:59:56.252013Z","iopub.execute_input":"2022-11-03T22:59:56.253223Z","iopub.status.idle":"2022-11-03T22:59:56.259641Z","shell.execute_reply.started":"2022-11-03T22:59:56.253159Z","shell.execute_reply":"2022-11-03T22:59:56.258300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_counts = train_part.groupby(['session'])['ts'].apply(count_seconds)\nfig, ax = plt.subplots(figsize=(16, 9))\nsns.distplot(x=time_counts.values, ax=ax, bins=30, kde=False)\nax.set_title('Length of each session', fontsize=12)\nax.set_ylabel('Frequency', fontsize=12)\nax.set_xlabel('Minutes', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-03T23:04:02.884633Z","iopub.execute_input":"2022-11-03T23:04:02.885403Z","iopub.status.idle":"2022-11-03T23:04:10.038450Z","shell.execute_reply.started":"2022-11-03T23:04:02.885343Z","shell.execute_reply":"2022-11-03T23:04:10.036982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finaly, let's visualize number of actions, performed in each session","metadata":{}},{"cell_type":"code","source":"event_types_in_session = train_part.groupby(['session']).count()\nfig, ax = plt.subplots(figsize=(16, 9))\nsns.distplot(x=event_types_in_session.values, ax=ax, bins=50, kde=False)\nax.set_title('Number of actions', fontsize=12)\nax.set_ylabel('Frequency', fontsize=12)\nax.set_xlabel('Actions', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-03T23:09:33.612615Z","iopub.execute_input":"2022-11-03T23:09:33.613264Z","iopub.status.idle":"2022-11-03T23:09:34.366122Z","shell.execute_reply.started":"2022-11-03T23:09:33.613173Z","shell.execute_reply":"2022-11-03T23:09:34.364770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, I want to test, whether users who order something has longer sessions and more events","metadata":{}},{"cell_type":"code","source":"order_in_session = train_part.groupby(['session']).apply(lambda x: x['type'].eq(\"orders\").any())\norder_in_session","metadata":{"execution":{"iopub.status.busy":"2022-11-03T23:17:37.893134Z","iopub.execute_input":"2022-11-03T23:17:37.893630Z","iopub.status.idle":"2022-11-03T23:17:51.130729Z","shell.execute_reply.started":"2022-11-03T23:17:37.893596Z","shell.execute_reply":"2022-11-03T23:17:51.129461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_data = pd.concat([event_types_in_session[\"aid\"], time_counts, order_in_session], axis=1)\ncombined_data","metadata":{"execution":{"iopub.status.busy":"2022-11-03T23:25:49.271858Z","iopub.execute_input":"2022-11-03T23:25:49.272280Z","iopub.status.idle":"2022-11-03T23:25:49.291548Z","shell.execute_reply.started":"2022-11-03T23:25:49.272246Z","shell.execute_reply":"2022-11-03T23:25:49.290176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Mean time spend by users who order: {combined_data.loc[combined_data[0] == True]['ts'].mean()}\")\nprint(f\"Mean time spend by users who didn't order: {combined_data.loc[combined_data[0] == False]['ts'].mean()}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-03T23:27:07.542464Z","iopub.execute_input":"2022-11-03T23:27:07.543051Z","iopub.status.idle":"2022-11-03T23:27:07.558480Z","shell.execute_reply.started":"2022-11-03T23:27:07.543003Z","shell.execute_reply":"2022-11-03T23:27:07.556412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Mean time spend by users who order: {combined_data.loc[combined_data[0] == True]['aid'].mean()}\")\nprint(f\"Mean time spend by users who didn't order: {combined_data.loc[combined_data[0] == False]['aid'].mean()}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-03T23:27:20.252931Z","iopub.execute_input":"2022-11-03T23:27:20.253524Z","iopub.status.idle":"2022-11-03T23:27:20.266436Z","shell.execute_reply.started":"2022-11-03T23:27:20.253483Z","shell.execute_reply":"2022-11-03T23:27:20.264803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Seems like users who order spend more time and perform more actions","metadata":{}}]}