{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":38760,"databundleVersionId":4493939,"sourceType":"competition"}],"dockerImageVersionId":30301,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# OTTO EDA\n","metadata":{}},{"cell_type":"markdown","source":"## Load libraries","metadata":{}},{"cell_type":"code","source":"import random\nfrom collections import Counter\nfrom datetime import timedelta\nimport warnings\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:15:05.271730Z","iopub.execute_input":"2024-06-03T06:15:05.272452Z","iopub.status.idle":"2024-06-03T06:15:05.280291Z","shell.execute_reply.started":"2024-06-03T06:15:05.272400Z","shell.execute_reply":"2024-06-03T06:15:05.279332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style(\"darkgrid\")\nwarnings.simplefilter(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:15:05.282129Z","iopub.execute_input":"2024-06-03T06:15:05.282886Z","iopub.status.idle":"2024-06-03T06:15:05.291704Z","shell.execute_reply.started":"2024-06-03T06:15:05.282832Z","shell.execute_reply":"2024-06-03T06:15:05.290217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{}},{"cell_type":"markdown","source":"It looks like pretty big dataset, so let's load it in chunks","metadata":{}},{"cell_type":"code","source":"chunksize = 20_000\n\ntrain_data = pd.read_json(\"../input/otto-recommender-system/train.jsonl\", lines=True, chunksize=chunksize)\ntest_data = pd.read_json(\"../input/otto-recommender-system/test.jsonl\", lines=True, chunksize=chunksize)\nsample_submission = pd.read_csv(\"../input/otto-recommender-system/sample_submission.csv\", chunksize=chunksize)","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:15:05.293608Z","iopub.execute_input":"2024-06-03T06:15:05.294010Z","iopub.status.idle":"2024-06-03T06:15:05.354429Z","shell.execute_reply.started":"2024-06-03T06:15:05.293977Z","shell.execute_reply":"2024-06-03T06:15:05.353343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's examine number of sessions in train and test","metadata":{}},{"cell_type":"code","source":"with open('../input/otto-recommender-system/train.jsonl', 'r') as f:\n    print(f\"Train: {len(f.readlines()):,} lines\")\nwith open('../input/otto-recommender-system/test.jsonl', 'r') as f:\n    print(f\"Test {len(f.readlines()):,} lines\")","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:15:05.355830Z","iopub.execute_input":"2024-06-03T06:15:05.357054Z","iopub.status.idle":"2024-06-03T06:16:55.705953Z","shell.execute_reply.started":"2024-06-03T06:15:05.357009Z","shell.execute_reply":"2024-06-03T06:16:55.704558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As it expected, dataset is realy big. So let's load the first chunk of data to examine structure of the datset","metadata":{}},{"cell_type":"code","source":"train_data_chunk = train_data.__next__()\ntrain_data_chunk","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:16:55.710324Z","iopub.execute_input":"2024-06-03T06:16:55.711056Z","iopub.status.idle":"2024-06-03T06:16:57.509203Z","shell.execute_reply.started":"2024-06-03T06:16:55.711017Z","shell.execute_reply":"2024-06-03T06:16:57.507975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's look at events of some session","metadata":{}},{"cell_type":"code","source":"train_data_chunk.iloc[112][\"events\"]","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:16:57.510606Z","iopub.execute_input":"2024-06-03T06:16:57.511011Z","iopub.status.idle":"2024-06-03T06:16:57.524075Z","shell.execute_reply.started":"2024-06-03T06:16:57.510978Z","shell.execute_reply":"2024-06-03T06:16:57.522908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we see, data has the next structure:\n- session - the unique id of session\n- events - ordered events, that happened in this session. Each event consists of:\n    - aid - the article id (product code) of the associated event\n    - ts - the Unix timestamp of the event (in microseconds)\n    - type - the event type (\"clicks\", \"carts\", \"orders\"). Whether a product was clicked, added to the user's cart, or ordered during the session","metadata":{}},{"cell_type":"markdown","source":"## Extract data to work with","metadata":{}},{"cell_type":"markdown","source":"Because full dataset is big, let's get a few chunks to perform analysis on","metadata":{}},{"cell_type":"code","source":"indices = [i for i in range(100)]\nrandom.shuffle(indices)\nindices = indices[:3]\nprint(f\"Chunks chosen: {indices}\")\n\nchunks_of_train = []\nfor idx, chunk in enumerate(train_data):\n    if idx in indices:\n        chunks_of_train.append(chunk)\n    if idx > max(indices):\n        break\n\nchunks_of_train = pd.concat(chunks_of_train)\nchunks_of_train","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:16:57.526188Z","iopub.execute_input":"2024-06-03T06:16:57.526705Z","iopub.status.idle":"2024-06-03T06:19:28.266956Z","shell.execute_reply.started":"2024-06-03T06:16:57.526610Z","shell.execute_reply":"2024-06-03T06:19:28.265570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we may transform dataset to csv format","metadata":{}},{"cell_type":"code","source":"events_dict = {\n    \"session\": [],\n    \"aid\": [],\n    \"ts\": [],\n    \"type\": [],\n}\n\nfor _, row in chunks_of_train.iterrows():\n    for event in row[\"events\"]:\n        events_dict[\"session\"].append(row[\"session\"])\n        events_dict[\"aid\"].append(event[\"aid\"])\n        events_dict[\"ts\"].append(event[\"ts\"])\n        events_dict[\"type\"].append(event[\"type\"])\n\ntrain_part = pd.DataFrame(events_dict)\ntrain_part","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:19:28.268499Z","iopub.execute_input":"2024-06-03T06:19:28.268891Z","iopub.status.idle":"2024-06-03T06:19:52.093216Z","shell.execute_reply.started":"2024-06-03T06:19:28.268857Z","shell.execute_reply":"2024-06-03T06:19:52.092002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Analysis","metadata":{}},{"cell_type":"markdown","source":"First of all, let's calculate number of sessions and events in this dataset","metadata":{}},{"cell_type":"code","source":"n_sessions = train_part[\"session\"].nunique()\nn_events = train_part.shape[0]\n\nprint(f\"Number of sessions: {n_sessions}\")\nprint(f\"Number of events: {n_events}\")\nprint(f\"Mean number of events in session: {n_events/n_sessions}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:19:52.094902Z","iopub.execute_input":"2024-06-03T06:19:52.095340Z","iopub.status.idle":"2024-06-03T06:19:52.121295Z","shell.execute_reply.started":"2024-06-03T06:19:52.095305Z","shell.execute_reply":"2024-06-03T06:19:52.119539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's dig into relation of numbers of events and sessions","metadata":{}},{"cell_type":"code","source":"events_in_session = train_part.groupby(['session'])['aid'].count().sort_values(ascending=False)\n\nfig, ax = plt.subplots(ncols=2, figsize=(16, 9))\nsns.distplot(x=events_in_session.values, ax=ax[0], bins=30, kde=False)\nax[0].set_title('Article frequency', fontsize=12)\nax[0].set_ylabel('N of article', fontsize=12)\nax[0].set_xlabel('Frequency', fontsize=12)\n\nsns.distplot(x=events_in_session.values, ax=ax[1], bins=30, kde=False)\nax[1].set_title('Article frequency (limited to 500)', fontsize=12)\nax[1].set_ylabel('N of articles', fontsize=12)\nax[1].set_xlabel('Frequency', fontsize=12)\nax[1].set_ylim(0, 500)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:19:52.123157Z","iopub.execute_input":"2024-06-03T06:19:52.123556Z","iopub.status.idle":"2024-06-03T06:19:53.337021Z","shell.execute_reply.started":"2024-06-03T06:19:52.123522Z","shell.execute_reply":"2024-06-03T06:19:53.335773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Seems like most of the articles has been opened just a few times","metadata":{}},{"cell_type":"markdown","source":"Now let's compare, how often every type of event occured","metadata":{}},{"cell_type":"code","source":"events_types = train_part.groupby(['type'])['type'].count().sort_values(ascending=False)\nfig, ax = plt.subplots(figsize=(16, 9))\nsns.barplot(x=events_types.index, y=events_types.values, ax=ax)\nax.set_title('Event types frequency', fontsize=12)\nax.set_ylabel('Frequency', fontsize=12)\nax.set_xlabel('Types', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:19:53.338713Z","iopub.execute_input":"2024-06-03T06:19:53.339095Z","iopub.status.idle":"2024-06-03T06:19:54.530023Z","shell.execute_reply.started":"2024-06-03T06:19:53.339062Z","shell.execute_reply":"2024-06-03T06:19:54.528732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Clicks occured much more often, then carts and orders. But this events are more important to predict due to competition metric","metadata":{}},{"cell_type":"code","source":"def count_seconds(x):\n    max_value = int(x.max())\n    min_value = int(x.min())\n    session_time = timedelta(microseconds=max_value - min_value)\n    return session_time.total_seconds() / 60 ","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:19:54.531406Z","iopub.execute_input":"2024-06-03T06:19:54.531755Z","iopub.status.idle":"2024-06-03T06:19:54.538540Z","shell.execute_reply.started":"2024-06-03T06:19:54.531725Z","shell.execute_reply":"2024-06-03T06:19:54.537201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_counts = train_part.groupby(['session'])['ts'].apply(count_seconds)\nfig, ax = plt.subplots(figsize=(16, 9))\nsns.distplot(x=time_counts.values, ax=ax, bins=30, kde=False)\nax.set_title('Length of each session', fontsize=12)\nax.set_ylabel('Frequency', fontsize=12)\nax.set_xlabel('Minutes', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:19:54.540122Z","iopub.execute_input":"2024-06-03T06:19:54.540478Z","iopub.status.idle":"2024-06-03T06:20:02.989571Z","shell.execute_reply.started":"2024-06-03T06:19:54.540448Z","shell.execute_reply":"2024-06-03T06:20:02.988348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finaly, let's visualize number of actions, performed in each session","metadata":{}},{"cell_type":"code","source":"event_types_in_session = train_part.groupby(['session']).count()\nfig, ax = plt.subplots(figsize=(16, 9))\nsns.distplot(x=event_types_in_session.values, ax=ax, bins=50, kde=False)\nax.set_title('Number of actions', fontsize=12)\nax.set_ylabel('Frequency', fontsize=12)\nax.set_xlabel('Actions', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:20:02.993640Z","iopub.execute_input":"2024-06-03T06:20:02.994075Z","iopub.status.idle":"2024-06-03T06:20:03.989412Z","shell.execute_reply.started":"2024-06-03T06:20:02.994039Z","shell.execute_reply":"2024-06-03T06:20:03.988165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, I want to test, whether users who order something has longer sessions and more events","metadata":{}},{"cell_type":"code","source":"order_in_session = train_part.groupby(['session']).apply(lambda x: x['type'].eq(\"orders\").any())\norder_in_session","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:20:03.990962Z","iopub.execute_input":"2024-06-03T06:20:03.991319Z","iopub.status.idle":"2024-06-03T06:20:18.739841Z","shell.execute_reply.started":"2024-06-03T06:20:03.991287Z","shell.execute_reply":"2024-06-03T06:20:18.738668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_data = pd.concat([event_types_in_session[\"aid\"], time_counts, order_in_session], axis=1)\ncombined_data","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:20:18.741888Z","iopub.execute_input":"2024-06-03T06:20:18.742390Z","iopub.status.idle":"2024-06-03T06:20:18.763393Z","shell.execute_reply.started":"2024-06-03T06:20:18.742345Z","shell.execute_reply":"2024-06-03T06:20:18.762220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Mean time spend by users who order: {combined_data.loc[combined_data[0] == True]['ts'].mean()}\")\nprint(f\"Mean time spend by users who didn't order: {combined_data.loc[combined_data[0] == False]['ts'].mean()}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:20:18.764979Z","iopub.execute_input":"2024-06-03T06:20:18.766154Z","iopub.status.idle":"2024-06-03T06:20:18.778820Z","shell.execute_reply.started":"2024-06-03T06:20:18.766106Z","shell.execute_reply":"2024-06-03T06:20:18.777366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Mean time spend by users who order: {combined_data.loc[combined_data[0] == True]['aid'].mean()}\")\nprint(f\"Mean time spend by users who didn't order: {combined_data.loc[combined_data[0] == False]['aid'].mean()}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-03T06:20:18.780231Z","iopub.execute_input":"2024-06-03T06:20:18.780823Z","iopub.status.idle":"2024-06-03T06:20:18.791217Z","shell.execute_reply.started":"2024-06-03T06:20:18.780771Z","shell.execute_reply":"2024-06-03T06:20:18.789874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Seems like users who order spend more time and perform more actions","metadata":{}}]}