{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":38760,"databundleVersionId":4493939,"sourceType":"competition"}],"dockerImageVersionId":30775,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import random\nfrom collections import Counter\nfrom datetime import timedelta\nimport warnings\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-27T08:43:29.862438Z","iopub.execute_input":"2024-09-27T08:43:29.862853Z","iopub.status.idle":"2024-09-27T08:43:32.331500Z","shell.execute_reply.started":"2024-09-27T08:43:29.862814Z","shell.execute_reply":"2024-09-27T08:43:32.330371Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.set_style(\"darkgrid\")\nwarnings.simplefilter(\"ignore\")\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:43:32.333263Z","iopub.execute_input":"2024-09-27T08:43:32.333758Z","iopub.status.idle":"2024-09-27T08:43:32.340167Z","shell.execute_reply.started":"2024-09-27T08:43:32.333719Z","shell.execute_reply":"2024-09-27T08:43:32.338344Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"chunksize = 20_000\n\ntrain_data = pd.read_json(\"../input/otto-recommender-system/train.jsonl\", lines=True, chunksize=chunksize)\ntest_data = pd.read_json(\"../input/otto-recommender-system/test.jsonl\", lines=True, chunksize=chunksize)\nsample_submission = pd.read_csv(\"../input/otto-recommender-system/sample_submission.csv\", chunksize=chunksize)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:43:32.341634Z","iopub.execute_input":"2024-09-27T08:43:32.342077Z","iopub.status.idle":"2024-09-27T08:43:32.371512Z","shell.execute_reply.started":"2024-09-27T08:43:32.342013Z","shell.execute_reply":"2024-09-27T08:43:32.370427Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open('../input/otto-recommender-system/train.jsonl', 'r') as f:\n    print(f\"Train: {len(f.readlines()):,} lines\")\nwith open('../input/otto-recommender-system/test.jsonl', 'r') as f:\n    print(f\"Test {len(f.readlines()):,} lines\")","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:43:32.374364Z","iopub.execute_input":"2024-09-27T08:43:32.374721Z","iopub.status.idle":"2024-09-27T08:46:02.133221Z","shell.execute_reply.started":"2024-09-27T08:43:32.374684Z","shell.execute_reply":"2024-09-27T08:46:02.131840Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data_chunk = train_data.__next__()\ntrain_data_chunk\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:46:02.135264Z","iopub.execute_input":"2024-09-27T08:46:02.135666Z","iopub.status.idle":"2024-09-27T08:46:04.457849Z","shell.execute_reply.started":"2024-09-27T08:46:02.135620Z","shell.execute_reply":"2024-09-27T08:46:04.456829Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data_chunk.iloc[0][\"events\"]","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-09-27T08:46:04.459154Z","iopub.execute_input":"2024-09-27T08:46:04.459470Z","iopub.status.idle":"2024-09-27T08:46:04.494985Z","shell.execute_reply.started":"2024-09-27T08:46:04.459437Z","shell.execute_reply":"2024-09-27T08:46:04.493722Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"indices = [i for i in range(100)]\nrandom.shuffle(indices)\nindices = indices[:3]\nprint(f\"Chunks chosen: {indices}\")\n\nchunks_of_train = []\nfor idx, chunk in enumerate(train_data):\n    if idx in indices:\n        chunks_of_train.append(chunk)\n    if idx > max(indices):\n        break\n\nchunks_of_train = pd.concat(chunks_of_train)\nchunks_of_train","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:46:04.496775Z","iopub.execute_input":"2024-09-27T08:46:04.497182Z","iopub.status.idle":"2024-09-27T08:48:11.113253Z","shell.execute_reply.started":"2024-09-27T08:46:04.497134Z","shell.execute_reply":"2024-09-27T08:48:11.111956Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#split attribute\nevents_dict = {\n    \"session\": [],\n    \"aid\": [],\n    \"ts\": [],\n    \"type\": [],\n}\n\nfor _, row in chunks_of_train.iterrows():\n    for event in row[\"events\"]:\n        events_dict[\"session\"].append(row[\"session\"])\n        events_dict[\"aid\"].append(event[\"aid\"])\n        events_dict[\"ts\"].append(event[\"ts\"])\n        events_dict[\"type\"].append(event[\"type\"])\n\ntrain_part = pd.DataFrame(events_dict)\ntrain_part","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:48:11.114726Z","iopub.execute_input":"2024-09-27T08:48:11.115110Z","iopub.status.idle":"2024-09-27T08:48:26.074997Z","shell.execute_reply.started":"2024-09-27T08:48:11.115066Z","shell.execute_reply":"2024-09-27T08:48:26.074003Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#visual freq of each type: click, add to cart, orders\nevents_types = train_part.groupby(['type'])['type'].count().sort_values(ascending=False)\nfig, ax = plt.subplots(figsize=(16, 9))\nsns.barplot(x=events_types.index, y=events_types.values, ax=ax)\nax.set_title('Event types frequency', fontsize=12)\nax.set_ylabel('Frequency', fontsize=12)\nax.set_xlabel('Types', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:48:26.076442Z","iopub.execute_input":"2024-09-27T08:48:26.076789Z","iopub.status.idle":"2024-09-27T08:48:27.318628Z","shell.execute_reply.started":"2024-09-27T08:48:26.076753Z","shell.execute_reply":"2024-09-27T08:48:27.317524Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def count_seconds(x):\n    max_value = int(x.max())\n    min_value = int(x.min())\n    session_time = timedelta(microseconds=max_value - min_value)\n    return session_time.total_seconds() / 60 ","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:48:27.321741Z","iopub.execute_input":"2024-09-27T08:48:27.322129Z","iopub.status.idle":"2024-09-27T08:48:27.327608Z","shell.execute_reply.started":"2024-09-27T08:48:27.322089Z","shell.execute_reply":"2024-09-27T08:48:27.326519Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# visual time of each session \ntime_counts = train_part.groupby(['session'])['ts'].apply(count_seconds)\nfig, ax = plt.subplots(figsize=(16, 9))\nsns.distplot(x=time_counts.values, ax=ax, bins=30, kde=False)\nax.set_title('Length of each session', fontsize=12)\nax.set_ylabel('Frequency', fontsize=12)\nax.set_xlabel('Minutes', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:48:27.329240Z","iopub.execute_input":"2024-09-27T08:48:27.330237Z","iopub.status.idle":"2024-09-27T08:48:32.227657Z","shell.execute_reply.started":"2024-09-27T08:48:27.330184Z","shell.execute_reply":"2024-09-27T08:48:32.226485Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# visual number of each actions in each session\nevent_types_in_session = train_part.groupby(['session']).count()\nfig, ax = plt.subplots(figsize=(16, 9))\nsns.distplot(x=event_types_in_session.values, ax=ax, bins=50, kde=False)\nax.set_title('Number of actions', fontsize=12)\nax.set_ylabel('Frequency', fontsize=12)\nax.set_xlabel('Actions', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:48:32.229140Z","iopub.execute_input":"2024-09-27T08:48:32.229485Z","iopub.status.idle":"2024-09-27T08:48:32.964200Z","shell.execute_reply.started":"2024-09-27T08:48:32.229447Z","shell.execute_reply":"2024-09-27T08:48:32.963063Z"},"trusted":true},"outputs":[],"execution_count":null}]}