{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport datetime","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Based on https://www.kaggle.com/code/konradb/dataset-as-df","metadata":{}},{"cell_type":"code","source":"train_sessions = pd.DataFrame()\nchunks = pd.read_json('/kaggle/input/otto-recommender-system/train.jsonl', lines=True, chunksize=100_000)\n\n\nfor e, chunk in tqdm(enumerate(chunks)):\n    event_dict = {\n        'session': [],\n        'aid': [],\n        'ts': [],\n        'type': [],\n    }\n    if e < 2:\n        # train_sessions = pd.concat([train_sessions, chunk])\n        for session, events in zip(chunk['session'].tolist(), chunk['events'].tolist()):\n            for event in events:\n                event_dict['session'].append(session)\n                event_dict['aid'].append(event['aid'])\n                event_dict['ts'].append(event['ts'])\n                event_dict['type'].append(event['type'])\n        chunk_session = pd.DataFrame(event_dict)\n        train_sessions = pd.concat([train_sessions, chunk_session])\n    else:\n        break\n        \ndf_train = train_sessions.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-03T10:07:27.269398Z","iopub.execute_input":"2022-11-03T10:07:27.269829Z","iopub.status.idle":"2022-11-03T10:08:32.204735Z","shell.execute_reply.started":"2022-11-03T10:07:27.269796Z","shell.execute_reply":"2022-11-03T10:08:32.203380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Only 1 month from 22/07/31~22/08/28","metadata":{}},{"cell_type":"code","source":"ser_ts = pd.to_datetime(df_train['ts'], unit='ms', utc=True)\nser_ts.describe()","metadata":{"execution":{"iopub.status.busy":"2022-11-03T10:19:21.269657Z","iopub.execute_input":"2022-11-03T10:19:21.271056Z","iopub.status.idle":"2022-11-03T10:19:25.086429Z","shell.execute_reply.started":"2022-11-03T10:19:21.271002Z","shell.execute_reply":"2022-11-03T10:19:25.085011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import defaultdict\n\ndf_ts = df_train[['ts']]\ndf_ts['UTC'] = ser_ts\ndf_ts['string'] = ser_ts.astype(str)\n\ndict_ts = defaultdict(list)\nidx = []\n_df = df_ts['string']#.iloc[:10]\nfor data in tqdm(_df):\n  d = data.split(\" \")\n  d_0 = d[0].split(\"-\")\n  d_1 = d[1].split(\":\")\n  dict_ts['year'].append(d_0[0])\n  dict_ts['month'].append(d_0[1])\n  dict_ts['date'].append(d_0[2])\n  dict_ts['hour'].append(d_1[0])\n  dict_ts['minute'].append(d_1[1])\n  dict_ts['second'].append(d_1[2].split(\".\")[0])\n  idx.append(idx)\n\ndf_ts = pd.concat([df_ts,pd.DataFrame(dict_ts,index=_df.index)],axis=1)\ndf_ts","metadata":{"execution":{"iopub.status.busy":"2022-11-03T10:19:29.724824Z","iopub.execute_input":"2022-11-03T10:19:29.725265Z","iopub.status.idle":"2022-11-03T10:21:09.809956Z","shell.execute_reply.started":"2022-11-03T10:19:29.725225Z","shell.execute_reply":"2022-11-03T10:21:09.808583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Maybe SALE on Augst 1st?","metadata":{}},{"cell_type":"code","source":"ser = df_ts['month'] + df_ts['date']\nser.value_counts().sort_index().plot.bar()","metadata":{"execution":{"iopub.status.busy":"2022-11-03T10:21:09.812316Z","iopub.execute_input":"2022-11-03T10:21:09.813265Z","iopub.status.idle":"2022-11-03T10:21:14.768030Z","shell.execute_reply.started":"2022-11-03T10:21:09.813214Z","shell.execute_reply":"2022-11-03T10:21:14.767036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Especially from 4AM to 6AM","metadata":{}},{"cell_type":"code","source":"ser = df_ts.query('( month == \"08\") & (date == \"01\")')\nser.value_counts('hour').sort_index().plot.bar()","metadata":{"execution":{"iopub.status.busy":"2022-11-03T10:21:14.769200Z","iopub.execute_input":"2022-11-03T10:21:14.770205Z","iopub.status.idle":"2022-11-03T10:21:24.784299Z","shell.execute_reply.started":"2022-11-03T10:21:14.770166Z","shell.execute_reply":"2022-11-03T10:21:24.782985Z"},"trusted":true},"execution_count":null,"outputs":[]}]}