{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import library.","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\n\nimport json\nimport datetime\n\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-30T16:17:40.040061Z","iopub.execute_input":"2022-11-30T16:17:40.040880Z","iopub.status.idle":"2022-11-30T16:17:41.215617Z","shell.execute_reply.started":"2022-11-30T16:17:40.040766Z","shell.execute_reply":"2022-11-30T16:17:41.214479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define file path","metadata":{}},{"cell_type":"code","source":"DATA_DIR = '/kaggle/input/otto-recommender-system'\n\nFP_TRAIN_JSONL = os.path.join(DATA_DIR, 'train.jsonl')\nFP_TEST_JSONL =  os.path.join(DATA_DIR, 'test.jsonl')\nFP_SAMPLE_SUBMISSION = os.path.join(DATA_DIR, 'sample_submission.csv')","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-30T16:17:41.217463Z","iopub.execute_input":"2022-11-30T16:17:41.217813Z","iopub.status.idle":"2022-11-30T16:17:41.224140Z","shell.execute_reply.started":"2022-11-30T16:17:41.217782Z","shell.execute_reply":"2022-11-30T16:17:41.222514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load jsonl as dataframe.","metadata":{}},{"cell_type":"code","source":"N_TRAIN = 216716096\nN_TEST = 6928123\n\n\ntype_to_id = {'clicks': 0, 'carts': 1, 'orders': 2}\n\n\ndef load_otto_jsonl_as_dataframe(jsonl_file_path, n_rows):\n    # session_id_array\n    sid_arr = np.zeros(n_rows, dtype=np.uint32)\n    # action_id_array\n    aid_arr = np.zeros(n_rows, dtype=np.uint32)\n    # time_stamp_array\n    ts_arr = np.zeros(n_rows, dtype=np.uint64)\n    # time_delta_array\n    td_arr = np.zeros(n_rows, dtype=np.uint32)\n    type_arr = np.zeros(n_rows, dtype=np.uint8)\n    datetime_arr = np.zeros(n_rows, dtype=np.datetime64)\n    with open(jsonl_file_path, 'r') as f:\n        row_count = 0\n        current_session_id = -1\n        start_ts = None\n\n        def calc_time_delta(session_id, ts):\n            nonlocal current_session_id\n            nonlocal start_ts\n            if current_session_id != session_id:\n                current_session_id = session_id\n                start_ts = ts\n                return 0\n            else:\n                return ts - start_ts\n\n        for line in f:\n            session = json.loads(line)\n            session_id = session['session']\n            events = session['events']\n            for event_count, event in enumerate(events):\n                sid_arr[row_count] = session_id\n                aid_arr[row_count] = event.get('aid')\n                ts_arr[row_count] = event.get('ts')\n                td_arr[row_count] = calc_time_delta(\n                    session_id,\n                    event.get('ts'))\n                type_arr[row_count] = type_to_id.get(event.get('type'))\n                row_count += 1\n    return pd.DataFrame({'session': sid_arr,\n                         'aid': aid_arr,\n#                         'ts': ts_arr,\n                         'td': td_arr,\n                         'datetime': ts_arr.astype('datetime64[ms]'),\n                         'type': type_arr})","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-30T16:17:41.225538Z","iopub.execute_input":"2022-11-30T16:17:41.225866Z","iopub.status.idle":"2022-11-30T16:17:41.239017Z","shell.execute_reply.started":"2022-11-30T16:17:41.225836Z","shell.execute_reply":"2022-11-30T16:17:41.238208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = load_otto_jsonl_as_dataframe(FP_TRAIN_JSONL, N_TRAIN)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-30T16:17:41.241230Z","iopub.execute_input":"2022-11-30T16:17:41.242101Z","iopub.status.idle":"2022-11-30T16:27:08.544599Z","shell.execute_reply.started":"2022-11-30T16:17:41.242066Z","shell.execute_reply":"2022-11-30T16:27:08.542390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## preprocess","metadata":{}},{"cell_type":"code","source":"def otto_preprocess(df, map_aid=None):\n    if not map_aid is None:\n        df['aid_id'] = df['aid'].map(map_aid).astype('int32')\n    df['unique_days'] = (df['datetime'] - np.datetime64('2022-07-31T00:00:00')).astype('timedelta64[D]').astype('int8')\n    df['unique_week'] = (df['unique_days'] // 7).astype('int8')\n    df['day_of_week'] = (df['unique_days'] % 7).astype('int8')\n    df['unique_hour'] = (df['datetime'] - np.datetime64('2022-07-31T22:00:00')).astype('timedelta64[h]').astype('int16')\n    df['hour'] = ((df['datetime'] - np.datetime64('2022-07-31T00:00:00')).astype('timedelta64[h]').astype('int8') % 24).astype('int8')\n    df.drop(columns=['datetime'])\n    gc.collect()\n    session_start_hours = df.groupby('session')['unique_hour'].min().to_dict()\n    unique_hour_to_session_start_count = dict()\n    for k, v in session_start_hours.items():\n        if v in unique_hour_to_session_start_count:\n            unique_hour_to_session_start_count[v].append(k)\n        else:\n            unique_hour_to_session_start_count[v] = [k]\n    df['session_start_count_by_hour_log'] = np.log2(df['unique_hour'].map(unique_hour_to_session_start_count).apply(len) + 1.0).astype('int8')\n    del session_start_hours, unique_hour_to_session_start_count\n    df['td_log'] = (np.log2(df['td'] * 1e-3 + 1.0) * 10.0).astype('int16')\n    return df","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-30T16:27:08.547675Z","iopub.execute_input":"2022-11-30T16:27:08.548178Z","iopub.status.idle":"2022-11-30T16:27:08.562334Z","shell.execute_reply.started":"2022-11-30T16:27:08.548129Z","shell.execute_reply":"2022-11-30T16:27:08.561231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# original_aids = pd.concat([train_df, test_df]).aid.unique()\n# original_aids = train_df.aid.unique()\n# converted_aids = np.arange(len(original_aids))\n# np.random.shuffle(converted_aids)\n# map_aid = {original_id: converted_id for original_id, converted_id in zip(original_aids, converted_aids)}\n# del original_aids, converted_aids\nmap_aid = None\n\ntrain_df = otto_preprocess(train_df, map_aid)\n# test_df = otto_preprocess(test_df)\ndel map_aid","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-30T16:27:08.563823Z","iopub.execute_input":"2022-11-30T16:27:08.565146Z","iopub.status.idle":"2022-11-30T16:29:37.666734Z","shell.execute_reply.started":"2022-11-30T16:27:08.565075Z","shell.execute_reply":"2022-11-30T16:29:37.665397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_df)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-11-30T16:29:37.668596Z","iopub.execute_input":"2022-11-30T16:29:37.669674Z","iopub.status.idle":"2022-11-30T16:29:37.702444Z","shell.execute_reply.started":"2022-11-30T16:29:37.669621Z","shell.execute_reply":"2022-11-30T16:29:37.700970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## About day of week.","metadata":{}},{"cell_type":"code","source":"# Data for week with holiday are excluded.\ngroupby_type_and_day_of_week = train_df.groupby(['type','unique_week', 'day_of_week'])","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:29:37.703946Z","iopub.execute_input":"2022-11-30T16:29:37.704335Z","iopub.status.idle":"2022-11-30T16:29:37.711270Z","shell.execute_reply.started":"2022-11-30T16:29:37.704302Z","shell.execute_reply":"2022-11-30T16:29:37.709688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"day_of_week_map = {\n    0: 'Sun',\n    1: 'Mon',\n    2: 'Tue',\n    3: 'Wed',\n    4: 'Thu',\n    5: 'Fri',\n    6: 'Sat',\n}\n\nunique_week_map = {\n    0: '7/31~8/6',\n    1: '8/7~8/13',\n    2: '8/14~8/20',\n    3: '8/21~8/27',\n    4: '8/28'\n}","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:29:37.713570Z","iopub.execute_input":"2022-11-30T16:29:37.714137Z","iopub.status.idle":"2022-11-30T16:29:37.723536Z","shell.execute_reply.started":"2022-11-30T16:29:37.714094Z","shell.execute_reply":"2022-11-30T16:29:37.722315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"click_count_by_day_of_week = groupby_type_and_day_of_week.count().loc[0]\nclick_count_by_day_of_week['unique_week_label'] = click_count_by_day_of_week.index.map(lambda index: unique_week_map[index[0]])\nclick_count_by_day_of_week['day_of_week_label'] = click_count_by_day_of_week.index.map(lambda index: day_of_week_map[index[1]])","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:29:37.728999Z","iopub.execute_input":"2022-11-30T16:29:37.729367Z","iopub.status.idle":"2022-11-30T16:30:14.518111Z","shell.execute_reply.started":"2022-11-30T16:29:37.729337Z","shell.execute_reply":"2022-11-30T16:30:14.516289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g= sns.catplot(\n    data=click_count_by_day_of_week,\n    x='day_of_week_label',\n    y='session',\n    col='unique_week_label',\n    kind='bar',\n)\ng.set_axis_labels('Day of week', 'Click count')\ng.set_titles('{col_name} week')\ndel click_count_by_day_of_week","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:30:14.520644Z","iopub.execute_input":"2022-11-30T16:30:14.521164Z","iopub.status.idle":"2022-11-30T16:30:15.556174Z","shell.execute_reply.started":"2022-11-30T16:30:14.521105Z","shell.execute_reply":"2022-11-30T16:30:15.554938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cart_count_by_day_of_week = groupby_type_and_day_of_week.count().loc[1]\ncart_count_by_day_of_week['unique_week_label'] = cart_count_by_day_of_week.index.map(lambda index: unique_week_map[index[0]])\ncart_count_by_day_of_week['day_of_week_label'] = cart_count_by_day_of_week.index.map(lambda index: day_of_week_map[index[1]])","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:30:15.558171Z","iopub.execute_input":"2022-11-30T16:30:15.558545Z","iopub.status.idle":"2022-11-30T16:30:26.528597Z","shell.execute_reply.started":"2022-11-30T16:30:15.558512Z","shell.execute_reply":"2022-11-30T16:30:26.527056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g= sns.catplot(\n    data=cart_count_by_day_of_week,\n    x='day_of_week_label',\n    y='session',\n    col='unique_week_label',\n    kind='bar',\n)\ng.set_axis_labels('Day of week', 'Cart count')\ng.set_titles('{col_name} week')\ndel cart_count_by_day_of_week","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:30:26.530816Z","iopub.execute_input":"2022-11-30T16:30:26.531181Z","iopub.status.idle":"2022-11-30T16:30:27.565704Z","shell.execute_reply.started":"2022-11-30T16:30:26.531149Z","shell.execute_reply":"2022-11-30T16:30:27.564249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"order_count_by_day_of_week = groupby_type_and_day_of_week.count().loc[1]\norder_count_by_day_of_week['unique_week_label'] = order_count_by_day_of_week.index.map(lambda index: unique_week_map[index[0]])\norder_count_by_day_of_week['day_of_week_label'] = order_count_by_day_of_week.index.map(lambda index: day_of_week_map[index[1]])","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:30:27.567006Z","iopub.execute_input":"2022-11-30T16:30:27.567377Z","iopub.status.idle":"2022-11-30T16:30:38.345464Z","shell.execute_reply.started":"2022-11-30T16:30:27.567345Z","shell.execute_reply":"2022-11-30T16:30:38.343930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g= sns.catplot(\n    data=order_count_by_day_of_week,\n    x='day_of_week_label',\n    y='session',\n    col='unique_week_label',\n    kind='bar',\n)\ng.set_axis_labels('Day of week', 'Order count')\ng.set_titles('{col_name} week')\ndel order_count_by_day_of_week","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:30:38.346864Z","iopub.execute_input":"2022-11-30T16:30:38.347187Z","iopub.status.idle":"2022-11-30T16:30:39.356520Z","shell.execute_reply.started":"2022-11-30T16:30:38.347157Z","shell.execute_reply":"2022-11-30T16:30:39.355358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"What happen on 8/9? time sale...?\nUnsurprisingly, Sundays have the most closed deals.","metadata":{}},{"cell_type":"code","source":"del groupby_type_and_day_of_week","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:30:39.357629Z","iopub.execute_input":"2022-11-30T16:30:39.357947Z","iopub.status.idle":"2022-11-30T16:30:39.612416Z","shell.execute_reply.started":"2022-11-30T16:30:39.357918Z","shell.execute_reply":"2022-11-30T16:30:39.610473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## About hour ","metadata":{}},{"cell_type":"code","source":"groupby_type_and_hour = train_df.groupby(['type', 'hour'])\nclick_count_by_hour = groupby_type_and_hour.count().loc[0].reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:30:39.613962Z","iopub.execute_input":"2022-11-30T16:30:39.614457Z","iopub.status.idle":"2022-11-30T16:31:10.377996Z","shell.execute_reply.started":"2022-11-30T16:30:39.614399Z","shell.execute_reply":"2022-11-30T16:31:10.377128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g= sns.catplot(\n    data=click_count_by_hour,\n    x='hour',\n    y='session',\n    kind='bar',\n)\ng.set_axis_labels('Hour', 'Click count')\ndel click_count_by_hour","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:31:10.379065Z","iopub.execute_input":"2022-11-30T16:31:10.380870Z","iopub.status.idle":"2022-11-30T16:31:11.002937Z","shell.execute_reply.started":"2022-11-30T16:31:10.380822Z","shell.execute_reply":"2022-11-30T16:31:11.001351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The local time in Germany is 1 hour ahead of UTC.","metadata":{}},{"cell_type":"code","source":"cart_count_by_hour = groupby_type_and_hour.count().loc[1].reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:31:11.004476Z","iopub.execute_input":"2022-11-30T16:31:11.004808Z","iopub.status.idle":"2022-11-30T16:31:22.821286Z","shell.execute_reply.started":"2022-11-30T16:31:11.004776Z","shell.execute_reply":"2022-11-30T16:31:22.820091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g= sns.catplot(\n    data=cart_count_by_hour,\n    x='hour',\n    y='session',\n    kind='bar',\n)\ng.set_axis_labels('Hour', 'Cart count')\ndel cart_count_by_hour","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:31:22.823009Z","iopub.execute_input":"2022-11-30T16:31:22.823896Z","iopub.status.idle":"2022-11-30T16:31:23.273236Z","shell.execute_reply.started":"2022-11-30T16:31:22.823846Z","shell.execute_reply":"2022-11-30T16:31:23.271886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"order_count_by_hour = groupby_type_and_hour.count().loc[2].reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:31:23.274730Z","iopub.execute_input":"2022-11-30T16:31:23.275053Z","iopub.status.idle":"2022-11-30T16:31:35.090660Z","shell.execute_reply.started":"2022-11-30T16:31:23.275023Z","shell.execute_reply":"2022-11-30T16:31:35.089273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g= sns.catplot(\n    data=order_count_by_hour,\n    x='hour',\n    y='session',\n    kind='bar',\n)\ng.set_axis_labels('Hour', 'Order count')\ndel order_count_by_hour","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:31:35.092064Z","iopub.execute_input":"2022-11-30T16:31:35.092414Z","iopub.status.idle":"2022-11-30T16:31:35.563376Z","shell.execute_reply.started":"2022-11-30T16:31:35.092383Z","shell.execute_reply":"2022-11-30T16:31:35.561879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Peak hours around lunch time and around 4am? I don't know much about German culture, so I don't have a good image of it...","metadata":{}},{"cell_type":"code","source":"del groupby_type_and_hour","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:31:35.564847Z","iopub.execute_input":"2022-11-30T16:31:35.565236Z","iopub.status.idle":"2022-11-30T16:31:35.806860Z","shell.execute_reply.started":"2022-11-30T16:31:35.565204Z","shell.execute_reply":"2022-11-30T16:31:35.805414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"About data bias.","metadata":{}},{"cell_type":"code","source":"min_session_length = min(train_df.groupby('session').size())\nprint('Minimum session length for training data is {}'.format(min_session_length))\nmax_session_length = max(train_df.groupby('session').size())\nprint('Maximum session length for training data is {}'.format(max_session_length))","metadata":{"execution":{"iopub.status.busy":"2022-11-30T16:39:06.945574Z","iopub.execute_input":"2022-11-30T16:39:06.946047Z","iopub.status.idle":"2022-11-30T16:39:25.827622Z","shell.execute_reply.started":"2022-11-30T16:39:06.946008Z","shell.execute_reply":"2022-11-30T16:39:25.826262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Production data would have included session lengths greater than 1 or 500. In other words, the compensation data is considered to have been artificially extracted.\nDoes this data make sense to analyze clicks per session per day or orders per day? I don't think it makes sense.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}