{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":38760,"databundleVersionId":4493939},{"sourceType":"datasetVersion","sourceId":15367278,"datasetId":9829979,"databundleVersionId":16278566},{"sourceType":"datasetVersion","sourceId":15390548,"datasetId":9845233,"databundleVersionId":16304440}],"dockerImageVersionId":31287,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Config","metadata":{"_uuid":"0425345b-fb92-4e26-b1e6-81e45cadc213","_cell_guid":"f9ffb5a7-54aa-4994-9086-bdaf1fee196a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"! pip install vl-convert-python","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-28T06:30:56.122052Z","iopub.execute_input":"2026-03-28T06:30:56.122441Z","iopub.status.idle":"2026-03-28T06:31:05.789902Z","shell.execute_reply.started":"2026-03-28T06:30:56.122407Z","shell.execute_reply":"2026-03-28T06:31:05.788228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\n\nfrom pathlib import Path\n\nfrom dotenv import load_dotenv\nimport polars as pl\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport altair as alt\nfrom datetime import datetime\nimport random\n\nDATASET_DIR = Path('/kaggle/input/competitions/otto-recommender-system')\nTRAIN_JSON = DATASET_DIR / 'train.jsonl'\nTEST_JSON = DATASET_DIR / 'test.jsonl'\nFIG_DIR = Path('/kaggle/working/figures')\nFIG_DIR.mkdir(exist_ok=True, parents=True)","metadata":{"_uuid":"0c855a88-71eb-49e2-a6d1-921ba593389a","_cell_guid":"5c50fd86-e3c4-46ad-8ed7-f9776183788e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:31:05.793492Z","iopub.execute_input":"2026-03-28T06:31:05.794653Z","iopub.status.idle":"2026-03-28T06:31:11.125668Z","shell.execute_reply.started":"2026-03-28T06:31:05.794604Z","shell.execute_reply":"2026-03-28T06:31:11.124236Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def save_altair(chart, filename):\n    stem = Path(filename).stem\n    chart.save(str(FIG_DIR / f\"{stem}.html\"))\n    chart.save(str(FIG_DIR / f\"{stem}.png\"), scale_factor=600/72)\n    chart.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-28T06:31:11.127019Z","iopub.execute_input":"2026-03-28T06:31:11.127684Z","iopub.status.idle":"2026-03-28T06:31:11.133722Z","shell.execute_reply.started":"2026-03-28T06:31:11.127648Z","shell.execute_reply":"2026-03-28T06:31:11.132636Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Convert JSONL to parquet","metadata":{"_uuid":"e6bd0bf0-a29b-44f3-beb6-305cee18c5c0","_cell_guid":"417a173c-0ace-4a32-84d3-d5b96b87ddaa","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# OUTPUT_PATH = Path('/kaggle/working')\n# TRAIN_PARQUET = OUTPUT_PATH / 'train.parquet'\n# TEST_PARQUET = OUTPUT_PATH / 'test.parquet'","metadata":{"_uuid":"26aa7dae-405c-4b9b-961c-2a53c45141e2","_cell_guid":"72c7a277-4bf9-458e-8406-d01bf413678e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:31:11.134842Z","iopub.execute_input":"2026-03-28T06:31:11.135184Z","iopub.status.idle":"2026-03-28T06:31:11.152492Z","shell.execute_reply.started":"2026-03-28T06:31:11.135137Z","shell.execute_reply":"2026-03-28T06:31:11.150974Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def json_to_parquet(file_path, output_path):\n# \tdf = pl.scan_ndjson(file_path) \\\n#     .sink_parquet(output_path, compression='zstd', compression_level=3)\n\n# json_to_parquet(\n# \tstr(TRAIN_JSON),\n# \tstr(TRAIN_PARQUET)\n# )\n\n# json_to_parquet(\n# \tstr(TEST_JSON), str(TEST_PARQUET)\n# )","metadata":{"_uuid":"f55e3ea4-5c04-4618-ba27-332d55174363","_cell_guid":"f54c9f43-7022-4c78-9d49-6fe5524f7784","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:31:11.153743Z","iopub.execute_input":"2026-03-28T06:31:11.154078Z","iopub.status.idle":"2026-03-28T06:31:11.174145Z","shell.execute_reply.started":"2026-03-28T06:31:11.154027Z","shell.execute_reply":"2026-03-28T06:31:11.173173Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. EDA","metadata":{"_uuid":"904ddef0-c54d-48d9-a54c-dbcd3fa90e90","_cell_guid":"dd09a133-1597-4cc7-a5ae-6aa721f7c982","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"PARQUET_DIR = Path('/kaggle/input/datasets/hngphongkiu/otto-parquet')\nTRAIN_PARQUET = PARQUET_DIR / 'train.parquet'\nTEST_PARQUET = PARQUET_DIR / 'test.parquet'","metadata":{"_uuid":"2f9bd599-276f-452f-8b9e-6914528c7c0f","_cell_guid":"ba6426f4-b47d-4fbb-9bec-abc267243fbd","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:31:11.175619Z","iopub.execute_input":"2026-03-28T06:31:11.176689Z","iopub.status.idle":"2026-03-28T06:31:11.198939Z","shell.execute_reply.started":"2026-03-28T06:31:11.176635Z","shell.execute_reply":"2026-03-28T06:31:11.197822Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pl.scan_parquet(TRAIN_PARQUET)\ntest_df = pl.scan_parquet(TEST_PARQUET)","metadata":{"_uuid":"b19d0191-50b7-4896-9b26-e16465930c4e","_cell_guid":"df91fec8-7192-4169-b340-cbd179eaba63","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:31:11.202654Z","iopub.execute_input":"2026-03-28T06:31:11.203034Z","iopub.status.idle":"2026-03-28T06:31:11.242090Z","shell.execute_reply.started":"2026-03-28T06:31:11.203005Z","shell.execute_reply":"2026-03-28T06:31:11.240730Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1.1. Xem dữ liệu train và test","metadata":{"_uuid":"4b6164ab-e171-43dd-a42a-ec621fa09943","_cell_guid":"cdf2daea-85fb-4215-9899-e9426b345a43","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# schema\ntrain_df.schema","metadata":{"_uuid":"cdccc2b4-43da-4400-a24b-9a87beae999f","_cell_guid":"6e16f97c-cf2d-4217-8d11-a2ead359de0e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:31:11.243344Z","iopub.execute_input":"2026-03-28T06:31:11.244194Z","iopub.status.idle":"2026-03-28T06:31:11.372739Z","shell.execute_reply.started":"2026-03-28T06:31:11.244085Z","shell.execute_reply":"2026-03-28T06:31:11.371815Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Xem một vài dữ liệu đầu tiên trong tập train và tập test\nprint('Tập train:')\nprint(train_df.head(3).collect())\n\nprint('-'*50)\n\nprint('Tập test:')\nprint(test_df.head(3).collect())","metadata":{"_uuid":"9ea8d767-d75b-4e1f-91aa-3b5b9918bb25","_cell_guid":"be79bc66-36e4-464d-bcb5-5763e9d0b334","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:31:11.374080Z","iopub.execute_input":"2026-03-28T06:31:11.374489Z","iopub.status.idle":"2026-03-28T06:31:15.048995Z","shell.execute_reply.started":"2026-03-28T06:31:11.374427Z","shell.execute_reply":"2026-03-28T06:31:15.048033Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head(1).collect() \\\n    .explode(\"events\") \\\n    .unnest(\"events\")","metadata":{"_uuid":"d6b8d5fb-062e-4f75-8063-6800470a4811","_cell_guid":"f2fde3cc-d5a9-40ab-b2f3-6e27a13ffe30","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:31:15.050198Z","iopub.execute_input":"2026-03-28T06:31:15.050566Z","iopub.status.idle":"2026-03-28T06:31:15.726551Z","shell.execute_reply.started":"2026-03-28T06:31:15.050530Z","shell.execute_reply":"2026-03-28T06:31:15.725381Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head(1).collect().explode('events').unnest('events')","metadata":{"_uuid":"f2e3356d-7aeb-4322-8f77-642db21aafe4","_cell_guid":"309824b9-10de-4913-bc43-ee34ba20aa56","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:31:15.728162Z","iopub.execute_input":"2026-03-28T06:31:15.728453Z","iopub.status.idle":"2026-03-28T06:31:15.807455Z","shell.execute_reply.started":"2026-03-28T06:31:15.728427Z","shell.execute_reply":"2026-03-28T06:31:15.806421Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1.2. Thống kê đơn giản","metadata":{"_uuid":"b4dd6516-355b-4c04-9584-093f6c4acc78","_cell_guid":"cc181425-d750-4edb-ba26-791d55a4bb6a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def get_events_stats(df, name):\n    n_events_per_session = df.collect().select(pl.col('events').list.len().alias('n_events'))['n_events']\n    return {\n        'Dataset': name,\n        'mean': f'{n_events_per_session.mean():.2f}',\n        'std': f'{n_events_per_session.std():.2f}',\n        'min': n_events_per_session.min(),\n        '50%': n_events_per_session.quantile(0.5),\n        '75%': n_events_per_session.quantile(0.75),\n        '90%': n_events_per_session.quantile(0.9),\n        '95%': n_events_per_session.quantile(0.95),\n        'max': n_events_per_session.max(),\n    }","metadata":{"_uuid":"5eaef0ae-06f8-44ea-86fa-858d669b3b02","_cell_guid":"3f2527ad-4b71-4075-b4ea-16fbf913d5e7","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:31:15.808878Z","iopub.execute_input":"2026-03-28T06:31:15.809227Z","iopub.status.idle":"2026-03-28T06:31:15.815611Z","shell.execute_reply.started":"2026-03-28T06:31:15.809184Z","shell.execute_reply":"2026-03-28T06:31:15.814553Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Thống kê về số events của từng phiên trong tập train')\n\nprint(get_events_stats(train_df, 'train'))\n\nprint('-'*50)\nprint('Thống kê về số events của từng phiên trong tập test')\n\nprint(get_events_stats(test_df, 'test'))","metadata":{"_uuid":"bac5fc5f-1b0c-4bd0-99ad-a39ef95753a1","_cell_guid":"cf4a1406-1e00-4ea6-b732-d46f33a9b17f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:31:15.816702Z","iopub.execute_input":"2026-03-28T06:31:15.817001Z","iopub.status.idle":"2026-03-28T06:31:39.744037Z","shell.execute_reply.started":"2026-03-28T06:31:15.816974Z","shell.execute_reply":"2026-03-28T06:31:39.743203Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Đếm số session, items, events, clicks, carts, orders, và Density\ndef dataset_stats(df, name):\n    df = df.collect()\n    n_session = df.height\n    \n    events = df.select(pl.col('events').list.explode().alias('event')).unnest('event')\n    n_events = events.height\n    n_items = events.select(pl.col('aid')).unique().height\n\n    type_counts_df = events.group_by('type').count()\n    type_dict = dict(zip(type_counts_df['type'], type_counts_df['count']))\n\n    n_clicks = type_dict.get('clicks', 0)\n    n_carts = type_dict.get('carts', 0)\n    n_orders = type_dict.get('orders', 0)\n\n    density = (n_events / (n_session * n_items) * 100) if (n_session * n_items) > 0 else 0\n\n    return {\n        'Dataset': name,\n        '#sessions': n_session,\n        '#items': n_items,\n        '#events': n_events,\n        '#clicks': n_clicks,\n        '#carts': n_carts,\n        '#orders': n_orders,\n        'Density': density\n    }","metadata":{"_uuid":"42b1dd11-b919-490e-b286-8a0a85528b1c","_cell_guid":"17ed1e78-8df3-465a-9123-ecaaea814549","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:31:39.745191Z","iopub.execute_input":"2026-03-28T06:31:39.745524Z","iopub.status.idle":"2026-03-28T06:31:39.753959Z","shell.execute_reply.started":"2026-03-28T06:31:39.745465Z","shell.execute_reply":"2026-03-28T06:31:39.752829Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Thống kê dataset trong tập train:')\n\nprint(dataset_stats(train_df, 'train'))\n\nprint('-'*50)\nprint('Thống kê dataset trong tập test:')\nprint(dataset_stats(test_df, 'test'))","metadata":{"_uuid":"8a631e14-8be0-47a1-a6dd-1db11d72199b","_cell_guid":"a3d1ec29-714c-46ca-ba66-040f21b546ef","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:31:39.755020Z","iopub.execute_input":"2026-03-28T06:31:39.755331Z","iopub.status.idle":"2026-03-28T06:32:28.967660Z","shell.execute_reply.started":"2026-03-28T06:31:39.755305Z","shell.execute_reply":"2026-03-28T06:32:28.966573Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1.3. Số lượng các loại hành vi trong tập huấn luyện và tập kiểm tra","metadata":{"_uuid":"5293a920-ebf9-41bd-9820-556369482591","_cell_guid":"4fabc578-3138-46b9-9133-f60dd75a3892","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def count_event_types(df, name):\n    return df.collect() \\\n            .explode('events') \\\n            .unnest('events') \\\n            .group_by('type') \\\n            .count() \\\n            .sort('count', descending=True)","metadata":{"_uuid":"cc949ef8-aeee-4339-a1ac-456d27ef6bca","_cell_guid":"f279c064-90b7-4ab8-aa75-5c06f1d12004","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:32:28.968828Z","iopub.execute_input":"2026-03-28T06:32:28.969191Z","iopub.status.idle":"2026-03-28T06:32:28.974787Z","shell.execute_reply.started":"2026-03-28T06:32:28.969153Z","shell.execute_reply":"2026-03-28T06:32:28.973233Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_events = count_event_types(train_df, 'train')\ntest_events = count_event_types(test_df, 'test')","metadata":{"_uuid":"6855b546-2ff8-40a8-812e-6d9887357636","_cell_guid":"b60f6b0e-198b-43d2-88c9-d00b58a9d556","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:32:28.976236Z","iopub.execute_input":"2026-03-28T06:32:28.976865Z","iopub.status.idle":"2026-03-28T06:32:57.481003Z","shell.execute_reply.started":"2026-03-28T06:32:28.976810Z","shell.execute_reply":"2026-03-28T06:32:57.480100Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Số lượng event type trong tập train:')\nprint(train_events)\n\nprint('-'*50)\nprint('Số lượng event type trong tập test:')\nprint(test_events)","metadata":{"_uuid":"39a1ce99-46f2-4504-af9e-c7e0dc402e93","_cell_guid":"f7870b7e-98a3-49c8-886a-549b97c26eb9","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:32:57.482180Z","iopub.execute_input":"2026-03-28T06:32:57.482441Z","iopub.status.idle":"2026-03-28T06:32:57.488622Z","shell.execute_reply.started":"2026-03-28T06:32:57.482416Z","shell.execute_reply":"2026-03-28T06:32:57.487312Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_chart(df, title, color):\n    return alt.Chart(df).mark_bar(color=color).encode(\n        x=alt.X('type:N', title='Loại sự kiện'),\n        y=alt.Y('count:Q', title='Số lượng'),\n        tooltip=['type', 'count']\n    ).properties(\n        width=500,\n        height=400,\n        title=title\n    )\n\nchart_train = make_chart(train_events, 'Biểu đồ số lượng các loại hành vi trong tập train', '#66c2a5')\nchart_test  = make_chart(test_events,  'Biểu đồ số lượng các loại hành vi trong tập train',  '#fc8d62')\n\nsave_altair(chart_train | chart_test, \"1_3_event_type_counts\")","metadata":{"_uuid":"bd7ea384-31e7-4772-954d-9f445ba64d06","_cell_guid":"256dd370-36e9-4660-a98f-fc56abf0e409","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:34:13.314153Z","iopub.execute_input":"2026-03-28T06:34:13.314571Z","iopub.status.idle":"2026-03-28T06:34:14.599487Z","shell.execute_reply.started":"2026-03-28T06:34:13.314537Z","shell.execute_reply":"2026-03-28T06:34:14.598517Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1.4. Xét phân phối longtail của sản phẩm trong tập train","metadata":{"_uuid":"533cff82-8f59-4a54-875e-7ea46ff211fd","_cell_guid":"f06ee6ca-fe6a-49d1-95ea-7cb030139cc1","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"item_counts = train_df.collect() \\\n                .explode('events') \\\n                .unnest('events') \\\n                .group_by('aid') \\\n                .agg(pl.len().alias('count')) \\\n                .sort('count', descending=True) \\\n                .with_row_index('rank')\n\nlongtail_chart = alt.Chart(item_counts.head(5000)).mark_area(\n    line={'strokeWidth': 2.5},\n    opacity=0.4\n).encode(\n    x=alt.X('rank:Q', title='Xếp hạng sản phẩm (Item rank)'),\n    y=alt.Y('count:Q', title='Số lượng tương tác'),\n    tooltip=['aid', 'count', 'rank']\n).properties(\n    width=700, height=400,\n    title=alt.TitleParams(\n        text='Số lượng tương tác theo sản phẩm trong tập train (top 5000 item)',\n        fontSize=16\n    )\n).configure_axis(\n    labelFontSize=12,\n    titleFontSize=13\n)\n\nsave_altair(longtail_chart, \"1_4_longtail_chart\")","metadata":{"_uuid":"0dea7fbe-940b-4d02-b15e-57b0cbf85a01","_cell_guid":"dd88b16f-8881-4077-856d-3ba19f97c1eb","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:48:31.235701Z","iopub.execute_input":"2026-03-28T06:48:31.236407Z","iopub.status.idle":"2026-03-28T06:49:20.249700Z","shell.execute_reply.started":"2026-03-28T06:48:31.236363Z","shell.execute_reply":"2026-03-28T06:49:20.247994Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1.5. Xét hành vi người dùng theo khung giờ trong ngày","metadata":{"_uuid":"7cf161ca-5923-4525-9050-b18c68e9a15d","_cell_guid":"0fe1ad2b-fd1a-491b-91de-4efc1ab17475","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"hourly_counts = train_df.collect() \\\n                    .explode('events') \\\n                    .unnest('events') \\\n                    .with_columns(\n                        pl.from_epoch('ts', time_unit='ms').dt.hour().alias('hour')\n                    ) \\\n                    .group_by(['hour', 'type']) \\\n                    .agg(pl.len().alias('count')) \\\n                    .sort('hour')","metadata":{"_uuid":"d0c9d8e4-5efb-49f1-94e1-eb6815fa9140","_cell_guid":"3ec7ac13-a7de-464b-a0e3-2b3b55553a4b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:49:20.251593Z","iopub.execute_input":"2026-03-28T06:49:20.252176Z","iopub.status.idle":"2026-03-28T06:50:05.521950Z","shell.execute_reply.started":"2026-03-28T06:49:20.252139Z","shell.execute_reply":"2026-03-28T06:50:05.520613Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"chart = alt.Chart(hourly_counts).mark_line(strokeWidth=2.5, point=True).encode(\n    x=alt.X('hour:O', title='Giờ trong ngày'),\n    y=alt.Y('count:Q', title='Số lượng hành vi'),\n    color=alt.Color('type:N', title='Loại hành vi'),\n    tooltip=['hour', 'type', 'count']\n).properties(\n    width=700,\n    height=400,\n    title=alt.TitleParams(\n        text='Hành vi người dùng theo khung giờ trong ngày (Train)',\n        fontSize=16,\n    )\n).configure_axis(\n    labelFontSize=12,\n    titleFontSize=13\n)\n\nsave_altair(chart, \"1_5_Interaction_hour\")","metadata":{"_uuid":"07783a10-f340-4a88-9c48-165fdeeab0b3","_cell_guid":"f386f277-3357-4a3b-960a-ab39b4f947d6","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T06:58:07.255392Z","iopub.execute_input":"2026-03-28T06:58:07.256381Z","iopub.status.idle":"2026-03-28T06:58:08.167895Z","shell.execute_reply.started":"2026-03-28T06:58:07.256339Z","shell.execute_reply":"2026-03-28T06:58:08.166410Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Create validation set","metadata":{"_uuid":"3684b797-f093-4f7a-aca4-0b23433d40ab","_cell_guid":"dcd25a6f-68a8-424c-8310-cda100fb29d1","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"OUTPUT_DIR = Path('/kaggle/working')\nTRAIN_SESSIONS_PARQUET = OUTPUT_DIR / 'train_sessions.parquet'\nVALID_PARQUET = OUTPUT_DIR / 'valid_inputs.parquet'\nVALID_LABELS_PARQUET = OUTPUT_DIR / 'valid_labels.parquet'","metadata":{"_uuid":"a8c13474-7bfd-411a-8bef-0fa6e9d2e4f4","_cell_guid":"78d3d209-12b6-455c-9569-a76d3f695b2a","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-28T07:01:48.752016Z","iopub.execute_input":"2026-03-28T07:01:48.752475Z","iopub.status.idle":"2026-03-28T07:01:48.758288Z","shell.execute_reply.started":"2026-03-28T07:01:48.752440Z","shell.execute_reply":"2026-03-28T07:01:48.757183Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nfrom pathlib import Path\nimport psutil\nimport random\nfrom copy import deepcopy\nfrom tqdm.auto import tqdm\n\n\ndef _ground_truth(events: list[dict]):\n    prev_labels = {\"clicks\": None, \"carts\": set(), \"orders\": set()}\n    for event in reversed(events):\n        event[\"labels\"] = {}\n        for label in ['clicks', 'carts', 'orders']:\n            if prev_labels[label]:\n                if label != 'clicks':\n                    event[\"labels\"][label] = prev_labels[label].copy()\n                else:\n                    event[\"labels\"][label] = prev_labels[label]\n        if event[\"type\"] == \"clicks\":\n            prev_labels['clicks'] = event[\"aid\"]\n        elif event[\"type\"] == \"carts\":\n            prev_labels['carts'].add(event[\"aid\"])\n        elif event[\"type\"] == \"orders\":\n            prev_labels['orders'].add(event[\"aid\"])\n    return events[:-1]\n\n\ndef check_ram_available(min_gb=2):\n    available = psutil.virtual_memory().available / (1024**3)\n    total = psutil.virtual_memory().total / (1024**3)\n    print(f\"\\nRAM CHECK:\")\n    print(f\"  Total:     {total:.1f} GB\")\n    print(f\"  Available: {available:.1f} GB\")\n    if available < min_gb:\n        print(f\"  WARNING: chỉ còn {available:.1f}GB (cần >= {min_gb}GB)\")\n        return False\n    print(f\"  OK\")\n    return True\n\n\ndef simple_split_parquet(train_parquet_path, output_dir, test_days=7, seed=42):\n    random.seed(seed)\n    check_ram_available(min_gb=2)\n\n    output_dir = Path(output_dir)\n    output_dir.mkdir(exist_ok=True, parents=True)\n\n    print(f\"\\n{'='*60}\")\n    print(\"OTTO TRAIN/VALID SPLIT\")\n    print(f\"{'='*60}\")\n\n    # Bước 1: Tính split_ts\n    print(\"\\n[1/6] Tính split timestamp...\")\n    events_df = pl.scan_parquet(train_parquet_path).explode('events').unnest('events')\n\n    max_ts = events_df.select(pl.col('ts').max()).collect().item()\n    split_ts = max_ts - test_days * 24 * 60 * 60 * 1000\n    print(f\"      max_ts   = {max_ts}\")\n    print(f\"      split_ts = {split_ts}\")\n    print(f\"      test period = {test_days} days\")\n\n    # Bước 2: Chia session train/test\n    print(\"\\n[2/6] Phân loại sessions...\")\n    session_first_ts = (\n        events_df\n        .group_by('session')\n        .agg(pl.col('ts').min().alias('first_ts'))\n    )\n\n    train_session_ids_df = session_first_ts.filter(pl.col('first_ts') <= split_ts).select('session')\n    test_session_ids_df  = session_first_ts.filter(pl.col('first_ts') >  split_ts).select('session')\n\n    n_train_sessions = train_session_ids_df.collect().height\n    n_test_sessions  = test_session_ids_df.collect().height\n    print(f\"      Train sessions: {n_train_sessions:,}\")\n    print(f\"      Test sessions:  {n_test_sessions:,}\")\n\n    # Bước 3: Tạo train_sessions\n    print(\"\\n[3/6] Tạo train sessions (trim events sau split_ts)...\")\n    with tqdm(total=1, desc=\"      Building train sessions\") as pbar:\n        train_sessions = (\n            events_df\n            .join(train_session_ids_df, on='session', how='semi')\n            .filter(pl.col('ts') < split_ts)\n            .group_by('session')\n            .agg(\n                pl.struct(['aid', 'ts', 'type'])\n                .sort_by('ts')\n                .alias('events')\n            )\n            .filter(pl.col('events').list.len() >= 2)\n            .collect()\n        )\n        pbar.update(1)\n    print(f\"      Train sessions sau filter: {train_sessions.height:,}\")\n\n    # Bước 4: Lọc unknown items trong test\n    print(\"\\n[4/6] Filter unknown items trong test sessions...\")\n    \n    with tqdm(total=1, desc=\"      Collecting train aids\") as pbar:\n        train_items_df = (\n            events_df\n            .join(train_session_ids_df, on='session', how='semi')\n            .filter(pl.col('ts') < split_ts)\n            .select('aid')\n            .unique()\n            .collect()\n        )\n        train_aids_set = set(train_items_df['aid'].to_list())\n        pbar.update(1)\n    print(f\"      Train unique aids: {len(train_aids_set):,}\")\n    \n    with tqdm(total=1, desc=\"      Collecting test events\") as pbar:\n        test_events_raw = (\n            events_df\n            .join(test_session_ids_df, on='session', how='semi')\n            .collect()\n        )\n        pbar.update(1)\n    \n    # Eager join sau collect\n    with tqdm(total=1, desc=\"      Filtering & grouping test sessions\") as pbar:\n        test_sessions_filtered = (\n            test_events_raw\n            .join(train_items_df, on='aid', how='semi')\n            .group_by('session')\n            .agg(\n                pl.struct(['aid', 'ts', 'type'])\n                .sort_by('ts')\n                .alias('events')\n            )\n            .filter(pl.col('events').list.len() >= 2)\n        )\n        pbar.update(1)\n    \n    print(f\"      Test sessions sau filter: {test_sessions_filtered.height:,}\")\n\n    # Bước 5: Tạo valid input/label pairs\n    print(\"\\n[5/6] Tạo valid input/label pairs...\")\n    valid_inputs_list = []\n    valid_labels_list = []\n    skipped = 0\n\n    for row in tqdm(test_sessions_filtered.iter_rows(named=True),\n                    total=test_sessions_filtered.height,\n                    desc=\"      Processing sessions\"):\n        session_id = row['session']\n        events = [dict(e) for e in row['events']]\n\n        if len(events) < 2:\n            skipped += 1\n            continue\n\n        test_events = _ground_truth(deepcopy(events))\n        if not test_events:\n            skipped += 1\n            continue\n\n        split_idx = random.randint(1, len(test_events))\n        test_events_trimmed = test_events[:split_idx]\n        raw_labels = test_events_trimmed[-1]['labels']\n\n        clean_clicks = raw_labels.get('clicks')\n        if clean_clicks is not None and clean_clicks not in train_aids_set:\n            clean_clicks = None\n\n        clean_carts  = [a for a in raw_labels.get('carts',  set()) if a in train_aids_set]\n        clean_orders = [a for a in raw_labels.get('orders', set()) if a in train_aids_set]\n\n        if not any([clean_clicks, clean_carts, clean_orders]):\n            skipped += 1\n            continue\n\n        history = [{k: v for k, v in e.items() if k != 'labels'}\n                   for e in test_events_trimmed]\n\n        valid_inputs_list.append({'session': session_id, 'events': history})\n        valid_labels_list.append({\n            'session': session_id,\n            'labels': {\n                'clicks': clean_clicks,\n                'carts':  clean_carts,\n                'orders': clean_orders,\n            }\n        })\n\n    print(f\"      Valid pairs tạo được: {len(valid_inputs_list):,}\")\n    print(f\"      Skipped: {skipped:,}\")\n\n    # Bước 6: Ghi parquet\n    print(\"\\n[6/6] Ghi parquet...\")\n\n    files = {\n        'train_sessions.parquet': train_sessions,\n        'valid_inputs.parquet':   pl.DataFrame(valid_inputs_list),\n        'valid_labels.parquet':   pl.DataFrame(valid_labels_list),\n    }\n\n    for fname, df in tqdm(files.items(), desc=\"      Writing files\"):\n        df.write_parquet(\n            str(output_dir / fname),\n            compression='zstd',\n            compression_level=3\n        )\n\n    print(f\"\\n{'='*60}\")\n    print(\"DONE!\")\n    print(f\"{'='*60}\")\n    print(f\"\\n  Train sessions : {train_sessions.height:,}\")\n    print(f\"  Valid sessions : {len(valid_inputs_list):,}\")\n    print(f\"\\n  Files saved to: {output_dir}\")\n    for fname in files:\n        path = output_dir / fname\n        size_mb = path.stat().st_size / (1024**2)\n        print(f\"    {fname:<30} {size_mb:.1f} MB\")\n\n\nsimple_split_parquet(\n    train_parquet_path=TRAIN_PARQUET,\n    output_dir=Path('/kaggle/working'),\n    test_days=7,\n    seed=42\n)","metadata":{"_uuid":"b367ba5d-bf6a-4ed0-b21d-57966503fe7c","_cell_guid":"864013ff-11fa-430b-8b4f-f64bb981174e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-27T04:10:44.702120Z","iopub.status.idle":"2026-03-27T04:10:44.702454Z","shell.execute_reply.started":"2026-03-27T04:10:44.702313Z","shell.execute_reply":"2026-03-27T04:10:44.702331Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nfrom pathlib import Path\n\nOUTPUT_DIR = Path('/kaggle/working')\nTRAIN_SESSIONS_PARQUET = OUTPUT_DIR / 'train_sessions.parquet'\nVALID_PARQUET          = OUTPUT_DIR / 'valid_inputs.parquet'\nVALID_LABELS_PARQUET   = OUTPUT_DIR / 'valid_labels.parquet'\n\ntrain  = pl.read_parquet(TRAIN_SESSIONS_PARQUET)\ninputs = pl.read_parquet(VALID_PARQUET)\nlabels = pl.read_parquet(VALID_LABELS_PARQUET)\n\nprint(\"=\" * 60)\nprint(\"1. SCHEMA & SHAPE\")\nprint(\"=\" * 60)\nprint(f\"\\ntrain_sessions : {train.shape}\")\nprint(train.schema)\nprint(f\"\\nvalid_inputs   : {inputs.shape}\")\nprint(inputs.schema)\nprint(f\"\\nvalid_labels   : {labels.shape}\")\nprint(labels.schema)\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"2. SAMPLE DATA\")\nprint(\"=\" * 60)\nprint(\"\\n-- train (1 row) --\")\nprint(train.head(1))\nprint(\"\\n-- valid_inputs (1 row) --\")\nprint(inputs.head(1))\nprint(\"\\n-- valid_labels (1 row) --\")\nprint(labels.head(1))\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"3. NULL CHECK\")\nprint(\"=\" * 60)\nprint(f\"\\ntrain  nulls: {train.null_count().row(0)}\")\nprint(f\"inputs nulls: {inputs.null_count().row(0)}\")\nprint(f\"labels nulls: {labels.null_count().row(0)}\")\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"4. SESSION ID CHECKS\")\nprint(\"=\" * 60)\n\ninput_sessions = set(inputs['session'].to_list())\nlabel_sessions = set(labels['session'].to_list())\ntrain_session_ids = set(train['session'].to_list())\n\nprint(f\"\\ninputs sessions : {len(input_sessions):,}\")\nprint(f\"labels sessions : {len(label_sessions):,}\")\nprint(f\"inputs == labels: {input_sessions == label_sessions}\")\n\noverlap = input_sessions & train_session_ids\nprint(f\"\\nTrain/valid session overlap: {len(overlap):,}  (phải = 0)\")\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"5. EVENTS SANITY CHECK\")\nprint(\"=\" * 60)\n\ntrain_event_lens = train.with_columns(\n    pl.col('events').list.len().alias('n_events')\n)['n_events']\ninput_event_lens = inputs.with_columns(\n    pl.col('events').list.len().alias('n_events')\n)['n_events']\n\nprint(f\"\\ntrain  events/session — min:{train_event_lens.min()}, mean:{train_event_lens.mean():.1f}, max:{train_event_lens.max()}\")\nprint(f\"inputs events/session — min:{input_event_lens.min()}, mean:{input_event_lens.mean():.1f}, max:{input_event_lens.max()}\")\nprint(f\"Any session with 0 events (train) : {(train_event_lens == 0).sum()}\")\nprint(f\"Any session with 0 events (inputs): {(input_event_lens == 0).sum()}\")\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"6. LABEL SANITY CHECK\")\nprint(\"=\" * 60)\n\nlabel_df = labels.with_columns([\n    pl.col('labels').struct.field('clicks').alias('clicks'),\n    pl.col('labels').struct.field('carts').alias('carts'),\n    pl.col('labels').struct.field('orders').alias('orders'),\n])\n\nn_has_clicks = label_df['clicks'].drop_nulls().len()\nn_has_carts  = label_df.filter(pl.col('carts').list.len() > 0).height\nn_has_orders = label_df.filter(pl.col('orders').list.len() > 0).height\ntotal        = labels.height\n\nprint(f\"\\nSessions có clicks label : {n_has_clicks:,} ({100*n_has_clicks/total:.1f}%)\")\nprint(f\"Sessions có carts  label : {n_has_carts:,}  ({100*n_has_carts/total:.1f}%)\")\nprint(f\"Sessions có orders label : {n_has_orders:,} ({100*n_has_orders/total:.1f}%)\")\nprint(f\"Sessions không có label nào: \"\n      f\"{label_df.filter(pl.col('clicks').is_null() & (pl.col('carts').list.len()==0) & (pl.col('orders').list.len()==0)).height:,}\")\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"7. ITEM LEAKAGE CHECK\")\nprint(\"=\" * 60)\n\n# So sánh với train_sessions — đúng nguồn model được train\ntrain_aids = set(\n    train\n    .explode('events')\n    .unnest('events')\n    ['aid'].to_list()\n)\nvalid_aids = set(\n    inputs\n    .explode('events')\n    .unnest('events')\n    ['aid'].to_list()\n)\n\nleaked = valid_aids - train_aids\nprint(f\"\\nTrain unique aids : {len(train_aids):,}\")\nprint(f\"Valid unique aids  : {len(valid_aids):,}\")\nprint(f\"Unknown aids in valid (phải = 0): {len(leaked):,}\")\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"8. TIMESTAMP ORDER CHECK (sample 1000 sessions)\")\nprint(\"=\" * 60)\n\nsample = inputs.sample(min(1000, inputs.height), seed=42)\nn_unsorted = 0\nfor row in sample.iter_rows(named=True):\n    ts_list = [e['ts'] for e in row['events']]\n    if ts_list != sorted(ts_list):\n        n_unsorted += 1\n\nprint(f\"\\nSessions có events không theo thứ tự ts: {n_unsorted} (phải = 0)\")\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"SUMMARY\")\nprint(\"=\" * 60)\nchecks = {\n    \"inputs == labels session ids\" : input_sessions == label_sessions,\n    \"No train/valid overlap\"       : len(overlap) == 0,\n    \"No 0-event sessions (train)\"  : (train_event_lens == 0).sum() == 0,\n    \"No 0-event sessions (inputs)\" : (input_event_lens == 0).sum() == 0,\n    \"No unknown aids in valid\"     : len(leaked) == 0,\n    \"Events sorted by ts\"          : n_unsorted == 0,\n}\nfor check, passed in checks.items():\n    status = \"PASS\" if passed else \"FAIL\"\n    print(f\"  [{status}] {check}\")","metadata":{"_uuid":"35c848d9-22b4-44e9-b434-01872b135308","_cell_guid":"97efb33a-c4cf-4883-9dab-8ad81a6b4755","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-27T04:10:44.704367Z","iopub.status.idle":"2026-03-27T04:10:44.704769Z","shell.execute_reply.started":"2026-03-27T04:10:44.704604Z","shell.execute_reply":"2026-03-27T04:10:44.704627Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}