{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nfrom glob import glob\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport json\n\nimport numpy as np\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2022-11-09T10:09:11.961541Z","iopub.execute_input":"2022-11-09T10:09:11.962168Z","iopub.status.idle":"2022-11-09T10:09:11.976411Z","shell.execute_reply.started":"2022-11-09T10:09:11.962099Z","shell.execute_reply":"2022-11-09T10:09:11.974497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    chunk_size = 10_0000","metadata":{"execution":{"iopub.status.busy":"2022-11-09T09:14:47.525988Z","iopub.execute_input":"2022-11-09T09:14:47.527405Z","iopub.status.idle":"2022-11-09T09:14:47.534201Z","shell.execute_reply.started":"2022-11-09T09:14:47.527332Z","shell.execute_reply":"2022-11-09T09:14:47.532594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ninput_dir = Path(\"../input/otto-recommender-system/\")\ntrain_jsonl_path = input_dir.joinpath(\"train.jsonl\")\ntest_jsonl_path = input_dir.joinpath(\"test.jsonl\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T09:14:48.066281Z","iopub.execute_input":"2022-11-09T09:14:48.066816Z","iopub.status.idle":"2022-11-09T09:14:48.077381Z","shell.execute_reply.started":"2022-11-09T09:14:48.066774Z","shell.execute_reply":"2022-11-09T09:14:48.075533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def raw_df_to_wide_df_helper(raw_df: pd.DataFrame) -> pd.DataFrame:\n    explode_df = raw_df.explode(\"events\", ignore_index=False)\n    explode_df[\"aid\"] = explode_df[\"events\"].apply(lambda x: x[\"aid\"])\n    explode_df[\"ts\"] = explode_df[\"events\"].apply(lambda x: x[\"ts\"])\n    explode_df[\"type\"] = explode_df[\"events\"].apply(lambda x: x[\"type\"])\n    wide_df = explode_df[[\"session\", \"ts\", \"type\", \"aid\"]]\n    return wide_df","metadata":{"execution":{"iopub.status.busy":"2022-11-09T09:52:55.352192Z","iopub.execute_input":"2022-11-09T09:52:55.353611Z","iopub.status.idle":"2022-11-09T09:52:55.360748Z","shell.execute_reply.started":"2022-11-09T09:52:55.353562Z","shell.execute_reply":"2022-11-09T09:52:55.359621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transform train dataset","metadata":{}},{"cell_type":"code","source":"%%time\ntrain_dfs = pd.read_json(train_jsonl_path, lines=True, chunksize=Config.chunk_size)\nos.makedirs(\"train_parquet\", exist_ok=True)\nfor train_df in tqdm(train_dfs, total=129):\n    train_wide_df = raw_df_to_wide_df_helper(train_df)\n    start_session_id = str(train_wide_df[\"session\"].min()).zfill(9)\n    end_session_id = str(train_wide_df[\"session\"].max()).zfill(9)\n    train_wide_df.to_parquet(f\"train_parquet/{start_session_id}_{end_session_id}.parquet\")\n    ","metadata":{"execution":{"iopub.status.busy":"2022-11-09T10:09:14.956930Z","iopub.execute_input":"2022-11-09T10:09:14.957860Z","iopub.status.idle":"2022-11-09T10:09:21.693463Z","shell.execute_reply.started":"2022-11-09T10:09:14.957787Z","shell.execute_reply":"2022-11-09T10:09:21.692552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transfrom test dataset","metadata":{}},{"cell_type":"code","source":"%%time\ntest_df = pd.read_json(test_jsonl_path, orient=\"records\", lines=True)\nos.makedirs(\"test_parquet\", exist_ok=True)\ntest_wide_df = raw_df_to_wide_df_helper(test_df)\ntest_wide_df.to_parquet(\"test_parquet/test.parquet\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T10:11:46.852842Z","iopub.execute_input":"2022-11-09T10:11:46.854441Z","iopub.status.idle":"2022-11-09T10:22:26.547857Z","shell.execute_reply.started":"2022-11-09T10:11:46.854357Z","shell.execute_reply":"2022-11-09T10:22:26.546355Z"},"trusted":true},"execution_count":null,"outputs":[]}]}