{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":38760,"databundleVersionId":4493939,"sourceType":"competition"}],"dockerImageVersionId":30615,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-12T05:59:20.067109Z","iopub.execute_input":"2023-12-12T05:59:20.067632Z","iopub.status.idle":"2023-12-12T05:59:21.224696Z","shell.execute_reply.started":"2023-12-12T05:59:20.067569Z","shell.execute_reply":"2023-12-12T05:59:21.223540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_data(df: pd.DataFrame) -> pd.DataFrame:\n    df_c = df.explode(\"events\", ignore_index=True)\n    df_c = pd.concat([df_c.drop([\"events\"], axis=1), df_c[\"events\"].apply(pd.Series)], axis=1)\n    return df_c","metadata":{"execution":{"iopub.status.busy":"2023-12-12T05:59:22.124974Z","iopub.execute_input":"2023-12-12T05:59:22.126151Z","iopub.status.idle":"2023-12-12T05:59:22.131999Z","shell.execute_reply.started":"2023-12-12T05:59:22.126098Z","shell.execute_reply":"2023-12-12T05:59:22.131131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_TS_START = 1659726648560\nTRAIN_TS_END = 1660304817956\n\nTEST_TS_START = TRAIN_TS_END\nTEST_TS_END = 1660608701141","metadata":{"execution":{"iopub.status.busy":"2023-12-12T05:59:23.017891Z","iopub.execute_input":"2023-12-12T05:59:23.019124Z","iopub.status.idle":"2023-12-12T05:59:23.024740Z","shell.execute_reply.started":"2023-12-12T05:59:23.019069Z","shell.execute_reply":"2023-12-12T05:59:23.023433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.system(f\"mkdir /kaggle/working/otto_exploded_dataset\")\n\nfor it in [\"clicks\", \"carts\", \"orders\"]:\n    os.system(f\"mkdir /kaggle/working/otto_exploded_dataset/{it}\")\n\nfor dt in [\"train\", \"test\"]:\n    for it in [\"clicks\", \"carts\", \"orders\"]:\n        os.system(f\"mkdir /kaggle/working/otto_exploded_dataset/{it}/{dt}\")","metadata":{"execution":{"iopub.status.busy":"2023-12-12T05:59:24.338512Z","iopub.execute_input":"2023-12-12T05:59:24.339921Z","iopub.status.idle":"2023-12-12T05:59:24.388489Z","shell.execute_reply.started":"2023-12-12T05:59:24.339852Z","shell.execute_reply":"2023-12-12T05:59:24.387128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunks = pd.read_json(\n    \"/kaggle/input/otto-recommender-system/train.jsonl\",\n    lines=True,\n    chunksize=2**15\n)\n\nfor i, df in enumerate(tqdm(chunks)):\n    df_pp = preprocess_data(df)\n    for itype in [\"clicks\", \"carts\", \"orders\"]:\n        print(itype)\n        df_pp_t = df_pp[df_pp[\"type\"] == itype]\n        \n        df_pp_t_train = df_pp_t[df_pp_t[\"ts\"].between(TRAIN_TS_START, TRAIN_TS_END, \"left\")]\n        df_pp_t_test = df_pp_t[df_pp_t[\"ts\"].between(TEST_TS_START, TEST_TS_END, \"left\")]\n    \n        df_pp_t_train.to_parquet(\n            \"/kaggle/working/otto_exploded_dataset/\" + itype + f\"/train/part_{i}.parquet\"\n        )\n        df_pp_t_test.to_parquet(\n            \"/kaggle/working/otto_exploded_dataset/\" + itype + f\"/test/part_{i}.parquet\"\n        )","metadata":{"execution":{"iopub.status.busy":"2023-12-12T05:59:32.302910Z","iopub.execute_input":"2023-12-12T05:59:32.303358Z","iopub.status.idle":"2023-12-12T08:47:46.900857Z","shell.execute_reply.started":"2023-12-12T05:59:32.303324Z","shell.execute_reply":"2023-12-12T08:47:46.899132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r otto_exploded_dataset.zip /kaggle/working/otto_exploded_dataset","metadata":{"execution":{"iopub.status.busy":"2023-12-12T08:49:33.373829Z","iopub.execute_input":"2023-12-12T08:49:33.374323Z","iopub.status.idle":"2023-12-12T08:49:50.474004Z","shell.execute_reply.started":"2023-12-12T08:49:33.374284Z","shell.execute_reply":"2023-12-12T08:49:50.472440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}