{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9849268,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **FOREWORD**\n\n\nIn this kernel, we prepare the train dataset for the EDA kernel up next. <br>\nWe load the train set and partition by date-id. From my preliminary work in this competition, I believe that this is a good strategy to store and use table chunks <br>","metadata":{}},{"cell_type":"code","source":"%%time \n\nimport polars as pl, pandas as pd, numpy as np\nfrom gc import collect","metadata":{"execution":{"iopub.status.busy":"2024-10-15T19:22:56.019277Z","iopub.execute_input":"2024-10-15T19:22:56.019724Z","iopub.status.idle":"2024-10-15T19:22:57.623565Z","shell.execute_reply.started":"2024-10-15T19:22:56.019678Z","shell.execute_reply":"2024-10-15T19:22:57.622345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **DATA LOADS**","metadata":{}},{"cell_type":"code","source":"%%time \n\ntrain = pl.scan_parquet(f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\")\ntrain.collect().write_parquet(\"XYtrain.parquet\", partition_by = \"date_id\")","metadata":{"execution":{"iopub.status.busy":"2024-10-15T19:27:10.506707Z","iopub.execute_input":"2024-10-15T19:27:10.507482Z","iopub.status.idle":"2024-10-15T19:29:33.655475Z","shell.execute_reply.started":"2024-10-15T19:27:10.507431Z","shell.execute_reply":"2024-10-15T19:29:33.653980Z"},"trusted":true},"execution_count":null,"outputs":[]}]}