{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --upgrade polars\nimport polars as pl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-28T05:44:05.376551Z","iopub.execute_input":"2024-12-28T05:44:05.376990Z","iopub.status.idle":"2024-12-28T05:44:16.429294Z","shell.execute_reply.started":"2024-12-28T05:44:05.376945Z","shell.execute_reply":"2024-12-28T05:44:16.427873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(pl.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T05:44:27.137255Z","iopub.execute_input":"2024-12-28T05:44:27.137663Z","iopub.status.idle":"2024-12-28T05:44:27.144398Z","shell.execute_reply.started":"2024-12-28T05:44:27.137628Z","shell.execute_reply":"2024-12-28T05:44:27.143041Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Configuration class to store constants\nclass CONFIG:\n    # Column to predict\n    target_col = \"responder_6\"\n\n    # Validation set ratio\n    valid_ratio = 0.2\n\n    # Start date to filter the data\n    start_dt = 1398","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T05:44:33.448481Z","iopub.execute_input":"2024-12-28T05:44:33.449039Z","iopub.status.idle":"2024-12-28T05:44:33.455351Z","shell.execute_reply.started":"2024-12-28T05:44:33.448998Z","shell.execute_reply":"2024-12-28T05:44:33.453661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load parquet files lazily using Polars\ntrain = pl.scan_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n).select(\n    # Add an ID column for unique indexing\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),  # Select all other columns\n).filter(\n    # Filter rows where the date is greater than CONFIG.start_dt\n    pl.col(\"date_id\").gt(CONFIG.start_dt)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T05:44:38.263891Z","iopub.execute_input":"2024-12-28T05:44:38.264311Z","iopub.status.idle":"2024-12-28T05:44:38.275279Z","shell.execute_reply.started":"2024-12-28T05:44:38.264275Z","shell.execute_reply":"2024-12-28T05:44:38.273575Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get the total number of rows\nlen_train = train.select(pl.col(\"date_id\")).collect().shape[0]\n\n# Calculate the number of validation records\nvalid_records = int(len_train * CONFIG.valid_ratio)\n\n# Calculate the length of the offline training dataset\nlen_ofl_mdl = len_train - valid_records\n\n# Calculate the last training date\nlast_tr_dt = train.select(pl.col(\"date_id\")).collect().row(len_ofl_mdl)[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T05:44:49.635590Z","iopub.execute_input":"2024-12-28T05:44:49.636260Z","iopub.status.idle":"2024-12-28T05:44:49.860286Z","shell.execute_reply.started":"2024-12-28T05:44:49.636215Z","shell.execute_reply":"2024-12-28T05:44:49.858916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Print summary of dataset splits\nprint(f\"\\n len_train = {len_train}\")  # Total number of rows\nprint(f\"\\n len_ofl_mdl = {len_ofl_mdl}\")  # Number of rows in training dataset\nprint(f\"\\n---> Last offline train date = {last_tr_dt}\\n\")  # Last training date","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T05:44:52.386302Z","iopub.execute_input":"2024-12-28T05:44:52.386789Z","iopub.status.idle":"2024-12-28T05:44:52.394985Z","shell.execute_reply.started":"2024-12-28T05:44:52.386735Z","shell.execute_reply":"2024-12-28T05:44:52.393339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Split data into training and validation sets\ntraining_data = train.filter(pl.col(\"date_id\").le(last_tr_dt))  # Training: date_id <= last_tr_dt\nvalidation_data = train.filter(pl.col(\"date_id\").gt(last_tr_dt))  # Validation: date_id > last_tr_dt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T05:44:55.684999Z","iopub.execute_input":"2024-12-28T05:44:55.685473Z","iopub.status.idle":"2024-12-28T05:44:55.690985Z","shell.execute_reply.started":"2024-12-28T05:44:55.685434Z","shell.execute_reply":"2024-12-28T05:44:55.689595Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Collect the LazyFrame into a DataFrame\ncollected_training_data = train.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T05:45:08.102402Z","iopub.execute_input":"2024-12-28T05:45:08.102942Z","iopub.status.idle":"2024-12-28T05:46:12.731494Z","shell.execute_reply.started":"2024-12-28T05:45:08.102904Z","shell.execute_reply":"2024-12-28T05:46:12.729250Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"collected_training_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T05:46:53.048904Z","iopub.execute_input":"2024-12-28T05:46:53.049478Z","iopub.status.idle":"2024-12-28T05:46:53.095822Z","shell.execute_reply.started":"2024-12-28T05:46:53.049440Z","shell.execute_reply":"2024-12-28T05:46:53.094109Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"training_data.collect().\\\nwrite_parquet(\n    \"training.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T05:47:04.171255Z","iopub.execute_input":"2024-12-28T05:47:04.171696Z","iopub.status.idle":"2024-12-28T05:48:04.664976Z","shell.execute_reply.started":"2024-12-28T05:47:04.171662Z","shell.execute_reply":"2024-12-28T05:48:04.663800Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data.collect().write_parquet(\n    \"validation.parquet\", partition_by=\"date_id\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T05:48:41.402610Z","iopub.execute_input":"2024-12-28T05:48:41.403153Z","iopub.status.idle":"2024-12-28T05:49:29.883370Z","shell.execute_reply.started":"2024-12-28T05:48:41.403113Z","shell.execute_reply":"2024-12-28T05:49:29.880824Z"}},"outputs":[],"execution_count":null}]}