{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":210940932,"sourceType":"kernelVersion"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport gc\nfrom matplotlib import pyplot as plt\nimport matplotlib.cm as cm\nfrom sklearn.model_selection import StratifiedGroupKFold","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-03T06:24:22.571007Z","iopub.execute_input":"2024-12-03T06:24:22.572446Z","iopub.status.idle":"2024-12-03T06:24:26.010549Z","shell.execute_reply.started":"2024-12-03T06:24:22.572382Z","shell.execute_reply":"2024-12-03T06:24:26.009351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CONFIG:\n    target_col = \"responder_6\"\n    lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_rename = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1\" for idx in range(9)}\n    valid_ratio = 0.1\n    start_dt = 1100","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T06:24:26.012415Z","iopub.execute_input":"2024-12-03T06:24:26.012910Z","iopub.status.idle":"2024-12-03T06:24:26.019652Z","shell.execute_reply.started":"2024-12-03T06:24:26.012860Z","shell.execute_reply":"2024-12-03T06:24:26.018363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use last 2 parquets\ntrain = pl.scan_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n).select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),\n).filter(\n    pl.col(\"date_id\").gt(CONFIG.start_dt)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T06:24:26.020820Z","iopub.execute_input":"2024-12-03T06:24:26.021168Z","iopub.status.idle":"2024-12-03T06:24:26.079155Z","shell.execute_reply.started":"2024-12-03T06:24:26.021137Z","shell.execute_reply":"2024-12-03T06:24:26.077783Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags = train.select(pl.col(CONFIG.lag_cols_original))\nlags = lags.rename(CONFIG.lag_cols_rename)\nlags = lags.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )\nlags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()  # pick up last record of previous date\nlags","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T06:24:26.081169Z","iopub.execute_input":"2024-12-03T06:24:26.081530Z","iopub.status.idle":"2024-12-03T06:24:26.517351Z","shell.execute_reply.started":"2024-12-03T06:24:26.081495Z","shell.execute_reply":"2024-12-03T06:24:26.516372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.join(lags, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\ntrain","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T06:24:26.518757Z","iopub.execute_input":"2024-12-03T06:24:26.519196Z","iopub.status.idle":"2024-12-03T06:24:26.555268Z","shell.execute_reply.started":"2024-12-03T06:24:26.519151Z","shell.execute_reply":"2024-12-03T06:24:26.554190Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Combine Features 73-78\ntrain = train.with_columns(\n    feature_73_74= (pl.col('feature_73') + pl.col('feature_74')) * 0.5,\n    feature_75_76= (pl.col('feature_75') + pl.col('feature_76')) * 0.5,\n    feature_77_78= (pl.col('feature_77') + pl.col('feature_78')) * 0.5,\n).drop([f'feature_{i}' for i in range(73, 79)])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T06:24:28.215928Z","iopub.execute_input":"2024-12-03T06:24:28.216301Z","iopub.status.idle":"2024-12-03T06:24:28.222845Z","shell.execute_reply.started":"2024-12-03T06:24:28.216270Z","shell.execute_reply":"2024-12-03T06:24:28.221667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature08_mean_by_time = train.select(['time_id', 'feature_08']).group_by('time_id').mean().sort('time_id').collect()\nfeature08_fill_dict = feature08_mean_by_time.to_pandas().set_index('time_id').to_dict()['feature_08']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T06:24:33.608357Z","iopub.execute_input":"2024-12-03T06:24:33.609116Z","iopub.status.idle":"2024-12-03T06:24:37.257005Z","shell.execute_reply.started":"2024-12-03T06:24:33.609075Z","shell.execute_reply":"2024-12-03T06:24:37.256075Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill Null Feature_08 by time_id \ntrain = train.with_columns(\n    pl.col(\"time_id\").map_elements(lambda x: feature08_fill_dict.get(x, None)).alias(\"feature_08_null_fill\")\n).with_columns(\n    pl.when(pl.col(\"feature_08\").is_null())\n    .then(pl.col(\"feature_08_null_fill\"))\n    .otherwise(pl.col(\"feature_08\"))\n    .alias(\"feature_08\")\n).drop('feature_08_null_fill')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T06:25:39.152564Z","iopub.execute_input":"2024-12-03T06:25:39.152987Z","iopub.status.idle":"2024-12-03T06:25:39.162929Z","shell.execute_reply.started":"2024-12-03T06:25:39.152948Z","shell.execute_reply":"2024-12-03T06:25:39.162048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"time_based_feat_cols = ['feature_39', 'feature_42', 'feature_53', 'feature_50', 'feature_15', 'feature_44', 'feature_41', 'feature_52', 'feature_55',]\nprefill_dict = {\n}\nfor col in time_based_feat_cols:\n    prefill_dict[col] = train.select(['time_id',col]).group_by('time_id').mean().drop_nulls().sort('time_id').select(col).head(1).collect()[col][0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T06:25:44.672302Z","iopub.execute_input":"2024-12-03T06:25:44.673282Z","iopub.status.idle":"2024-12-03T06:26:07.853138Z","shell.execute_reply.started":"2024-12-03T06:25:44.673243Z","shell.execute_reply":"2024-12-03T06:26:07.851949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prefill_dict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T06:26:07.854744Z","iopub.execute_input":"2024-12-03T06:26:07.855090Z","iopub.status.idle":"2024-12-03T06:26:07.861476Z","shell.execute_reply.started":"2024-12-03T06:26:07.855058Z","shell.execute_reply":"2024-12-03T06:26:07.860528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in time_based_feat_cols:\n    train = train.with_columns(\n        pl.when(pl.col(col).is_null()).then(prefill_dict[col]).otherwise(pl.col(col)).alias(col)\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T16:21:04.155722Z","iopub.execute_input":"2024-12-01T16:21:04.157181Z","iopub.status.idle":"2024-12-01T16:21:04.163127Z","shell.execute_reply.started":"2024-12-01T16:21:04.157136Z","shell.execute_reply":"2024-12-01T16:21:04.161975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len_train   = train.select(pl.col(\"date_id\")).collect().shape[0]\nvalid_records = int(len_train * CONFIG.valid_ratio)\nlen_ofl_mdl = len_train - valid_records\nlast_tr_dt  = train.select(pl.col(\"date_id\")).collect().row(len_ofl_mdl)[0]\n\nprint(f\"\\n len_train = {len_train}\")\nprint(f\"\\n len_ofl_mdl = {len_ofl_mdl}\")\nprint(f\"\\n---> Last offline train date = {last_tr_dt}\\n\")\n\ntraining_data = train.filter(pl.col(\"date_id\").le(last_tr_dt))\nvalidation_data   = train.filter(pl.col(\"date_id\").gt(last_tr_dt))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T16:21:04.164713Z","iopub.execute_input":"2024-12-01T16:21:04.165772Z","iopub.status.idle":"2024-12-01T16:21:07.214338Z","shell.execute_reply.started":"2024-12-01T16:21:04.165719Z","shell.execute_reply":"2024-12-01T16:21:07.212956Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T16:21:07.215745Z","iopub.execute_input":"2024-12-01T16:21:07.216264Z","iopub.status.idle":"2024-12-01T16:21:07.253230Z","shell.execute_reply.started":"2024-12-01T16:21:07.216227Z","shell.execute_reply":"2024-12-01T16:21:07.252012Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"training_data.collect().\\\nwrite_parquet(\n    f\"training.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T16:21:07.254493Z","iopub.execute_input":"2024-12-01T16:21:07.254819Z","iopub.status.idle":"2024-12-01T16:22:33.169959Z","shell.execute_reply.started":"2024-12-01T16:21:07.254784Z","shell.execute_reply":"2024-12-01T16:22:33.169017Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data.collect().\\\nwrite_parquet(\n    \"validation.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T16:22:33.173653Z","iopub.execute_input":"2024-12-01T16:22:33.174387Z","iopub.status.idle":"2024-12-01T16:23:26.618511Z","shell.execute_reply.started":"2024-12-01T16:22:33.174350Z","shell.execute_reply":"2024-12-01T16:23:26.617234Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}