{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport gc\nfrom matplotlib import pyplot as plt\nimport matplotlib.cm as cm\nfrom sklearn.model_selection import StratifiedGroupKFold","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:23:28.475045Z","iopub.execute_input":"2024-12-26T11:23:28.475461Z","iopub.status.idle":"2024-12-26T11:23:29.941637Z","shell.execute_reply.started":"2024-12-26T11:23:28.475415Z","shell.execute_reply":"2024-12-26T11:23:29.940604Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Configurations","metadata":{}},{"cell_type":"code","source":"class CONFIG:\n    target_col = \"responder_6\"\n    lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_rename = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1\" for idx in range(9)}\n    valid_ratio = 0.1\n    start_dt = 1500","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:23:29.943821Z","iopub.execute_input":"2024-12-26T11:23:29.944447Z","iopub.status.idle":"2024-12-26T11:23:29.951945Z","shell.execute_reply.started":"2024-12-26T11:23:29.944386Z","shell.execute_reply":"2024-12-26T11:23:29.950754Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load training data","metadata":{}},{"cell_type":"code","source":"# Use last 2 parquets\ntrain = pl.read_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n).select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),\n).with_columns(\n    (pl.col(CONFIG.target_col)*2).cast(pl.Int32).alias(\"label\"),\n).filter(\n    pl.col(\"date_id\").gt(CONFIG.start_dt)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:23:29.953143Z","iopub.execute_input":"2024-12-26T11:23:29.953504Z","iopub.status.idle":"2024-12-26T11:24:32.117598Z","shell.execute_reply.started":"2024-12-26T11:23:29.953471Z","shell.execute_reply":"2024-12-26T11:24:32.115533Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lag_cols_original = [\"date_id\", \"time_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n\nlags = train.select(pl.col(lag_cols_original))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:25:42.232020Z","iopub.execute_input":"2024-12-26T11:25:42.232453Z","iopub.status.idle":"2024-12-26T11:25:42.243454Z","shell.execute_reply.started":"2024-12-26T11:25:42.232418Z","shell.execute_reply":"2024-12-26T11:25:42.241802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags = lags.with_columns(date_id = pl.col(\"date_id\") + 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:26:18.054684Z","iopub.execute_input":"2024-12-26T11:26:18.055133Z","iopub.status.idle":"2024-12-26T11:26:18.073301Z","shell.execute_reply.started":"2024-12-26T11:26:18.055060Z","shell.execute_reply":"2024-12-26T11:26:18.072173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"date_ids = lags.select(\"date_id\").unique().to_series()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:26:38.773921Z","iopub.execute_input":"2024-12-26T11:26:38.774340Z","iopub.status.idle":"2024-12-26T11:26:38.858697Z","shell.execute_reply.started":"2024-12-26T11:26:38.774305Z","shell.execute_reply":"2024-12-26T11:26:38.857787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_agg_list(day, columns):\n    agg_mean_list = [pl.col(c).mean().name.suffix(f\"_mean_{day}d\") for c in columns]\n    agg_std_list = [pl.col(c).std().name.suffix(f\"_std_{day}d\") for c in columns]\n    agg_max_list = [pl.col(c).max().name.suffix(f\"_max_{day}d\") for c in columns]\n    agg_last_list = [pl.col(c).last().name.suffix(f\"_last_{day}d\") for c in columns]\n    agg_list = agg_mean_list + agg_std_list + agg_max_list + agg_last_list\n    return agg_list\n    \nlag_target_cols_name = [f\"responder_{idx}\" for idx in range(9)]\nagg_list = create_agg_list(1, lag_target_cols_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:28:08.074947Z","iopub.execute_input":"2024-12-26T11:28:08.075386Z","iopub.status.idle":"2024-12-26T11:28:08.086154Z","shell.execute_reply.started":"2024-12-26T11:28:08.075348Z","shell.execute_reply":"2024-12-26T11:28:08.084577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(agg_list)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:28:26.403453Z","iopub.execute_input":"2024-12-26T11:28:26.403856Z","iopub.status.idle":"2024-12-26T11:28:26.411206Z","shell.execute_reply.started":"2024-12-26T11:28:26.403823Z","shell.execute_reply":"2024-12-26T11:28:26.409958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\nresult = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:42:05.214374Z","iopub.execute_input":"2024-12-26T11:42:05.214821Z","iopub.status.idle":"2024-12-26T11:42:05.220943Z","shell.execute_reply.started":"2024-12-26T11:42:05.214783Z","shell.execute_reply":"2024-12-26T11:42:05.219420Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"    for date_id in tqdm(date_ids, total=len(date_ids)):\n        try:\n            lags_ = lags.filter((pl.col(\"date_id\") == date_id - 1+1))\n            lags_ = lags_.with_columns(date_id=date_id)\n            lags_ = lags_.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).agg(agg_list)\n            result.append(lags_)\n        except:\n            continue","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:42:05.564655Z","iopub.execute_input":"2024-12-26T11:42:05.565094Z","iopub.status.idle":"2024-12-26T11:42:06.686770Z","shell.execute_reply.started":"2024-12-26T11:42:05.565034Z","shell.execute_reply":"2024-12-26T11:42:06.685629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lag_Ndays = pl.concat(result).sort(\"date_id\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:43:17.782133Z","iopub.execute_input":"2024-12-26T11:43:17.782690Z","iopub.status.idle":"2024-12-26T11:43:17.832541Z","shell.execute_reply.started":"2024-12-26T11:43:17.782642Z","shell.execute_reply":"2024-12-26T11:43:17.831368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lag_Ndays = lag_Ndays.cast({\"date_id\": pl.Int16})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:43:25.508824Z","iopub.execute_input":"2024-12-26T11:43:25.509212Z","iopub.status.idle":"2024-12-26T11:43:25.517812Z","shell.execute_reply.started":"2024-12-26T11:43:25.509180Z","shell.execute_reply":"2024-12-26T11:43:25.516657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.join(lag_Ndays, on=[\"date_id\", \"symbol_id\"],  how=\"left\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:43:33.972584Z","iopub.execute_input":"2024-12-26T11:43:33.972983Z","iopub.status.idle":"2024-12-26T11:43:35.568946Z","shell.execute_reply.started":"2024-12-26T11:43:33.972948Z","shell.execute_reply":"2024-12-26T11:43:35.567779Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:43:39.382400Z","iopub.execute_input":"2024-12-26T11:43:39.382830Z","iopub.status.idle":"2024-12-26T11:43:39.401870Z","shell.execute_reply.started":"2024-12-26T11:43:39.382790Z","shell.execute_reply":"2024-12-26T11:43:39.400550Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.\\\nwrite_parquet(\n    f\"training.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:44:26.748424Z","iopub.execute_input":"2024-12-26T11:44:26.748806Z","iopub.status.idle":"2024-12-26T11:44:41.939523Z","shell.execute_reply.started":"2024-12-26T11:44:26.748776Z","shell.execute_reply":"2024-12-26T11:44:41.938307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}