{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport gc\nfrom matplotlib import pyplot as plt\nimport matplotlib.cm as cm\nfrom sklearn.model_selection import StratifiedGroupKFold","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T15:38:24.301255Z","iopub.execute_input":"2025-01-05T15:38:24.301763Z","iopub.status.idle":"2025-01-05T15:38:27.294314Z","shell.execute_reply.started":"2025-01-05T15:38:24.301711Z","shell.execute_reply":"2025-01-05T15:38:27.292954Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Configurations","metadata":{}},{"cell_type":"code","source":"class CONFIG:\n    target_col = \"responder_6\"\n    lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_rename = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1\" for idx in range(9)}\n    lag_cols_rename_max = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1_max\" for idx in range(9)}\n    lag_cols_rename_min = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1_min\" for idx in range(9)}\n    lag_cols_rename_first = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1_first\" for idx in range(9)}\n    valid_ratio = 0.20\n    start_dt = 1000","metadata":{"execution":{"iopub.status.busy":"2025-01-05T15:38:27.296968Z","iopub.execute_input":"2025-01-05T15:38:27.297531Z","iopub.status.idle":"2025-01-05T15:38:27.306612Z","shell.execute_reply.started":"2025-01-05T15:38:27.297492Z","shell.execute_reply":"2025-01-05T15:38:27.305206Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load training data","metadata":{}},{"cell_type":"code","source":"train = (\n    pl.scan_parquet(f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\")\n    .select(\n        pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n        pl.all(),\n    )\n    .with_columns(\n        (pl.col(CONFIG.target_col) * 2).cast(pl.Int32).alias(\"label\"),\n    )\n    .filter(\n        pl.col(\"date_id\").gt(CONFIG.start_dt)\n    )\n    .with_columns(\n        # 计算每行的 time_id 在其所在 date_id 分组的相对大小\n        (\n            (pl.col(\"time_id\") - pl.col(\"time_id\").over(\"date_id\").min())\n            / (pl.col(\"time_id\").over(\"date_id\").max() - pl.col(\"time_id\").over(\"date_id\").min())\n        ).alias(\"time_relative\"),\n        # 新增列：date_id 分别除以 5, 21, 63\n        (pl.col(\"date_id\") % 5).alias(\"date_div_5\"),\n        (pl.col(\"date_id\") % 21).alias(\"date_div_21\"),\n        (pl.col(\"date_id\") % 63).alias(\"date_div_63\"),\n    )\n)\ntrain","metadata":{"execution":{"iopub.status.busy":"2025-01-05T15:38:27.308323Z","iopub.execute_input":"2025-01-05T15:38:27.308750Z","iopub.status.idle":"2025-01-05T15:38:27.699212Z","shell.execute_reply.started":"2025-01-05T15:38:27.308714Z","shell.execute_reply":"2025-01-05T15:38:27.697972Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create Lags data from training data","metadata":{}},{"cell_type":"code","source":"lags = train.select(pl.col(CONFIG.lag_cols_original))\nlags = lags.rename(CONFIG.lag_cols_rename)\nlags = lags.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )\nlags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()  # pick up last record of previous date\nlags\n\nlags1 = train.select(pl.col(CONFIG.lag_cols_original))\nlags1 = lags1.rename(CONFIG.lag_cols_rename_max)\nlags1 = lags1.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )\nlags1 = lags1.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).max()  # pick up last record of previous date\nlags1\n\nlags2 = train.select(pl.col(CONFIG.lag_cols_original))\nlags2 = lags2.rename(CONFIG.lag_cols_rename_min)\nlags2 = lags2.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )\nlags2 = lags2.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).min()  # pick up last record of previous date\nlags2\n\nlags3 = train.select(pl.col(CONFIG.lag_cols_original))\nlags3 = lags3.rename(CONFIG.lag_cols_rename_first)\nlags3 = lags3.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )\nlags3 = lags3.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).first()  # pick up last record of previous date\nlags3","metadata":{"execution":{"iopub.status.busy":"2025-01-05T15:38:27.700801Z","iopub.execute_input":"2025-01-05T15:38:27.701971Z","iopub.status.idle":"2025-01-05T15:38:27.740712Z","shell.execute_reply.started":"2025-01-05T15:38:27.701915Z","shell.execute_reply":"2025-01-05T15:38:27.739437Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags = lags.join(lags1, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\nlags = lags.join(lags2, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\nlags = lags.join(lags3, on=[\"date_id\", \"symbol_id\"],  how=\"left\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T15:38:27.743469Z","iopub.execute_input":"2025-01-05T15:38:27.743808Z","iopub.status.idle":"2025-01-05T15:38:37.306894Z","shell.execute_reply.started":"2025-01-05T15:38:27.743777Z","shell.execute_reply":"2025-01-05T15:38:37.305556Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Merge training data and lags data","metadata":{}},{"cell_type":"code","source":"train = train.join(lags, on=[\"date_id\", \"symbol_id\"],  how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2025-01-05T15:38:37.308137Z","iopub.execute_input":"2025-01-05T15:38:37.308524Z","iopub.status.idle":"2025-01-05T15:38:37.314062Z","shell.execute_reply.started":"2025-01-05T15:38:37.308487Z","shell.execute_reply":"2025-01-05T15:38:37.312781Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"train_df = train.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T15:38:37.315494Z","iopub.execute_input":"2025-01-05T15:38:37.315832Z","iopub.status.idle":"2025-01-05T15:39:52.842207Z","shell.execute_reply.started":"2025-01-05T15:38:37.315797Z","shell.execute_reply":"2025-01-05T15:39:52.839282Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Split training data and validation data","metadata":{}},{"cell_type":"code","source":"len_train   = train.select(pl.col(\"date_id\")).collect().shape[0]\nvalid_records = int(len_train * CONFIG.valid_ratio)\nlen_ofl_mdl = len_train - valid_records\nlast_tr_dt  = train.select(pl.col(\"date_id\")).collect().row(len_ofl_mdl)[0]\n\nprint(f\"\\n len_train = {len_train}\")\nprint(f\"\\n len_ofl_mdl = {len_ofl_mdl}\")\nprint(f\"\\n---> Last offline train date = {last_tr_dt}\\n\")\n\ntraining_data = train.filter(pl.col(\"date_id\").le(last_tr_dt))\n#validation_data   = train.filter(pl.col(\"date_id\").gt(last_tr_dt))","metadata":{"execution":{"iopub.status.busy":"2025-01-05T15:39:52.846031Z","iopub.execute_input":"2025-01-05T15:39:52.846770Z","iopub.status.idle":"2025-01-05T15:40:03.609777Z","shell.execute_reply.started":"2025-01-05T15:39:52.846682Z","shell.execute_reply":"2025-01-05T15:40:03.608467Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Save data as parquets","metadata":{}},{"cell_type":"code","source":"training_data.collect().\\\nwrite_parquet(\n    f\"training.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T15:40:03.611125Z","iopub.execute_input":"2025-01-05T15:40:03.611526Z","execution_failed":"2025-01-05T15:41:28.943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#validation_data.collect().\\\n#write_parquet(\n#    \"validation.parquet\", partition_by = \"date_id\",\n#)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-05T15:41:28.944Z"}},"outputs":[],"execution_count":null}]}