{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport gc\nfrom matplotlib import pyplot as plt\nimport matplotlib.cm as cm\nfrom sklearn.model_selection import StratifiedGroupKFold","metadata":{"execution":{"iopub.status.busy":"2024-12-28T06:38:16.798371Z","iopub.execute_input":"2024-12-28T06:38:16.798816Z","iopub.status.idle":"2024-12-28T06:38:16.804326Z","shell.execute_reply.started":"2024-12-28T06:38:16.798792Z","shell.execute_reply":"2024-12-28T06:38:16.802938Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Configurations","metadata":{}},{"cell_type":"code","source":"#定义了一个类\nclass CONFIG:\n    target_col = \"responder_6\"   #指明目标\n    #指明列表--两个list拼接起来的\n    lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_rename = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1\" for idx in range(9)}\n    #线下验证集的验证比例\n    valid_ratio = 0.1\n    start_dt = 1500\n    #1100天需要很多的优化和处理","metadata":{"execution":{"iopub.status.busy":"2024-12-28T06:38:16.806157Z","iopub.execute_input":"2024-12-28T06:38:16.806502Z","iopub.status.idle":"2024-12-28T06:38:16.833455Z","shell.execute_reply.started":"2024-12-28T06:38:16.806449Z","shell.execute_reply":"2024-12-28T06:38:16.832004Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load training data","metadata":{}},{"cell_type":"code","source":"train = pl.read_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n).select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),\n).with_columns(\n    (pl.col(CONFIG.target_col)*2).cast(pl.Int32).alias(\"label\"),\n).filter(\n    pl.col(\"date_id\").gt(CONFIG.start_dt)\n)\n#对文件进行处理","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:38:16.835212Z","iopub.execute_input":"2024-12-28T06:38:16.835654Z","iopub.status.idle":"2024-12-28T06:39:18.350210Z","shell.execute_reply.started":"2024-12-28T06:38:16.835618Z","shell.execute_reply":"2024-12-28T06:39:18.348967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:39:18.353934Z","iopub.execute_input":"2024-12-28T06:39:18.354254Z","iopub.status.idle":"2024-12-28T06:39:18.379099Z","shell.execute_reply.started":"2024-12-28T06:39:18.354227Z","shell.execute_reply":"2024-12-28T06:39:18.378114Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags = train.select(pl.col(CONFIG.lag_cols_original))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:39:18.380201Z","iopub.execute_input":"2024-12-28T06:39:18.380451Z","iopub.status.idle":"2024-12-28T06:39:18.413826Z","shell.execute_reply.started":"2024-12-28T06:39:18.380424Z","shell.execute_reply":"2024-12-28T06:39:18.412875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:39:18.415003Z","iopub.execute_input":"2024-12-28T06:39:18.415324Z","iopub.status.idle":"2024-12-28T06:39:18.430297Z","shell.execute_reply.started":"2024-12-28T06:39:18.415296Z","shell.execute_reply":"2024-12-28T06:39:18.429427Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags = lags.rename(CONFIG.lag_cols_rename)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:39:18.431271Z","iopub.execute_input":"2024-12-28T06:39:18.431520Z","iopub.status.idle":"2024-12-28T06:39:18.451875Z","shell.execute_reply.started":"2024-12-28T06:39:18.431496Z","shell.execute_reply":"2024-12-28T06:39:18.450922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags = lags.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:39:18.452891Z","iopub.execute_input":"2024-12-28T06:39:18.453113Z","iopub.status.idle":"2024-12-28T06:39:18.479108Z","shell.execute_reply.started":"2024-12-28T06:39:18.453089Z","shell.execute_reply":"2024-12-28T06:39:18.478185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()  # pick up last record of previous date","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:39:18.482771Z","iopub.execute_input":"2024-12-28T06:39:18.483629Z","iopub.status.idle":"2024-12-28T06:39:18.794961Z","shell.execute_reply.started":"2024-12-28T06:39:18.483576Z","shell.execute_reply":"2024-12-28T06:39:18.792976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.join(lags, on=[\"date_id\", \"symbol_id\"],  how=\"left\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:39:18.795843Z","iopub.execute_input":"2024-12-28T06:39:18.796138Z","iopub.status.idle":"2024-12-28T06:39:19.589322Z","shell.execute_reply.started":"2024-12-28T06:39:18.796109Z","shell.execute_reply":"2024-12-28T06:39:19.585229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:39:19.590566Z","iopub.execute_input":"2024-12-28T06:39:19.594423Z","iopub.status.idle":"2024-12-28T06:39:19.627743Z","shell.execute_reply.started":"2024-12-28T06:39:19.594384Z","shell.execute_reply":"2024-12-28T06:39:19.626435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use last 2 parquets\ntrain = pl. read_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n).select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),\n).with_columns(\n    (pl.col(CONFIG.target_col)*2).cast(pl.Int32).alias(\"label\"),\n).filter(\n    pl.col(\"date_id\").gt(CONFIG.start_dt)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:39:19.628601Z","iopub.execute_input":"2024-12-28T06:39:19.628835Z","iopub.status.idle":"2024-12-28T06:40:20.317810Z","shell.execute_reply.started":"2024-12-28T06:39:19.628812Z","shell.execute_reply":"2024-12-28T06:40:20.316350Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use last 2 parquets\ntrain = pl. scan_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n).select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),\n).with_columns(\n    (pl.col(CONFIG.target_col)*2).cast(pl.Int32).alias(\"label\"),\n).filter(\n    pl.col(\"date_id\").gt(CONFIG.start_dt)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:40:20.321917Z","iopub.execute_input":"2024-12-28T06:40:20.322283Z","iopub.status.idle":"2024-12-28T06:40:20.397169Z","shell.execute_reply.started":"2024-12-28T06:40:20.322257Z","shell.execute_reply":"2024-12-28T06:40:20.396153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create Lags data from training data","metadata":{}},{"cell_type":"code","source":"lags = train.select(pl.col(CONFIG.lag_cols_original))\nlags = lags.rename(CONFIG.lag_cols_rename)\nlags = lags.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )\nlags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()  # pick up last record of previous date\nlags","metadata":{"execution":{"iopub.status.busy":"2024-12-28T06:40:20.398524Z","iopub.execute_input":"2024-12-28T06:40:20.398837Z","iopub.status.idle":"2024-12-28T06:40:21.138280Z","shell.execute_reply.started":"2024-12-28T06:40:20.398808Z","shell.execute_reply":"2024-12-28T06:40:21.137532Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Merge training data and lags data","metadata":{}},{"cell_type":"code","source":"train = train.join(lags, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\ntrain","metadata":{"execution":{"iopub.status.busy":"2024-12-28T06:40:21.139079Z","iopub.execute_input":"2024-12-28T06:40:21.139303Z","iopub.status.idle":"2024-12-28T06:40:21.189089Z","shell.execute_reply.started":"2024-12-28T06:40:21.139279Z","shell.execute_reply":"2024-12-28T06:40:21.185724Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Split training data and validation data","metadata":{}},{"cell_type":"code","source":"len_train   = train.select(pl.col(\"date_id\")).collect().shape[0]\nvalid_records = int(len_train * CONFIG.valid_ratio)\nlen_ofl_mdl = len_train - valid_records\nlast_tr_dt  = train.select(pl.col(\"date_id\")).collect().row(len_ofl_mdl)[0]\n\nprint(f\"\\n len_train = {len_train}\")\nprint(f\"\\n len_ofl_mdl = {len_ofl_mdl}\")\nprint(f\"\\n---> Last offline train date = {last_tr_dt}\\n\")\n\ntraining_data = train.filter(pl.col(\"date_id\").le(last_tr_dt))\nvalidation_data   = train.filter(pl.col(\"date_id\").gt(last_tr_dt))","metadata":{"execution":{"iopub.status.busy":"2024-12-28T06:40:21.190104Z","iopub.execute_input":"2024-12-28T06:40:21.190366Z","iopub.status.idle":"2024-12-28T06:40:22.955802Z","shell.execute_reply.started":"2024-12-28T06:40:21.190341Z","shell.execute_reply":"2024-12-28T06:40:22.954285Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data","metadata":{"execution":{"iopub.status.busy":"2024-12-28T06:40:22.956900Z","iopub.execute_input":"2024-12-28T06:40:22.957184Z","iopub.status.idle":"2024-12-28T06:40:22.983739Z","shell.execute_reply.started":"2024-12-28T06:40:22.957158Z","shell.execute_reply":"2024-12-28T06:40:22.982613Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Save data as parquets","metadata":{}},{"cell_type":"code","source":"training_data.collect().\\\nwrite_parquet(\n    f\"training.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:40:22.984818Z","iopub.execute_input":"2024-12-28T06:40:22.986381Z","iopub.status.idle":"2024-12-28T06:41:40.437542Z","shell.execute_reply.started":"2024-12-28T06:40:22.986335Z","shell.execute_reply":"2024-12-28T06:41:40.436545Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data.collect().\\\nwrite_parquet(\n    \"validation.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T06:41:40.438595Z","iopub.execute_input":"2024-12-28T06:41:40.438866Z","iopub.status.idle":"2024-12-28T06:42:50.263406Z","shell.execute_reply.started":"2024-12-28T06:41:40.438839Z","shell.execute_reply":"2024-12-28T06:42:50.262534Z"}},"outputs":[],"execution_count":null}]}