{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is an expansion of @motono0223's excellent lag creation notebook:\nhttps://www.kaggle.com/code/motono0223/js24-preprocessing-create-lags\nWhich was very useful for me as it is the first time I've worked with parquet files and the polars package. I've extended it to include a rolling average for the features to treat the problem as a time series as well as collect the lags for every time id instead of just the closing responder values","metadata":{}},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport gc\nfrom matplotlib import pyplot as plt\nimport matplotlib.cm as cm\nfrom sklearn.model_selection import StratifiedGroupKFold","metadata":{"execution":{"iopub.status.busy":"2025-01-06T16:43:40.684392Z","iopub.execute_input":"2025-01-06T16:43:40.684765Z","iopub.status.idle":"2025-01-06T16:43:41.964800Z","shell.execute_reply.started":"2025-01-06T16:43:40.684730Z","shell.execute_reply":"2025-01-06T16:43:41.963700Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Configurations","metadata":{}},{"cell_type":"code","source":"featurenums = range(79)\nfeaturelist = []\nfor i in featurenums:\n    featurelist.append(str(f\"{i:02}\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-06T16:43:41.966382Z","iopub.execute_input":"2025-01-06T16:43:41.966833Z","iopub.status.idle":"2025-01-06T16:43:41.972042Z","shell.execute_reply.started":"2025-01-06T16:43:41.966777Z","shell.execute_reply":"2025-01-06T16:43:41.970998Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CONFIG:\n    target_col = \"responder_6\"\n    lag_cols_original = [\"date_id\", \"symbol_id\", \"time_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_renamed = [\"date_id\", \"symbol_id\", \"time_id\"] + [f\"responder_{idx}_lag_1\" for idx in range(9)]\n    feat_cols_original = [\"date_id\", \"symbol_id\", \"time_id\"] + [f\"feature_{i}\" for i in featurelist]\n    feat_cols_rename = { f\"feature_{i}\" : f\"feature_{i}_mvg_avg\" for i in featurelist}\n    valid_ratio = 0.05\n    start_dt = 1100","metadata":{"execution":{"iopub.status.busy":"2025-01-06T16:43:41.973562Z","iopub.execute_input":"2025-01-06T16:43:41.974098Z","iopub.status.idle":"2025-01-06T16:43:41.986201Z","shell.execute_reply.started":"2025-01-06T16:43:41.974050Z","shell.execute_reply":"2025-01-06T16:43:41.985225Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load training data","metadata":{}},{"cell_type":"code","source":"# Use last 2 parquets\ntrain = pl.scan_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n).select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.col('date_id').cast(pl.Int32).alias(\"date_id\"), #need to recast to group for rolling avg\n    pl.col('symbol_id').cast(pl.Int32).alias(\"symbol_id\"), #need to recast to group for rolling avg\n    pl.exclude('date_id', 'symbol_id'),\n).with_columns(\n    (pl.col(CONFIG.target_col)*2).cast(pl.Int32).alias(\"label\"),\n).filter(\n    pl.col(\"date_id\").gt(CONFIG.start_dt)\n)","metadata":{"execution":{"iopub.status.busy":"2025-01-06T16:43:41.988252Z","iopub.execute_input":"2025-01-06T16:43:41.988561Z","iopub.status.idle":"2025-01-06T16:43:42.028285Z","shell.execute_reply.started":"2025-01-06T16:43:41.988532Z","shell.execute_reply":"2025-01-06T16:43:42.027251Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create Lags data from training data","metadata":{}},{"cell_type":"code","source":"lags = train.select(pl.col(CONFIG.lag_cols_original))\nresponders = [c for c in lags.collect_schema().names() if \"responder\" in c]\n\nlags = lags.with_columns(\n    pl.col(responders).shift().over(\"symbol_id\", \"time_id\").name.suffix(\"_lag_1\"), #shift each responder back a single date id for each symbol and time\n)\nlags = lags.select(pl.col(CONFIG.lag_cols_renamed))","metadata":{"execution":{"iopub.status.busy":"2025-01-06T16:43:42.029527Z","iopub.execute_input":"2025-01-06T16:43:42.030031Z","iopub.status.idle":"2025-01-06T16:43:42.148276Z","shell.execute_reply.started":"2025-01-06T16:43:42.029998Z","shell.execute_reply":"2025-01-06T16:43:42.147459Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create rolling averages","metadata":{}},{"cell_type":"code","source":"rollingdf = train.select(pl.col(CONFIG.feat_cols_original))\nrollingdf = rollingdf.rename(CONFIG.feat_cols_rename)\nrenamedfeatures = [c for c in rollingdf.collect_schema().names() if \"feature\" in c]\nrollingdf = rollingdf.sort([\"symbol_id\", \"date_id\", \"time_id\"]) \nrollingdf = rollingdf.with_columns(\n                                  pl.col(renamedfeatures).rolling_mean(window_size=1000, min_periods=0).over(\"symbol_id\"))\n# create 1000 window moving average for each symbol id","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-06T16:43:42.149739Z","iopub.execute_input":"2025-01-06T16:43:42.150192Z","iopub.status.idle":"2025-01-06T16:43:42.158173Z","shell.execute_reply.started":"2025-01-06T16:43:42.150147Z","shell.execute_reply":"2025-01-06T16:43:42.157017Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Join lags and rolling averages to training dataset","metadata":{}},{"cell_type":"code","source":"train = train.join(rollingdf, on=[\"date_id\", \"symbol_id\", \"time_id\"], how=\"left\")\ntrain = train.join(lags, on=[\"date_id\", \"symbol_id\", \"time_id\"],  how=\"left\")\n","metadata":{"execution":{"iopub.status.busy":"2025-01-06T16:43:42.159601Z","iopub.execute_input":"2025-01-06T16:43:42.160227Z","iopub.status.idle":"2025-01-06T16:43:42.179396Z","shell.execute_reply.started":"2025-01-06T16:43:42.160179Z","shell.execute_reply":"2025-01-06T16:43:42.178268Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Split training data and validation data","metadata":{}},{"cell_type":"code","source":"len_train   = train.select(pl.col(\"date_id\")).collect().shape[0]\nvalid_records = int(len_train * CONFIG.valid_ratio)\nlen_ofl_mdl = len_train - valid_records\nlast_tr_dt  = train.select(pl.col(\"date_id\")).collect().row(len_ofl_mdl)[0]\n\nprint(f\"\\n len_train = {len_train}\")\nprint(f\"\\n len_ofl_mdl = {len_ofl_mdl}\")\nprint(f\"\\n---> Last offline train date = {last_tr_dt}\\n\")\n\ntraining_data = train.filter(pl.col(\"date_id\").le(last_tr_dt))\nvalidation_data   = train.filter(pl.col(\"date_id\").gt(last_tr_dt))","metadata":{"execution":{"iopub.status.busy":"2025-01-06T16:43:42.180681Z","iopub.execute_input":"2025-01-06T16:43:42.181107Z","iopub.status.idle":"2025-01-06T16:44:22.351317Z","shell.execute_reply.started":"2025-01-06T16:43:42.181073Z","shell.execute_reply":"2025-01-06T16:44:22.350298Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data","metadata":{"execution":{"iopub.status.busy":"2025-01-06T16:44:22.352852Z","iopub.execute_input":"2025-01-06T16:44:22.353264Z","iopub.status.idle":"2025-01-06T16:44:22.550109Z","shell.execute_reply.started":"2025-01-06T16:44:22.353217Z","shell.execute_reply":"2025-01-06T16:44:22.549018Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Save data as parquets","metadata":{}},{"cell_type":"code","source":"training_data.collect().\\\nwrite_parquet(\n    f\"training.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-06T16:44:22.553532Z","iopub.execute_input":"2025-01-06T16:44:22.554065Z","iopub.status.idle":"2025-01-06T16:47:56.570783Z","shell.execute_reply.started":"2025-01-06T16:44:22.554029Z","shell.execute_reply":"2025-01-06T16:47:56.569833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data.collect().\\\nwrite_parquet(\n    \"validation.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-06T16:47:56.572120Z","iopub.execute_input":"2025-01-06T16:47:56.572549Z","iopub.status.idle":"2025-01-06T16:50:17.599035Z","shell.execute_reply.started":"2025-01-06T16:47:56.572504Z","shell.execute_reply":"2025-01-06T16:50:17.598047Z"}},"outputs":[],"execution_count":null}]}