{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport gc\nimport joblib\nfrom matplotlib import pyplot as plt\nimport matplotlib.cm as cm\nfrom sklearn.model_selection import StratifiedGroupKFold","metadata":{"execution":{"iopub.status.busy":"2025-01-06T13:54:22.742499Z","iopub.execute_input":"2025-01-06T13:54:22.742942Z","iopub.status.idle":"2025-01-06T13:54:22.749557Z","shell.execute_reply.started":"2025-01-06T13:54:22.742903Z","shell.execute_reply":"2025-01-06T13:54:22.748054Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Configurations","metadata":{}},{"cell_type":"code","source":"class CONFIG:\n    target_col = \"responder_6\"\n    lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_rename = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1\" for idx in range(9)}\n    valid_ratio = 0.05\n    start_dt = 1100","metadata":{"execution":{"iopub.status.busy":"2025-01-06T13:53:32.766806Z","iopub.execute_input":"2025-01-06T13:53:32.767385Z","iopub.status.idle":"2025-01-06T13:53:32.776267Z","shell.execute_reply.started":"2025-01-06T13:53:32.767334Z","shell.execute_reply":"2025-01-06T13:53:32.774316Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load training data","metadata":{}},{"cell_type":"code","source":"# Use last 2 parquets\ntrain = pl.scan_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n).select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),\n).with_columns(\n    (pl.col(CONFIG.target_col)*2).cast(pl.Int32).alias(\"label\"),\n).filter(\n    pl.col(\"date_id\").gt(CONFIG.start_dt)\n)","metadata":{"execution":{"iopub.status.busy":"2025-01-06T13:53:32.778601Z","iopub.execute_input":"2025-01-06T13:53:32.779325Z","iopub.status.idle":"2025-01-06T13:53:32.835003Z","shell.execute_reply.started":"2025-01-06T13:53:32.779246Z","shell.execute_reply":"2025-01-06T13:53:32.833306Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create Lags data from training data","metadata":{}},{"cell_type":"code","source":"lags = train.select(pl.col(CONFIG.lag_cols_original))\nlags = lags.rename(CONFIG.lag_cols_rename)\nlags = lags.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )\nlags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()  # pick up last record of previous date\nlags","metadata":{"execution":{"iopub.status.busy":"2025-01-06T13:53:32.838892Z","iopub.execute_input":"2025-01-06T13:53:32.840187Z","iopub.status.idle":"2025-01-06T13:53:33.217163Z","shell.execute_reply.started":"2025-01-06T13:53:32.840101Z","shell.execute_reply":"2025-01-06T13:53:33.215653Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Merge training data and lags data","metadata":{}},{"cell_type":"code","source":"train = train.join(lags, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\ntrain","metadata":{"execution":{"iopub.status.busy":"2025-01-06T13:53:33.219043Z","iopub.execute_input":"2025-01-06T13:53:33.219525Z","iopub.status.idle":"2025-01-06T13:53:33.256989Z","shell.execute_reply.started":"2025-01-06T13:53:33.219477Z","shell.execute_reply":"2025-01-06T13:53:33.255883Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Split training data and validation data","metadata":{}},{"cell_type":"code","source":"len_train   = train.select(pl.col(\"date_id\")).collect().shape[0]\nvalid_records = int(len_train * CONFIG.valid_ratio)\nlen_ofl_mdl = len_train - valid_records\nlast_tr_dt  = train.select(pl.col(\"date_id\")).collect().row(len_ofl_mdl)[0]\n\nprint(f\"\\n len_train = {len_train}\")\nprint(f\"\\n len_ofl_mdl = {len_ofl_mdl}\")\nprint(f\"\\n---> Last offline train date = {last_tr_dt}\\n\")\n\ntraining_data = train.filter(pl.col(\"date_id\").le(last_tr_dt))\nvalidation_data   = train.filter(pl.col(\"date_id\").gt(last_tr_dt))","metadata":{"execution":{"iopub.status.busy":"2025-01-06T13:53:33.258442Z","iopub.execute_input":"2025-01-06T13:53:33.258927Z","iopub.status.idle":"2025-01-06T13:53:36.948107Z","shell.execute_reply.started":"2025-01-06T13:53:33.258878Z","shell.execute_reply":"2025-01-06T13:53:36.946991Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data","metadata":{"execution":{"iopub.status.busy":"2025-01-06T13:53:36.949488Z","iopub.execute_input":"2025-01-06T13:53:36.950451Z","iopub.status.idle":"2025-01-06T13:53:36.979338Z","shell.execute_reply.started":"2025-01-06T13:53:36.950412Z","shell.execute_reply":"2025-01-06T13:53:36.978165Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Save data as parquets","metadata":{}},{"cell_type":"code","source":"# Compute mean and standard deviation for each feature\ncols_features_resp_lags = [f'feature_0{i}' for i in range(9)] + [f'feature_{j}' for j in range(12, 79, 1)]+[f'responder_{i}_lag_1' for i in range(9)]\n\nfeature_stats_train = training_data.select(\n    [pl.col(f).mean().alias(f\"{f}_mean\") for f in cols_features_resp_lags] + \n    [pl.col(f).std().alias(f\"{f}_std\") for f in cols_features_resp_lags]\n).collect()\n\ndict_mean = {\n    col.replace(\"_mean\", \"\"): val\n    for col, val in zip(feature_stats_train.columns[:len(cols_features_resp_lags)], feature_stats_train.row(0)[:len(cols_features_resp_lags)])\n}\ndict_std = {\n    col.replace(\"_std\", \"\"): val\n    for col, val in zip(feature_stats_train.columns[len(cols_features_resp_lags):], feature_stats_train.row(0)[len(cols_features_resp_lags):])\n}\n\nstats_dict_train = {\n    \"mean\": dict_mean,\n    \"std\": dict_std\n}\n\njoblib.dump(stats_dict_train,'training_stats.pk1')\n\nprint('training stats saved')\n\nfeature_stats_val = validation_data.select(\n    [pl.col(f).mean().alias(f\"{f}_mean\") for f in cols_features_resp_lags] + \n    [pl.col(f).std().alias(f\"{f}_std\") for f in cols_features_resp_lags]\n).collect()\n\nstats_dict_val = {\n    \"mean\": {col.replace(\"_mean\", \"\"): val for col, val in zip(feature_stats_val.columns[:len(cols_features_resp_lags)], feature_stats_val.row(0)[:len(cols_features_resp_lags)])\n    },\n    \"std\": {\n    col.replace(\"_std\", \"\"): val for col, val in zip(feature_stats_val.columns[len(cols_features_resp_lags):], feature_stats_val.row(0)[len(cols_features_resp_lags):])\n    }\n}\n\njoblib.dump(stats_dict_val,'validation_stats.pk1')\nprint('validation stats saved')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-06T13:54:27.686643Z","iopub.execute_input":"2025-01-06T13:54:27.687069Z","iopub.status.idle":"2025-01-06T13:54:47.595739Z","shell.execute_reply.started":"2025-01-06T13:54:27.687032Z","shell.execute_reply":"2025-01-06T13:54:47.594270Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"training_data.collect().\\\nwrite_parquet(\n    f\"training.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-06T13:54:06.095672Z","iopub.status.idle":"2025-01-06T13:54:06.096165Z","shell.execute_reply.started":"2025-01-06T13:54:06.095918Z","shell.execute_reply":"2025-01-06T13:54:06.095940Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data.collect().\\\nwrite_parquet(\n    \"validation.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-06T13:54:06.097035Z","iopub.status.idle":"2025-01-06T13:54:06.102667Z","shell.execute_reply.started":"2025-01-06T13:54:06.102308Z","shell.execute_reply":"2025-01-06T13:54:06.102349Z"}},"outputs":[],"execution_count":null}]}