{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport gc\nfrom matplotlib import pyplot as plt\nimport matplotlib.cm as cm\nfrom sklearn.model_selection import StratifiedGroupKFold","metadata":{"execution":{"iopub.status.busy":"2024-10-29T15:17:39.693959Z","iopub.execute_input":"2024-10-29T15:17:39.694384Z","iopub.status.idle":"2024-10-29T15:17:42.539733Z","shell.execute_reply.started":"2024-10-29T15:17:39.694341Z","shell.execute_reply":"2024-10-29T15:17:42.538596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configurations","metadata":{}},{"cell_type":"code","source":"class CONFIG:\n    target_col = \"responder_6\"\n    lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_rename = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1\" for idx in range(9)}\n    valid_ratio = 0.05\n    start_dt = 1100","metadata":{"execution":{"iopub.status.busy":"2024-10-29T15:17:42.542008Z","iopub.execute_input":"2024-10-29T15:17:42.542681Z","iopub.status.idle":"2024-10-29T15:17:42.551737Z","shell.execute_reply.started":"2024-10-29T15:17:42.542630Z","shell.execute_reply":"2024-10-29T15:17:42.549285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_col = \"responder_6\"\nfeature_cols = [\"symbol_id\", \"time_id\"] \\\n    + [f\"feature_{idx:02d}\" for idx in range(79)] \\\n    + [f\"responder_{idx}_lag_1\" for idx in range(9)]","metadata":{"execution":{"iopub.status.busy":"2024-10-29T15:17:58.692649Z","iopub.execute_input":"2024-10-29T15:17:58.693114Z","iopub.status.idle":"2024-10-29T15:17:58.699136Z","shell.execute_reply.started":"2024-10-29T15:17:58.693069Z","shell.execute_reply":"2024-10-29T15:17:58.697792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load training data","metadata":{}},{"cell_type":"code","source":"# Use last 2 parquets\ntrain = pl.scan_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n).select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),\n).with_columns(\n    (pl.col(CONFIG.target_col)*2).cast(pl.Int32).alias(\"label\"),\n).filter(\n    pl.col(\"date_id\").gt(CONFIG.start_dt)\n)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T15:18:18.761577Z","iopub.execute_input":"2024-10-29T15:18:18.761978Z","iopub.status.idle":"2024-10-29T15:18:18.797742Z","shell.execute_reply.started":"2024-10-29T15:18:18.761945Z","shell.execute_reply":"2024-10-29T15:18:18.796535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Lags data from training data","metadata":{}},{"cell_type":"code","source":"lags = train.select(pl.col(CONFIG.lag_cols_original))\nlags = lags.rename(CONFIG.lag_cols_rename)\nlags = lags.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )\nlags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()  # pick up last record of previous date\nlags","metadata":{"execution":{"iopub.status.busy":"2024-10-29T15:18:19.069020Z","iopub.execute_input":"2024-10-29T15:18:19.069462Z","iopub.status.idle":"2024-10-29T15:18:19.391677Z","shell.execute_reply.started":"2024-10-29T15:18:19.069404Z","shell.execute_reply":"2024-10-29T15:18:19.390413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Merge training data and lags data","metadata":{}},{"cell_type":"code","source":"train = train.join(lags, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\ntrain","metadata":{"execution":{"iopub.status.busy":"2024-10-29T15:18:20.876964Z","iopub.execute_input":"2024-10-29T15:18:20.877358Z","iopub.status.idle":"2024-10-29T15:18:20.909008Z","shell.execute_reply.started":"2024-10-29T15:18:20.877323Z","shell.execute_reply":"2024-10-29T15:18:20.907896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split training data and validation data","metadata":{}},{"cell_type":"code","source":"X = train.select(feature_cols).collect().to_numpy()\nnp.save('X.npy', X)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T15:25:52.999480Z","iopub.execute_input":"2024-10-29T15:25:52.999974Z","iopub.status.idle":"2024-10-29T15:26:47.473223Z","shell.execute_reply.started":"2024-10-29T15:25:52.999926Z","shell.execute_reply":"2024-10-29T15:26:47.471114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train.select(target_col).collect()[target_col].to_numpy()\nnp.save('y.npy', y)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T15:31:19.125962Z","iopub.execute_input":"2024-10-29T15:31:19.129232Z","iopub.status.idle":"2024-10-29T15:31:20.867266Z","shell.execute_reply.started":"2024-10-29T15:31:19.129169Z","shell.execute_reply":"2024-10-29T15:31:20.866118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"w = train.select('weight').collect()['weight'].to_numpy()\nnp.save('w.npy', w)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T15:31:26.865278Z","iopub.execute_input":"2024-10-29T15:31:26.866631Z","iopub.status.idle":"2024-10-29T15:31:28.548481Z","shell.execute_reply.started":"2024-10-29T15:31:26.866579Z","shell.execute_reply":"2024-10-29T15:31:28.547268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len_train   = train.select(pl.col(\"date_id\")).collect().shape[0]\n# valid_records = int(len_train * CONFIG.valid_ratio)\n# len_ofl_mdl = len_train - valid_records\n# last_tr_dt  = train.select(pl.col(\"date_id\")).collect().row(len_ofl_mdl)[0]\n\n# print(f\"\\n len_train = {len_train}\")\n# print(f\"\\n len_ofl_mdl = {len_ofl_mdl}\")\n# print(f\"\\n---> Last offline train date = {last_tr_dt}\\n\")\n\n# training_data = train.filter(pl.col(\"date_id\").le(last_tr_dt))\n# validation_data   = train.filter(pl.col(\"date_id\").gt(last_tr_dt))","metadata":{"execution":{"iopub.status.busy":"2024-10-29T15:31:28.550517Z","iopub.execute_input":"2024-10-29T15:31:28.550907Z","iopub.status.idle":"2024-10-29T15:31:28.556238Z","shell.execute_reply.started":"2024-10-29T15:31:28.550871Z","shell.execute_reply":"2024-10-29T15:31:28.554649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# validation_data","metadata":{"execution":{"iopub.status.busy":"2024-10-26T05:14:06.979216Z","iopub.execute_input":"2024-10-26T05:14:06.979605Z","iopub.status.idle":"2024-10-26T05:14:07.012111Z","shell.execute_reply.started":"2024-10-26T05:14:06.979551Z","shell.execute_reply":"2024-10-26T05:14:07.010818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save data as parquets","metadata":{}},{"cell_type":"code","source":"# training_data.collect().\\\n# write_parquet(\n#     f\"training.parquet\", partition_by = \"date_id\",\n# )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# validation_data.collect().\\\n# write_parquet(\n#     \"validation.parquet\", partition_by = \"date_id\",\n# )","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}