{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Adding Lags, CV split\n\nAdapted from [motono0223](https://www.kaggle.com/code/motono0223/js24-preprocessing-create-lags)\n\nTrain from 1000 onwards with backtest split:\n* 700-825: Historical Data\n* 1000-1698: Training Data\n\nNotebooks:\n- Data Processing [here](https://www.kaggle.com/code/mizaazir/jane-street-data-pre-processing)\n- Training (Tree) [here](https://www.kaggle.com/code/mizaazir/jane-street-train-tree-model)\n- Training (Else) [here](https://www.kaggle.com/code/mizaazir/jane-street-nn-model)\n- Inference [here](https://www.kaggle.com/code/mizaazir/jane-street-inference)\n\nExperiment details [here](https://docs.google.com/spreadsheets/d/1LtcLXCBjfKMdBc0a81onSFvEVDY-KppSEw560F3sWH0/edit?usp=sharing)","metadata":{}},{"cell_type":"code","source":"import gc\nfrom tqdm import tqdm\n\n\nimport pandas as pd\nimport polars as pl\nimport numpy as np\n\nfrom matplotlib import pyplot as plt\nimport matplotlib.cm as cm\nfrom sklearn.model_selection import StratifiedGroupKFold","metadata":{"execution":{"iopub.status.busy":"2024-11-10T15:04:06.075501Z","iopub.execute_input":"2024-11-10T15:04:06.075988Z","iopub.status.idle":"2024-11-10T15:04:07.494572Z","shell.execute_reply.started":"2024-11-10T15:04:06.075927Z","shell.execute_reply":"2024-11-10T15:04:07.493555Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_corr = ['feature_16', 'feature_36', 'feature_17', 'feature_06', \n             'feature_04', 'feature_07', 'feature_15', 'feature_45', \n             'feature_56', 'feature_05', 'feature_42', 'feature_60', \n             'feature_48', 'feature_44', 'feature_19', 'feature_58', \n             'feature_66', 'feature_39', 'feature_51', 'feature_70', \n             'feature_25', 'feature_12', 'feature_34', 'feature_46', \n             'feature_41', 'feature_67', 'feature_33', 'feature_72', \n             'feature_24', 'feature_01', 'feature_62', 'feature_68', \n             'feature_64', 'feature_20', 'feature_43', 'feature_76', \n             'feature_47', 'feature_35', 'feature_75', 'feature_74', \n             'feature_31', 'feature_63', 'feature_73', 'feature_78', \n             'feature_21', 'feature_49', 'feature_32', 'feature_22', \n             'feature_50']\n\nworst_corr = ['feature_54', 'feature_18', 'feature_69', 'feature_57', \n              'feature_00', 'feature_26', 'feature_37', 'feature_28', \n              'feature_52', 'feature_29', 'feature_40', 'feature_55', \n              'feature_10', 'feature_27', 'feature_61']\n\nclass CONFIG:\n    target_col = \"responder_6\"\n    lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_rename = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1\" for idx in range(9)}\n    \n    hist_date_start = 700\n    hist_date_end = 825\n    \n    train_date_start = 1000","metadata":{"execution":{"iopub.status.busy":"2024-11-10T15:04:07.496706Z","iopub.execute_input":"2024-11-10T15:04:07.497685Z","iopub.status.idle":"2024-11-10T15:04:07.506398Z","shell.execute_reply.started":"2024-11-10T15:04:07.497626Z","shell.execute_reply":"2024-11-10T15:04:07.505321Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load training data","metadata":{}},{"cell_type":"code","source":"df = pl.scan_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n).select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),\n).filter(\n    ((pl.col('date_id') >= CONFIG.hist_date_start-2) & (pl.col('date_id') < CONFIG.hist_date_end)) | ((pl.col('date_id') >= CONFIG.train_date_start)) \n)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T15:04:07.508022Z","iopub.execute_input":"2024-11-10T15:04:07.508508Z","iopub.status.idle":"2024-11-10T15:04:07.545071Z","shell.execute_reply.started":"2024-11-10T15:04:07.508457Z","shell.execute_reply":"2024-11-10T15:04:07.544013Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Create Lags data from training data","metadata":{}},{"cell_type":"code","source":"lags = df.select(pl.col(CONFIG.lag_cols_original))\nlags = lags.rename(CONFIG.lag_cols_rename)\nlags = lags.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )\nlags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()  # pick up last record of previous date\nlags","metadata":{"execution":{"iopub.status.busy":"2024-11-10T15:04:07.547919Z","iopub.execute_input":"2024-11-10T15:04:07.548343Z","iopub.status.idle":"2024-11-10T15:04:07.864231Z","shell.execute_reply.started":"2024-11-10T15:04:07.548278Z","shell.execute_reply":"2024-11-10T15:04:07.862886Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Merge training data and lags data","metadata":{}},{"cell_type":"code","source":"df = df.join(lags, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\ndf","metadata":{"execution":{"iopub.status.busy":"2024-11-10T15:04:07.866252Z","iopub.execute_input":"2024-11-10T15:04:07.866603Z","iopub.status.idle":"2024-11-10T15:04:07.908827Z","shell.execute_reply.started":"2024-11-10T15:04:07.866565Z","shell.execute_reply":"2024-11-10T15:04:07.907749Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Add Positional Feature and Groups","metadata":{}},{"cell_type":"code","source":"df = df.with_columns(\n    (2 * np.pi * pl.col('time_id') / 967).sin().alias('sin_time_id')\n)\ndf = df.with_columns(\n    (2 * np.pi * pl.col('time_id') / 967).cos().alias('cos_time_id')\n)\ndf = df.with_columns(\n    (2 * np.pi * pl.col('time_id') / 483).sin().alias('sin_time_id_2')\n)\ndf = df.with_columns(\n    (2 * np.pi * pl.col('time_id') / 483).cos().alias('cos_time_id_2')\n)\ndf = df.with_columns(\n    ((pl.col('date_id')-1000) // 100).alias('group_id')\n)\ndf","metadata":{"execution":{"iopub.status.busy":"2024-11-10T15:04:07.910522Z","iopub.execute_input":"2024-11-10T15:04:07.911289Z","iopub.status.idle":"2024-11-10T15:04:07.948389Z","shell.execute_reply.started":"2024-11-10T15:04:07.911235Z","shell.execute_reply":"2024-11-10T15:04:07.947267Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Add extra training features","metadata":{}},{"cell_type":"code","source":"# Code Used to get Best Corr\n\n# def get_best_corr(df):\n#     corr = df[feature_columns + ['responder_6']].corr()['responder_6'].reset_index()\n#     corr = corr[['index', 'responder_6']]\n#     corr['responder_6'] = abs(corr['responder_6'])\n#     corr.sort_values('responder_6', ascending = False, inplace = True)\n#     best_corr = corr.iloc[3:50, 0].to_list()\n#     return best_corr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T15:04:07.949979Z","iopub.execute_input":"2024-11-10T15:04:07.950422Z","iopub.status.idle":"2024-11-10T15:04:07.955647Z","shell.execute_reply.started":"2024-11-10T15:04:07.950372Z","shell.execute_reply":"2024-11-10T15:04:07.954602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.with_columns(\n    pl.concat_str([pl.col('date_id').cast(pl.Utf8), pl.col('time_id').cast(pl.Utf8)], separator='_').alias('date_time')\n)\n\ndf = df.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T15:04:07.957377Z","iopub.execute_input":"2024-11-10T15:04:07.957743Z","iopub.status.idle":"2024-11-10T15:05:10.080068Z","shell.execute_reply.started":"2024-11-10T15:04:07.957705Z","shell.execute_reply":"2024-11-10T15:05:10.078888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in tqdm(best_corr):\n    # Calculate mean of each column in `best_corr` grouped by 'date_time'\n    mapper = (\n        df.group_by('date_time').agg(pl.col(col).mean().alias('mean_col'))\n    )\n\n    # Join the computed mean back to the original dataframe on 'date_time'\n    df = df.join(mapper, on='date_time', how='left')\n\n    # Rename and cast the column\n    df = df.rename({\"mean_col\": f\"time_id_{col}\"})\n    df = df.with_columns(pl.col(f\"time_id_{col}\").cast(pl.Float32))\n\ndf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T15:05:10.081724Z","iopub.execute_input":"2024-11-10T15:05:10.082163Z","iopub.status.idle":"2024-11-10T15:06:48.274900Z","shell.execute_reply.started":"2024-11-10T15:05:10.082123Z","shell.execute_reply":"2024-11-10T15:06:48.273795Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T15:06:48.278133Z","iopub.execute_input":"2024-11-10T15:06:48.278493Z","iopub.status.idle":"2024-11-10T15:06:48.283850Z","shell.execute_reply.started":"2024-11-10T15:06:48.278454Z","shell.execute_reply":"2024-11-10T15:06:48.282926Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Split training data and validation data","metadata":{}},{"cell_type":"code","source":"historical_data = df.filter((pl.col('date_id') >= CONFIG.hist_date_start) & (pl.col('date_id') < CONFIG.hist_date_end))\ntraining_data = df.filter((pl.col('date_id') >= CONFIG.train_date_start))\n\ndel df\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T15:06:48.285174Z","iopub.execute_input":"2024-11-10T15:06:48.285584Z","iopub.status.idle":"2024-11-10T15:06:50.447454Z","shell.execute_reply.started":"2024-11-10T15:06:48.285533Z","shell.execute_reply":"2024-11-10T15:06:50.446518Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Total row in training data {training_data.select(pl.col('date_id')).shape[0]}\")","metadata":{"execution":{"iopub.status.busy":"2024-11-10T15:06:50.448750Z","iopub.execute_input":"2024-11-10T15:06:50.449098Z","iopub.status.idle":"2024-11-10T15:06:50.455205Z","shell.execute_reply.started":"2024-11-10T15:06:50.449064Z","shell.execute_reply":"2024-11-10T15:06:50.454111Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Total row in historical data {historical_data.select(pl.col('date_id')).shape[0]}\")","metadata":{"execution":{"iopub.status.busy":"2024-11-10T15:06:50.456674Z","iopub.execute_input":"2024-11-10T15:06:50.457074Z","iopub.status.idle":"2024-11-10T15:06:50.466802Z","shell.execute_reply.started":"2024-11-10T15:06:50.457034Z","shell.execute_reply":"2024-11-10T15:06:50.465728Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Save data as parquets","metadata":{}},{"cell_type":"code","source":"%%time\nhistorical_data.write_parquet(\n    f\"historical.parquet\",\n    use_pyarrow=True,\n    pyarrow_options={\"partition_cols\": [\"partition_id\"]},\n    row_group_size=5000000,\n)\nprint(\"Done\")\n\ndel historical_data\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T15:06:50.468123Z","iopub.execute_input":"2024-11-10T15:06:50.468451Z","iopub.status.idle":"2024-11-10T15:07:23.258975Z","shell.execute_reply.started":"2024-11-10T15:06:50.468415Z","shell.execute_reply":"2024-11-10T15:07:23.257548Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ntraining_data.write_parquet(\n    f\"training.parquet\",\n    use_pyarrow=True,\n    pyarrow_options={\"partition_cols\": [\"partition_id\"]},\n    row_group_size=5000000,\n)\nprint(\"Done\")\n\ndel training_data\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T15:07:23.260974Z","iopub.execute_input":"2024-11-10T15:07:23.261470Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}