{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\n\nThis notebook preprocesses the data in a way that parquets can be loaded by date_id. This can be useful in later model fitting where you want to (cross)-validate by the date ids.\nA few comments:\n* Some simple feature engineering is provided but they don't seem to be all too useful (e.g. cos transformation of time)\n* I tried making symbol_id and Features 9/10/11 categorical (using `pl.cast`) but couldn't quite perform XGBoost training with it (the R2 errors were -30 for some reason, so there is probably some encoding error).\n    * One-hot-encoding for these columns is also expensive (~40 columns for symbols, many columns for features 9/10/11) so maybe some smart clustering (e.g. 3-5 groups for the symbols) could be useful\n* **I am preparing a notebook where we can use XGBoost training on a large part of the data solely using the Kaggle environment - watch this space for updates!**\n\nFeel free to copy or leave some suggestions, always curious to hear what you do with it!\n\nSome acknowledgements where I got some inspiration:\n* The work of [motono0223](https://www.kaggle.com/motono0223): https://www.kaggle.com/code/motono0223/js24-inference-gbdt-with-lags-singlemodel","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport polars as pl\nfrom sklearn.preprocessing import OneHotEncoder\n\nkaggle_dir = '/kaggle/input/jane-street-real-time-market-data-forecasting/'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-16T17:01:53.230453Z","iopub.execute_input":"2024-11-16T17:01:53.230902Z","iopub.status.idle":"2024-11-16T17:01:56.004276Z","shell.execute_reply.started":"2024-11-16T17:01:53.230838Z","shell.execute_reply":"2024-11-16T17:01:56.003256Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Feature columns\nkey_cols = [\"date_id\", \"time_id\", \"symbol_id\"]\nweight_cols = ['weight']\n\ntarget_cols = [f\"responder_{i}\" for i in range(9)]\nfeature_cols = [f\"feature_{i:02d}\" for i in range(79)]\nlag_cols = [x + \"_lag_1\" for x in target_cols]\n\n# symbol encoder\nsymbol_encoder = OneHotEncoder(categories=[range(41)], handle_unknown='ignore', sparse_output=False)\nsymbol_cols = [f'symbol_{i}' for i in range(41)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-16T01:28:36.897145Z","iopub.execute_input":"2024-11-16T01:28:36.897683Z","iopub.status.idle":"2024-11-16T01:28:36.906048Z","shell.execute_reply.started":"2024-11-16T01:28:36.897642Z","shell.execute_reply":"2024-11-16T01:28:36.904324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Example loading\ni = 0\n\n# raw training data + responders\nraw_parquet = pl.read_parquet(os.path.join(kaggle_dir, f'train.parquet/partition_id={i}/part-0.parquet'))\nlags_parquet0 = pl.read_parquet(os.path.join(kaggle_dir, 'lags.parquet/date_id=0/part-0.parquet'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-16T01:28:36.907749Z","iopub.execute_input":"2024-11-16T01:28:36.908282Z","iopub.status.idle":"2024-11-16T01:28:39.170594Z","shell.execute_reply.started":"2024-11-16T01:28:36.908229Z","shell.execute_reply":"2024-11-16T01:28:39.169368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# get number of unique categories in features (we do not use this here)\nfeature_cols_cat = [f'feature_{i:02d}' for i in [9, 10, 11]]\nfeature_cat_dict = {x: set() for x in feature_cols_cat}\n\nfor i in range(10):\n    raw_parquet = pl.read_parquet(os.path.join(kaggle_dir, f'train.parquet/partition_id={i}/part-0.parquet'))\n    \n    for categ in feature_cols_cat:\n        feature_cat_dict[categ].update(\n            set(raw_parquet[categ].unique())\n        )\n\nprint(\"No. Categorical Features in Features 9/10/11\", [len(feature_cat_dict[x]) for x in feature_cols_cat])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-16T01:28:39.172403Z","iopub.execute_input":"2024-11-16T01:28:39.172747Z","iopub.status.idle":"2024-11-16T01:29:23.123531Z","shell.execute_reply.started":"2024-11-16T01:28:39.172711Z","shell.execute_reply":"2024-11-16T01:29:23.122344Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 0. Full Loop","metadata":{}},{"cell_type":"code","source":"for i in range(10):\n    print(i)\n    raw_parquet = pl.read_parquet(os.path.join(kaggle_dir, f'train.parquet/partition_id={i}/part-0.parquet'))\n\n    # symbol_ohe\n    # print(\"Symbol OHE\")\n    # symbol_ohe = symbol_encoder.fit_transform(raw_parquet.select('symbol_id'))\n    # symbol_ohe_pl = pl.from_numpy(symbol_ohe, schema=symbol_cols)\n\n    # lags\n    print(\"Lags\")\n    concat_lags = (\n        raw_parquet\n        .select(key_cols + target_cols)\n        .filter(pl.col('time_id') == 0)\n        .unique(subset=key_cols)\n        .sort(['date_id', 'symbol_id'])\n        .rename({x: x+\"_lag_1\" for x in target_cols})\n        .drop(['time_id'])\n        .with_columns(\n            date_id = pl.col('date_id') + 1\n        )\n    )\n    \n    # only for completeness\n    train_lags = pl.concat([lags_parquet0.drop('time_id'), concat_lags], how='vertical')\n\n    # train df with some feature engineering\n    print(\"Saving Train\")\n    train = (\n        raw_parquet\n        .with_columns(\n            pl.Series(name='date_id_cos360', values=np.cos(2*np.pi/360 * raw_parquet.select(pl.col('date_id'))).reshape(-1), dtype = pl.Float64),\n            pl.Series(name='time_id_cos360', values=np.cos(2*np.pi/360 * raw_parquet.select(pl.col('time_id'))).reshape(-1), dtype = pl.Float64),\n            pl.Series(name='date_id_cos180', values=np.cos(2*np.pi/180 * raw_parquet.select(pl.col('date_id'))).reshape(-1), dtype = pl.Float64),\n            pl.Series(name='time_id_cos180', values=np.cos(2*np.pi/180 * raw_parquet.select(pl.col('time_id'))).reshape(-1), dtype = pl.Float64),\n            pl.Series(name='date_id_cos90', values=np.cos(2*np.pi/90 * raw_parquet.select(pl.col('date_id'))).reshape(-1), dtype = pl.Float64),\n            pl.Series(name='time_id_cos90', values=np.cos(2*np.pi/90 * raw_parquet.select(pl.col('time_id'))).reshape(-1), dtype = pl.Float64),\n            (10*pl.col(\"time_id\") / pl.when(pl.col(\"date_id\") <= 676).then(848).otherwise(967)).alias(\"time_id_adj\"),\n            # pl.col(\"feature_09\").cast(pl.String).cast(pl.Categorical).alias(\"feature_09\"),\n            # pl.col(\"feature_10\").cast(pl.String).cast(pl.Categorical).alias(\"feature_10\"),\n            # pl.col(\"feature_11\").cast(pl.String).cast(pl.Categorical).alias(\"feature_11\")\n         )\n        .join(train_lags, on=['date_id', 'symbol_id'], how='left')\n        # .with_columns(\n        #     pl.col(\"symbol_id\").cast(pl.String).cast(pl.Categorical).alias(\"symbol_id\"),\n        # )\n    )\n\n    # train.write_parquet(f\"train_{i}.parquet\")\n    train.write_parquet(f\"training\", partition_by = \"date_id\")\n\n    # saving dates\n    print(\"Saving Dates\")\n    (\n        raw_parquet\n        .select(pl.col('date_id'))\n        .unique()\n        .sort('date_id')\n        .write_csv(f'dates_{i}.csv')\n    )\n","metadata":{"execution":{"iopub.status.busy":"2024-11-13T05:43:49.509354Z","iopub.execute_input":"2024-11-13T05:43:49.509815Z"},"trusted":true},"outputs":[],"execution_count":null}]}