{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":777.076344,"end_time":"2025-07-20T14:10:42.981129","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-07-20T13:57:45.904785","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Staked Random Forest with Light GBM\n\n- First tried `linear regression` : $R^2 \\approx 0.05$\n- Data not centered, variance similar between all $X$ variables\n- `random forest` : $R^2 \\approx 0.17$,\n    - expected to avoid overfitting,\n- apply learning rate to compute the residuals for the next layer,\n- `LGBM` best performer so far (standalone $R^2 \\approx 0.56$)  \n\nImproved standalone LGBM by a small margin : $R^2 \\approx 0.58$.","metadata":{"papermill":{"duration":0.003631,"end_time":"2025-07-20T13:57:50.729214","exception":false,"start_time":"2025-07-20T13:57:50.725583","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nfrom typing import List\nfrom pathlib import Path\nfrom scipy.stats import pearsonr\nfrom sklearn.metrics import r2_score\nfrom sklearn.model_selection import train_test_split\n\nfrom sklearn.ensemble import RandomForestRegressor\nimport lightgbm as lgb\n\nfor dirname, _, filenames in os.walk(\"/kaggle/\"):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nKAGGLE = True  # define paths accordingly\nSUBMISSION = True  # use smaller datasets during dev\n\nif KAGGLE:\n    crypto_folder = Path(\"/kaggle/input/drw-crypto-market-prediction\")\nelse:\n    crypto_folder = Path(\"../raw_data/crypto\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2025-07-20T21:56:01.221871Z","iopub.status.idle":"2025-07-20T21:56:01.222138Z","shell.execute_reply.started":"2025-07-20T21:56:01.222012Z","shell.execute_reply":"2025-07-20T21:56:01.222026Z"},"papermill":{"duration":10.264046,"end_time":"2025-07-20T13:58:00.996596","exception":false,"start_time":"2025-07-20T13:57:50.732550","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Helper functions","metadata":{"papermill":{"duration":0.002666,"end_time":"2025-07-20T13:58:01.002348","exception":false,"start_time":"2025-07-20T13:58:00.999682","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_clean_crypto_data(train: bool = True) -> pl.LazyFrame:\n    \"\"\"\n    Load and clean crypto data, returning either train or test set.\n\n    Args:\n        train: If True, return training set. If False, return test set.\n\n    Returns:\n        Cleaned lazy frame with columns that have variance and no infinite values.\n    \"\"\"\n\n    filename = \"train.parquet\" if train else \"test.parquet\"\n\n    # load data\n    crypto_lazy = pl.scan_parquet(crypto_folder / filename)\n    n_cols = len(crypto_lazy.collect_schema().names())\n\n    if train and KAGGLE:\n        # rename timestamp column\n        crypto_lazy = crypto_lazy.with_columns(\n            pl.col(\"__index_level_0__\").alias(\"timestamp\")\n        ).drop([\"__index_level_0__\"])\n\n    # Remove columns with zero variance in the training set\n    train_lazy = pl.scan_parquet(crypto_folder / \"train.parquet\")\n    if KAGGLE:\n        train_lazy = train_lazy.with_columns(\n            pl.col(\"__index_level_0__\").alias(\"timestamp\")\n        ).drop([\"__index_level_0__\"])\n\n    # Get column names and calculate variance on training set (for consistency)\n    crypto_var = train_lazy.select(pl.exclude([\"timestamp\"]).var())\n\n    crypto_var_cols = (\n        crypto_var.select(pl.all() == 0.0)\n        .first()\n        .collect()\n        .to_pandas()\n        .T.rename(columns={0: \"is_variance_null\"})\n        .reset_index()\n        .rename(columns={\"index\": \"column_name\"})\n        .groupby(\"is_variance_null\")[\"column_name\"]\n        .unique()\n    )\n\n    crypto_cols_with_var = crypto_var_cols[False]\n\n    try:\n        cols_no_var = crypto_var_cols[True]\n        print(f\"Columns with no variance : {cols_no_var}\")\n    except KeyError:\n        print(\"All columns have variance in the train set\")\n\n    # remove columns that have no variance in the training set\n    train_lazy = train_lazy.select(\n        [\"timestamp\"] + [pl.col(c) for c in crypto_cols_with_var]\n    )\n\n    # Remove columns with infinite values (check on training set)\n    current_columns = train_lazy.collect_schema().names()\n    contains_infinite_cols = (\n        train_lazy.select(pl.exclude(\"timestamp\").abs().max().is_infinite())\n        .collect()\n        .to_pandas()\n        .T.rename(columns={0: \"contains_infinite\"})\n        .reset_index()\n        .rename(columns={\"index\": \"column_name\"})\n        .groupby(\"contains_infinite\")[\"column_name\"]\n        .unique()\n    )\n\n    try:\n        cols_with_inf_vals = contains_infinite_cols[True]\n        print(f\"Columns with infinite values : {cols_with_inf_vals}\")\n    except KeyError:\n        print(\"No columns with infinite values\")\n\n    if not train:\n        # add dummy timestamps\n        crypto_lazy = crypto_lazy.with_columns(\n            ID=range(1, crypto_lazy.select(pl.len()).collect().item() + 1)\n        )\n    # Filter clean columns based on what's available in the current dataset\n    clean_columns = [\n        c for c in current_columns if c in contains_infinite_cols[False]\n    ] + [\"timestamp\", \"ID\"]\n    available_columns = crypto_lazy.collect_schema().names()\n    final_columns = [c for c in clean_columns if c in available_columns]\n    print(f\"Eventually {len(final_columns)}, removed {n_cols - len(final_columns)}\")\n\n    return crypto_lazy.select(final_columns)\n\n\ndef get_diff_features(df: pl.LazyFrame, stats_columns: List[str]):\n    return (\n        df.with_columns(pl.exclude(stats_columns).diff())\n        .with_row_index()\n        .fill_null(strategy=\"backward\")\n        .select(pl.exclude(\"index\"))\n    )\n\ndef get_ma_features(df: pl.LazyFrame, cols: List[str], ws:int=100):\n    return (\n        df.with_columns(\n            pl.col(cols)\n            .rolling_mean(\n                window_size=ws, \n                min_samples=1\n                )\n        )\n    )\n\ndef get_rolling_var(df: pl.LazyFrame, cols: List[str], ws:int=100):\n    return (\n        df.with_columns(\n            pl.col(cols)\n            .rolling_var(\n                window_size=ws,\n                min_samples=1\n            ).backward_fill()\n        )\n    )","metadata":{"execution":{"iopub.status.busy":"2025-07-20T21:56:01.223097Z","iopub.status.idle":"2025-07-20T21:56:01.223494Z","shell.execute_reply.started":"2025-07-20T21:56:01.223285Z","shell.execute_reply":"2025-07-20T21:56:01.223301Z"},"papermill":{"duration":0.018102,"end_time":"2025-07-20T13:58:01.023357","exception":false,"start_time":"2025-07-20T13:58:01.005255","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Metadata","metadata":{"papermill":{"duration":0.002735,"end_time":"2025-07-20T13:58:01.029189","exception":false,"start_time":"2025-07-20T13:58:01.026454","status":"completed"},"tags":[]}},{"cell_type":"code","source":"stats_columns = [\n    \"timestamp\",\n    \"bid_qty\",\n    \"ask_qty\",\n    \"buy_qty\",\n    \"sell_qty\",\n    \"volume\",\n    \"label\",\n]\nstats_columns_test = [\n    \"ID\",\n    \"bid_qty\",\n    \"ask_qty\",\n    \"buy_qty\",\n    \"sell_qty\",\n    \"volume\",\n    \"label\",\n]\nX_exclude = [\"timestamp\", \"label\"]\nX_test_exclude = [\"ID\", \"label\"]","metadata":{"execution":{"iopub.status.busy":"2025-07-20T21:56:01.224919Z","iopub.status.idle":"2025-07-20T21:56:01.225198Z","shell.execute_reply.started":"2025-07-20T21:56:01.225057Z","shell.execute_reply":"2025-07-20T21:56:01.225069Z"},"papermill":{"duration":0.010025,"end_time":"2025-07-20T13:58:01.042131","exception":false,"start_time":"2025-07-20T13:58:01.032106","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load data","metadata":{"papermill":{"duration":0.002802,"end_time":"2025-07-20T13:58:01.048043","exception":false,"start_time":"2025-07-20T13:58:01.045241","status":"completed"},"tags":[]}},{"cell_type":"code","source":"crypto_lazy_clean = get_clean_crypto_data(train=True)\ncols = crypto_lazy_clean.collect_schema().names()\n","metadata":{"execution":{"iopub.status.busy":"2025-07-20T21:56:01.226938Z","iopub.status.idle":"2025-07-20T21:56:01.227214Z","shell.execute_reply.started":"2025-07-20T21:56:01.227094Z","shell.execute_reply":"2025-07-20T21:56:01.227106Z"},"papermill":{"duration":28.727997,"end_time":"2025-07-20T13:58:29.778904","exception":false,"start_time":"2025-07-20T13:58:01.050907","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# join level with diff values\n# crypto_lazy_clean = crypto_lazy_clean.join(\n#     get_diff_features(crypto_lazy_clean, stats_columns),\n#     on=stats_columns,\n#     how=\"inner\",\n#     suffix=\"_diff\",\n# )\n\ncrypto_lazy_clean = crypto_lazy_clean.join(\n    get_ma_features(crypto_lazy_clean, [c for c in cols if c not in stats_columns]),\n    on=stats_columns,\n    how=\"inner\",\n    suffix=\"_ma23\",\n)\nprint(len(crypto_lazy_clean.collect_schema().names()))\ncrypto_lazy_clean = crypto_lazy_clean.join(\n    get_rolling_var(\n        crypto_lazy_clean.select(cols), \n        [c for c in cols if c not in stats_columns]\n        ).fill_null(0.),\n    on=stats_columns,\n    how=\"inner\",\n    suffix=\"_var23\",\n)\nprint(len(crypto_lazy_clean.collect_schema().names()))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-20T21:35:55.247677Z","iopub.execute_input":"2025-07-20T21:35:55.248040Z","iopub.status.idle":"2025-07-20T21:35:55.294937Z","shell.execute_reply.started":"2025-07-20T21:35:55.247987Z","shell.execute_reply":"2025-07-20T21:35:55.293835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nX = crypto_lazy_clean.select(pl.exclude(X_exclude)).collect().to_numpy()\ny = crypto_lazy_clean.select(pl.col(\"label\")).collect().to_numpy().T[0]\n\nif not SUBMISSION:\n    X_train, X_test, y_train, y_test = train_test_split(\n        X,\n        y,\n        test_size=0.2,\n        shuffle=False,  # TODO : question this, whether timestamps are independant draws\n        random_state=42,\n    )\nelse:\n    X_train, y_train = X, y\ndel X\ndel y\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-20T21:35:55.296317Z","iopub.execute_input":"2025-07-20T21:35:55.296684Z","iopub.status.idle":"2025-07-20T21:35:56.832775Z","shell.execute_reply.started":"2025-07-20T21:35:55.296645Z","shell.execute_reply":"2025-07-20T21:35:56.831839Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train model","metadata":{"papermill":{"duration":0.003242,"end_time":"2025-07-20T13:58:29.786476","exception":false,"start_time":"2025-07-20T13:58:29.783234","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\nlr = 1.\n\nlin = RandomForestRegressor(\n    # fit_intercept=True,\n    n_estimators=80,\n    n_jobs=-1,\n    max_depth=10,\n    min_samples_split=100,\n    min_samples_leaf=50,\n    max_features=\"sqrt\",\n    max_samples=0.5,\n    random_state=41,\n)\n# n_samples = 80_000\nlin.fit(\n    X_train,\n    y_train,\n    # sample_weight=np.flip(1.0 / np.sqrt(np.arange(1, n_samples+1)))\n)\n\ny_train_lin = lin.predict(X_train)\n\nprint(f\"R2 train lin: {r2_score(y_train, y_train_lin)}\")\nprint(f\"Pearson train lin : {pearsonr(y_train, y_train_lin)}\")\n\ny_train_res = y_train - lr * y_train_lin\n\n\nlgb_model = lgb.LGBMRegressor(\n    random_state=42,\n    # weight=np.flip(1.0 / np.sqrt(np.arange(1, len(X_train)+1))),\n    # n_estimators=80,\n    # max_depth=10,\n    n_jobs=-1,\n)\nlgb_model.fit(X_train, y_train_res)\n\ny_train_hat = lgb_model.predict(X_train)\n\nprint(f\"R2 train : {r2_score(y_train, y_train_hat + lr * y_train_lin)}\")\nprint(f\"Pearson train : {pearsonr(y_train, y_train_hat + lr * y_train_lin)}\")\n\nif not SUBMISSION:\n    y_test_lin = lin.predict(X_test)\n\n    print(f\"R2 test lin : {r2_score(y_test, y_test_lin)}\")\n    print(f\"Pearson test lin : {pearsonr(y_test, y_test_lin)}\")\n\n    y_test_hat = lgb_model.predict(X_test)\n\n    print(f\"R2 test : {r2_score(y_test, y_test_hat + lr * y_test_lin)}\")\n    print(f\"Pearson test : {pearsonr(y_test, y_test_hat + lr * y_test_lin)}\")\n\ndel y_train_lin\ndel y_train_res\ndel y_train_hat\ndel X_train\n\n    ","metadata":{"execution":{"iopub.status.busy":"2025-07-20T21:35:56.833549Z","iopub.execute_input":"2025-07-20T21:35:56.833885Z","iopub.status.idle":"2025-07-20T21:36:04.654160Z","shell.execute_reply.started":"2025-07-20T21:35:56.833859Z","shell.execute_reply":"2025-07-20T21:36:04.653227Z"},"papermill":{"duration":663.54052,"end_time":"2025-07-20T14:09:33.330618","exception":false,"start_time":"2025-07-20T13:58:29.790098","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load test data","metadata":{"papermill":{"duration":0.003269,"end_time":"2025-07-20T14:09:33.337700","exception":false,"start_time":"2025-07-20T14:09:33.334431","status":"completed"},"tags":[]}},{"cell_type":"code","source":"crypto_lazy_test = get_clean_crypto_data(train=False)\n# create unique row identifier\n# n = crypto_lazy_test.select(pl.len()).collect().item()\n# crypto_lazy_test = crypto_lazy_test.with_columns(ID=range(1, n + 1))\n\n# print(crypto_lazy_test.select(pl.len()).collect().item())\n\n# crypto_lazy_test = crypto_lazy_test.join(\n#     get_diff_features(crypto_lazy_test, stats_columns_test),\n#     on=stats_columns_test,\n#     how=\"inner\",\n#     suffix=\"_diff\",\n# )\ncols_test = crypto_lazy_test.collect_schema().names()\ncrypto_lazy_test = crypto_lazy_test.join(\n    get_ma_features(crypto_lazy_test, [c for c in cols_test if c not in stats_columns_test]),\n    on=stats_columns_test,\n    how=\"inner\",\n    suffix=\"_ma23\",\n)\ncrypto_lazy_test = crypto_lazy_test.join(\n    get_rolling_var(\n        crypto_lazy_test.select(cols_test), \n        [c for c in cols_test if c not in stats_columns_test]\n        ),\n    on=stats_columns_test,\n    how=\"inner\",\n    suffix=\"_var23\",\n)\n\n# crypto_lazy_test = get_diff_features(crypto_lazy_test, stats_columns_test)\n# assert n == crypto_lazy_test.select(pl.len()).collect().item()","metadata":{"execution":{"iopub.status.busy":"2025-07-20T21:36:04.655080Z","iopub.execute_input":"2025-07-20T21:36:04.655360Z","iopub.status.idle":"2025-07-20T21:36:09.108401Z","shell.execute_reply.started":"2025-07-20T21:36:04.655340Z","shell.execute_reply":"2025-07-20T21:36:09.107395Z"},"papermill":{"duration":15.886968,"end_time":"2025-07-20T14:09:49.228000","exception":false,"start_time":"2025-07-20T14:09:33.341032","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Predict target \\& submit","metadata":{"papermill":{"duration":0.003187,"end_time":"2025-07-20T14:09:49.235751","exception":false,"start_time":"2025-07-20T14:09:49.232564","status":"completed"},"tags":[]}},{"cell_type":"code","source":"X_test = crypto_lazy_test.select(pl.exclude(X_test_exclude)).collect().to_numpy()\ny_lin_test = lin.predict(X_test)\ny_hat_lgb_test = lgb_model.predict(X_test)\n\ndel X_test","metadata":{"execution":{"iopub.status.busy":"2025-07-20T21:36:09.110581Z","iopub.execute_input":"2025-07-20T21:36:09.111492Z","iopub.status.idle":"2025-07-20T21:36:16.780946Z","shell.execute_reply.started":"2025-07-20T21:36:09.111453Z","shell.execute_reply":"2025-07-20T21:36:16.780097Z"},"papermill":{"duration":41.873573,"end_time":"2025-07-20T14:10:31.112620","exception":false,"start_time":"2025-07-20T14:09:49.239047","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"crypto_lazy_test = crypto_lazy_test.with_columns(\n    prediction=y_hat_lgb_test + lr * y_lin_test\n)\n# crypto_lazy_test.head(5).collect()\ncrypto_lazy_test.select([pl.col(\"ID\"), pl.col(\"prediction\")]).collect().write_csv(\n    Path(\"submission.csv\")\n)","metadata":{"execution":{"iopub.status.busy":"2025-07-20T21:36:16.782098Z","iopub.execute_input":"2025-07-20T21:36:16.782427Z","iopub.status.idle":"2025-07-20T21:36:16.871645Z","shell.execute_reply.started":"2025-07-20T21:36:16.782399Z","shell.execute_reply":"2025-07-20T21:36:16.870781Z"},"papermill":{"duration":7.39843,"end_time":"2025-07-20T14:10:38.517545","exception":false,"start_time":"2025-07-20T14:10:31.119115","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}