{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Intro to Invariant Learning for Finance\nThis notebook is design to highlight the benefits of a new GBDT algo for learning *invariant* signal from data, all in the context of the fastest GPU-enabled GBM currently on the market: **WarpGBM, Invariant Edition**. This notebook with show how to implement **Directional Era-Splitting (DES)** and how to compare it to a baseline model.\n\nFor a quick intro into the field of OOD Generalization and the concept of invariance in ML, head over to the README on the WarpGBM GitHub page.\n\n[https://github.com/jefferythewind/warpgbm](http://)\n\nThis notebook is going to quickly show, without any feature engineering, how dropping in an array of *eras***** can quickly take OOS performance from **0.043% to 0.062%** correlation w/ target witth the same underlying GBM model. This is how you enable era-wise awareness and unlock invariant learning. \n\n## Outline:\n\n1) install prerequisites and WarpGBM\n2) Train baseline model in CV without DES\n3) Train DES model in CV with DES, passing array or eras.\n4) Compare performance.\n\nI am trying to create more awareness for this tool. Try dropping it into you're pipeline. Make your GBM era-aware. And I believe we will push up the top scores at the head of the leaderboard.\n\nGood Luck.","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Upgrade Torch to 2.6.0+CUDA 12.4\n!pip install --upgrade torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124\n\n# Confirm torch version\nimport torch\nprint(\"Torch version:\", torch.__version__)\nprint(\"Torch CUDA version:\", torch.version.cuda)\n\nimport torch\nprint(torch.__version__)\nprint(torch.version.cuda)\n\n!pip install warpgbm --no-build-isolation","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-30T14:16:20.037034Z","iopub.execute_input":"2025-05-30T14:16:20.037284Z","iopub.status.idle":"2025-05-30T14:16:34.693370Z","shell.execute_reply.started":"2025-05-30T14:16:20.037265Z","shell.execute_reply":"2025-05-30T14:16:34.692529Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import TimeSeriesSplit\nfrom warpgbm import WarpGBM","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T14:18:22.476743Z","iopub.execute_input":"2025-05-30T14:18:22.477322Z","iopub.status.idle":"2025-05-30T14:18:23.787976Z","shell.execute_reply.started":"2025-05-30T14:18:22.477293Z","shell.execute_reply":"2025-05-30T14:18:23.787095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\ntest = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')\nsample = pd.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T14:16:34.695075Z","iopub.execute_input":"2025-05-30T14:16:34.695715Z","iopub.status.idle":"2025-05-30T14:17:19.625276Z","shell.execute_reply.started":"2025-05-30T14:16:34.695691Z","shell.execute_reply":"2025-05-30T14:17:19.624396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target = 'label'\nfeatures = [ f for f in list(train) if f != target ]\nall_splits = list(TimeSeriesSplit(n_splits=2, max_train_size=100_000_000, gap=1).split(train.index))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T14:18:27.045144Z","iopub.execute_input":"2025-05-30T14:18:27.046040Z","iopub.status.idle":"2025-05-30T14:18:27.050955Z","shell.execute_reply.started":"2025-05-30T14:18:27.046009Z","shell.execute_reply":"2025-05-30T14:18:27.050143Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Selection","metadata":{}},{"cell_type":"code","source":"def get_best_feats(train_df):\n    correlations = train_df[features].apply(lambda x: x.corr(train_df[target]))\n    correlations = correlations.sort_values(ascending=False)\n    best_feats = correlations.loc[ correlations.abs() > 0.025 ].index.tolist()\n    return best_feats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T14:58:07.406079Z","iopub.execute_input":"2025-05-30T14:58:07.406384Z","iopub.status.idle":"2025-05-30T14:58:07.411063Z","shell.execute_reply.started":"2025-05-30T14:58:07.406363Z","shell.execute_reply":"2025-05-30T14:58:07.410273Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Naive GBM","metadata":{}},{"cell_type":"code","source":"train['pred_naive'] = 1 * np.nan\ntest_preds_naive = []\nfor split_id, (train_idx, val_idx) in enumerate( all_splits ):\n    train_rows = train.index[train_idx]\n    val_rows = train.index[val_idx]\n    best_feats = get_best_feats( train.loc[train_rows] )\n    X = train.loc[ train_rows, best_feats ].values\n    y = train.loc[ train_rows, target ].values\n    X_val = train.loc[ val_rows, best_feats ].values\n    y_val = train.loc[ val_rows, target ].values\n    model = WarpGBM(\n        max_depth=10,\n        num_bins=100,\n        n_estimators=100,\n        learning_rate=0.1,\n        colsample_bytree=1.0,\n        min_child_weight=4\n    )\n    model.fit(\n        X,\n        y,\n        X_eval=X_val,\n        y_eval=y_val,\n        eval_every_n_trees=1,\n        early_stopping_rounds=10,\n        eval_metric=\"corr\",\n    )\n    #keep best model\n    best_i = int(np.argmin(model.eval_loss))\n    opt_num_trees = best_i * model.eval_every_n_trees  # +1 since first eval is at tree N, not 0\n    model.forest = model.forest[:( opt_num_trees + 1)]\n    \n    preds = model.predict(X_val)\n    train.loc[ val_rows, 'pred_naive' ] = preds\n    test_preds_naive.append( model.predict(test.loc[:, features ].values) )\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T14:58:14.004091Z","iopub.execute_input":"2025-05-30T14:58:14.004371Z","iopub.status.idle":"2025-05-30T15:01:13.541638Z","shell.execute_reply.started":"2025-05-30T14:58:14.004353Z","shell.execute_reply":"2025-05-30T15:01:13.540769Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Era-Aware GBM","metadata":{}},{"cell_type":"code","source":"train['era'] = pd.qcut(range(len(train)), q=20, labels=False, duplicates='drop')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T15:01:13.542832Z","iopub.execute_input":"2025-05-30T15:01:13.543092Z","iopub.status.idle":"2025-05-30T15:01:13.601169Z","shell.execute_reply.started":"2025-05-30T15:01:13.543074Z","shell.execute_reply":"2025-05-30T15:01:13.600578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['pred_aware'] = 1 * np.nan\ntest_preds_aware = []\nopt_trees = []\nfor split_id, (train_idx, val_idx) in enumerate( all_splits ):\n    train_rows = train.index[train_idx]\n    val_rows = train.index[val_idx]\n\n    '''Define an Array of Integers to Define the Eras'''\n    eras = train.loc[ train_rows, 'era' ].values\n    print(\"Era Vector: \", eras)\n\n    best_feats = get_best_feats( train.loc[train_rows] )\n    X = train.loc[ train_rows, best_feats ].values\n    y = train.loc[ train_rows, target ].values\n    X_val = train.loc[ val_rows, best_feats ].values\n    y_val = train.loc[ val_rows, target ].values\n    model = WarpGBM(\n        max_depth=10,\n        num_bins=100,\n        n_estimators=100,\n        learning_rate=0.1,\n        colsample_bytree=1.0,\n        min_child_weight=4\n    )\n    model.fit(\n        X,\n        y,\n        eras, # use the eras here, in .fit()\n        X_eval=X_val,\n        y_eval=y_val,\n        eval_every_n_trees=1,\n        early_stopping_rounds=10,\n        eval_metric=\"corr\",\n    )\n    #keep best model\n    best_i = int(np.argmin(model.eval_loss))\n    opt_num_trees = best_i * model.eval_every_n_trees  # +1 since first eval is at tree N, not 0\n    model.forest = model.forest[:( opt_num_trees + 1 )]\n    opt_trees.append(opt_num_trees)\n    \n    preds = model.predict(X_val)\n    train.loc[ val_rows, 'pred_aware' ] = preds\n    test_preds_aware.append( model.predict(test.loc[:, features ].values) )\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T14:56:34.556682Z","iopub.execute_input":"2025-05-30T14:56:34.556948Z","iopub.status.idle":"2025-05-30T14:57:20.467116Z","shell.execute_reply.started":"2025-05-30T14:56:34.556929Z","shell.execute_reply":"2025-05-30T14:57:20.466517Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Compare Results","metadata":{}},{"cell_type":"code","source":"print( \"Naive Model Corr w/ Target:\", train.loc[ :, ['pred_naive', target]].dropna().corr().iloc[0,1] )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T14:57:24.495281Z","iopub.execute_input":"2025-05-30T14:57:24.495827Z","iopub.status.idle":"2025-05-30T14:57:24.520569Z","shell.execute_reply.started":"2025-05-30T14:57:24.495805Z","shell.execute_reply":"2025-05-30T14:57:24.520023Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print( \"Era Aware Model Corr w/ Target:\", train.loc[ :, ['pred_aware', target]].dropna().corr().iloc[0,1] )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T14:57:26.238531Z","iopub.execute_input":"2025-05-30T14:57:26.239020Z","iopub.status.idle":"2025-05-30T14:57:26.264378Z","shell.execute_reply.started":"2025-05-30T14:57:26.238996Z","shell.execute_reply":"2025-05-30T14:57:26.263638Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submissions","metadata":{}},{"cell_type":"code","source":"sub_naive = sample.copy()\nsub_naive['prediction'] = np.mean( test_preds_naive, axis=0)\nsub_naive.to_csv('submission_naive.csv', index=False)\n\nsub_aware = sample.copy()\nsub_aware['prediction'] = test_preds_aware[1]#np.mean( test_preds_aware, axis=0)\nsub_aware.to_csv('submission_aware.csv', index=False)\nsub_aware.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T13:53:16.890756Z","iopub.execute_input":"2025-05-30T13:53:16.890968Z","iopub.status.idle":"2025-05-30T13:53:18.498407Z","shell.execute_reply.started":"2025-05-30T13:53:16.890953Z","shell.execute_reply":"2025-05-30T13:53:18.497559Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_naive","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T13:53:18.499481Z","iopub.execute_input":"2025-05-30T13:53:18.499762Z","iopub.status.idle":"2025-05-30T13:53:18.511469Z","shell.execute_reply.started":"2025-05-30T13:53:18.499740Z","shell.execute_reply":"2025-05-30T13:53:18.510666Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}