{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":null,"end_time":null,"environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-05-23T19:03:14.791476","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install scikit-learn==1.5.2","metadata":{"_kg_hide-output":true,"papermill":{"duration":9.209373,"end_time":"2025-05-23T19:03:29.658522","exception":false,"start_time":"2025-05-23T19:03:20.449149","status":"completed"},"scrolled":true,"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T07:52:46.375918Z","iopub.execute_input":"2025-05-29T07:52:46.376240Z","iopub.status.idle":"2025-05-29T07:52:57.129652Z","shell.execute_reply.started":"2025-05-29T07:52:46.376213Z","shell.execute_reply":"2025-05-29T07:52:57.128319Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Imports and configs","metadata":{"papermill":{"duration":0.004193,"end_time":"2025-05-23T19:03:29.66754","exception":false,"start_time":"2025-05-23T19:03:29.663347","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn.linear_model import Ridge\nfrom lightgbm import LGBMRegressor\nfrom scipy.stats import pearsonr\nfrom xgboost import XGBRegressor\nfrom sklearn.linear_model import ElasticNet\nfrom sklearn.linear_model import Lasso\nfrom sklearn.base import clone\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nimport warnings\nimport optuna\nimport gc\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"_kg_hide-output":true,"papermill":{"duration":8.748199,"end_time":"2025-05-23T19:03:38.419964","exception":false,"start_time":"2025-05-23T19:03:29.671765","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T07:52:57.131800Z","iopub.execute_input":"2025-05-29T07:52:57.132120Z","iopub.status.idle":"2025-05-29T07:53:04.690462Z","shell.execute_reply.started":"2025-05-29T07:52:57.132090Z","shell.execute_reply":"2025-05-29T07:53:04.689512Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n    target = \"label\"\n    n_folds = 5\n    seed = 42\n\n    run_optuna = True\n    n_optuna_trials = 250","metadata":{"papermill":{"duration":0.012323,"end_time":"2025-05-23T19:03:38.437012","exception":false,"start_time":"2025-05-23T19:03:38.424689","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T07:53:04.691496Z","iopub.execute_input":"2025-05-29T07:53:04.692063Z","iopub.status.idle":"2025-05-29T07:53:04.697822Z","shell.execute_reply.started":"2025-05-29T07:53:04.692038Z","shell.execute_reply":"2025-05-29T07:53:04.696717Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data loading and preprocessing","metadata":{"papermill":{"duration":0.004168,"end_time":"2025-05-23T19:03:38.445661","exception":false,"start_time":"2025-05-23T19:03:38.441493","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def reduce_mem_usage(dataframe, dataset):    \n    print('Reducing memory usage for:', dataset)\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    \n    for col in dataframe.columns:\n        col_type = dataframe[col].dtype\n\n        c_min = dataframe[col].min()\n        c_max = dataframe[col].max()\n        if str(col_type)[:3] == 'int':\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                dataframe[col] = dataframe[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                dataframe[col] = dataframe[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                dataframe[col] = dataframe[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                dataframe[col] = dataframe[col].astype(np.int64)\n        else:\n            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                dataframe[col] = dataframe[col].astype(np.float16)\n            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                dataframe[col] = dataframe[col].astype(np.float32)\n            else:\n                dataframe[col] = dataframe[col].astype(np.float64)\n\n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    print('--- Memory usage before: {:.2f} MB'.format(initial_mem_usage))\n    print('--- Memory usage after: {:.2f} MB'.format(final_mem_usage))\n    print('--- Decreased memory usage by {:.1f}%\\n'.format(100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage))\n\n    return dataframe","metadata":{"papermill":{"duration":0.016817,"end_time":"2025-05-23T19:03:38.46674","exception":false,"start_time":"2025-05-23T19:03:38.449923","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-05-29T07:53:04.698816Z","iopub.execute_input":"2025-05-29T07:53:04.699104Z","iopub.status.idle":"2025-05-29T07:53:04.727206Z","shell.execute_reply.started":"2025-05-29T07:53:04.699082Z","shell.execute_reply":"2025-05-29T07:53:04.726210Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_to_drop = [\n    'X697', 'X698', 'X699', 'X700', 'X701', 'X702', 'X703', 'X704', 'X705', 'X706', \n    'X707', 'X708', 'X709', 'X710', 'X711', 'X712', 'X713', 'X714', 'X715', 'X716',\n    'X717', 'X864', 'X867', 'X869', 'X870', 'X871', 'X872', 'X104', 'X110', 'X116',\n    'X122', 'X128', 'X134', 'X140', 'X146', 'X152', 'X158', 'X164', 'X170', 'X176',\n    'X182', 'X351', 'X357', 'X363', 'X369', 'X375', 'X381', 'X387', 'X393', 'X399',\n    'X405', 'X411', 'X417', 'X423', 'X429'\n]","metadata":{"papermill":{"duration":0.011921,"end_time":"2025-05-23T19:03:38.483197","exception":false,"start_time":"2025-05-23T19:03:38.471276","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T07:53:04.730161Z","iopub.execute_input":"2025-05-29T07:53:04.730466Z","iopub.status.idle":"2025-05-29T07:53:04.759487Z","shell.execute_reply.started":"2025-05-29T07:53:04.730442Z","shell.execute_reply":"2025-05-29T07:53:04.758358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_parquet(CFG.train_path).reset_index(drop=True)\ntest = pd.read_parquet(CFG.test_path).reset_index(drop=True)\n\ntrain = train.drop(columns=cols_to_drop)\ntest = test.drop(columns=[\"label\"] + cols_to_drop)\n\ntrain = reduce_mem_usage(train, \"train\")\ntest = reduce_mem_usage(test, \"test\")\n\n# Create interaction features for train\ntrain['bid_ask_interaction'] = train['bid_qty'] * train['ask_qty']\ntrain['bid_buy_interaction'] = train['bid_qty'] * train['buy_qty']\ntrain['bid_sell_interaction'] = train['bid_qty'] * train['sell_qty']\ntrain['ask_buy_interaction'] = train['ask_qty'] * train['buy_qty']\ntrain['ask_sell_interaction'] = train['ask_qty'] * train['sell_qty']\ntrain['buy_sell_interaction'] = train['buy_qty'] * train['sell_qty']\n\n# Calculate spread indicators for train\ntrain['spread_indicator'] = (train['ask_qty'] - train['bid_qty']) / (train['ask_qty'] + train['bid_qty'] + 1e-8)\n\n# Volume-weighted features for train\ntrain['volume_weighted_buy'] = train['buy_qty'] * train['volume']\ntrain['volume_weighted_sell'] = train['sell_qty'] * train['volume']\ntrain['volume_weighted_bid'] = train['bid_qty'] * train['volume']\ntrain['volume_weighted_ask'] = train['ask_qty'] * train['volume']\n\n# NEW FEATURES - Add ratio features\ntrain['buy_sell_ratio'] = train['buy_qty'] / (train['sell_qty'] + 1e-8)\ntrain['bid_ask_ratio'] = train['bid_qty'] / (train['ask_qty'] + 1e-8)\n\n# NEW FEATURES - Add order flow imbalance\ntrain['order_flow_imbalance'] = (train['buy_qty'] - train['sell_qty']) / (train['volume'] + 1e-8)\n\n# NEW FEATURES - Add market pressure indicators\ntrain['buying_pressure'] = train['buy_qty'] / (train['volume'] + 1e-8)\ntrain['selling_pressure'] = train['sell_qty'] / (train['volume'] + 1e-8)\n\n# ADDITIONAL NEW MARKET FEATURES - Liquidity measures\ntrain['total_liquidity'] = train['bid_qty'] + train['ask_qty']\ntrain['liquidity_imbalance'] = (train['bid_qty'] - train['ask_qty']) / (train['total_liquidity'] + 1e-8)\ntrain['relative_spread'] = (train['ask_qty'] - train['bid_qty']) / (train['volume'] + 1e-8)\n\n# ADDITIONAL NEW MARKET FEATURES - Trade intensity\ntrain['trade_intensity'] = (train['buy_qty'] + train['sell_qty']) / (train['volume'] + 1e-8)\ntrain['avg_trade_size'] = train['volume'] / (train['buy_qty'] + train['sell_qty'] + 1e-8)\ntrain['net_trade_flow'] = (train['buy_qty'] - train['sell_qty']) / (train['buy_qty'] + train['sell_qty'] + 1e-8)\n\n# ADDITIONAL NEW MARKET FEATURES - Market depth and activity\ntrain['depth_ratio'] = train['total_liquidity'] / (train['volume'] + 1e-8)\ntrain['volume_participation'] = (train['buy_qty'] + train['sell_qty']) / (train['total_liquidity'] + 1e-8)\ntrain['market_activity'] = train['volume'] * train['total_liquidity']\n\n# ADDITIONAL NEW MARKET FEATURES - Execution quality indicators\ntrain['effective_spread_proxy'] = np.abs(train['buy_qty'] - train['sell_qty']) / (train['volume'] + 1e-8)\ntrain['realized_volatility_proxy'] = np.abs(train['order_flow_imbalance']) * train['volume']\n\n# ADDITIONAL NEW MARKET FEATURES - Normalized volumes\ntrain['normalized_buy_volume'] = train['buy_qty'] / (train['bid_qty'] + 1e-8)\ntrain['normalized_sell_volume'] = train['sell_qty'] / (train['ask_qty'] + 1e-8)\n\n# ADDITIONAL NEW MARKET FEATURES - Complex interactions\ntrain['liquidity_adjusted_imbalance'] = train['order_flow_imbalance'] * train['depth_ratio']\ntrain['pressure_spread_interaction'] = train['buying_pressure'] * train['spread_indicator']\n\n# Replace any inf or -inf values with NaN, then fill NaN with 0\ntrain = train.replace([np.inf, -np.inf], np.nan)\ntrain = train.fillna(0)\n\n# Create same features for test\ntest['bid_ask_interaction'] = test['bid_qty'] * test['ask_qty']\ntest['bid_buy_interaction'] = test['bid_qty'] * test['buy_qty']\ntest['bid_sell_interaction'] = test['bid_qty'] * test['sell_qty']\ntest['ask_buy_interaction'] = test['ask_qty'] * test['buy_qty']\ntest['ask_sell_interaction'] = test['ask_qty'] * test['sell_qty']\ntest['buy_sell_interaction'] = test['buy_qty'] * test['sell_qty']\n\n# Calculate spread indicators for test\ntest['spread_indicator'] = (test['ask_qty'] - test['bid_qty']) / (test['ask_qty'] + test['bid_qty'] + 1e-8)\n\n# Volume-weighted features for test\ntest['volume_weighted_buy'] = test['buy_qty'] * test['volume']\ntest['volume_weighted_sell'] = test['sell_qty'] * test['volume']\ntest['volume_weighted_bid'] = test['bid_qty'] * test['volume']\ntest['volume_weighted_ask'] = test['ask_qty'] * test['volume']\n\n# NEW FEATURES FOR TEST - Add ratio features\ntest['buy_sell_ratio'] = test['buy_qty'] / (test['sell_qty'] + 1e-8)\ntest['bid_ask_ratio'] = test['bid_qty'] / (test['ask_qty'] + 1e-8)\n\n# NEW FEATURES FOR TEST - Add order flow imbalance\ntest['order_flow_imbalance'] = (test['buy_qty'] - test['sell_qty']) / (test['volume'] + 1e-8)\n\n# NEW FEATURES FOR TEST - Add market pressure indicators\ntest['buying_pressure'] = test['buy_qty'] / (test['volume'] + 1e-8)\ntest['selling_pressure'] = test['sell_qty'] / (test['volume'] + 1e-8)\n\n# ADDITIONAL NEW MARKET FEATURES FOR TEST - Liquidity measures\ntest['total_liquidity'] = test['bid_qty'] + test['ask_qty']\ntest['liquidity_imbalance'] = (test['bid_qty'] - test['ask_qty']) / (test['total_liquidity'] + 1e-8)\ntest['relative_spread'] = (test['ask_qty'] - test['bid_qty']) / (test['volume'] + 1e-8)\n\n# ADDITIONAL NEW MARKET FEATURES FOR TEST - Trade intensity\ntest['trade_intensity'] = (test['buy_qty'] + test['sell_qty']) / (test['volume'] + 1e-8)\ntest['avg_trade_size'] = test['volume'] / (test['buy_qty'] + test['sell_qty'] + 1e-8)\ntest['net_trade_flow'] = (test['buy_qty'] - test['sell_qty']) / (test['buy_qty'] + test['sell_qty'] + 1e-8)\n\n# ADDITIONAL NEW MARKET FEATURES FOR TEST - Market depth and activity\ntest['depth_ratio'] = test['total_liquidity'] / (test['volume'] + 1e-8)\ntest['volume_participation'] = (test['buy_qty'] + test['sell_qty']) / (test['total_liquidity'] + 1e-8)\ntest['market_activity'] = test['volume'] * test['total_liquidity']\n\n# ADDITIONAL NEW MARKET FEATURES FOR TEST - Execution quality indicators\ntest['effective_spread_proxy'] = np.abs(test['buy_qty'] - test['sell_qty']) / (test['volume'] + 1e-8)\ntest['realized_volatility_proxy'] = np.abs(test['order_flow_imbalance']) * test['volume']\n\n# ADDITIONAL NEW MARKET FEATURES FOR TEST - Normalized volumes\ntest['normalized_buy_volume'] = test['buy_qty'] / (test['bid_qty'] + 1e-8)\ntest['normalized_sell_volume'] = test['sell_qty'] / (test['ask_qty'] + 1e-8)\n\n# ADDITIONAL NEW MARKET FEATURES FOR TEST - Complex interactions\ntest['liquidity_adjusted_imbalance'] = test['order_flow_imbalance'] * test['depth_ratio']\ntest['pressure_spread_interaction'] = test['buying_pressure'] * test['spread_indicator']\n\n# Replace any inf or -inf values with NaN, then fill NaN with 0\ntest = test.replace([np.inf, -np.inf], np.nan)\ntest = test.fillna(0)\n\nX = train.drop(CFG.target, axis=1)\ny = train[CFG.target]\nX_test = test\n\n# Ensure no inf values in X and X_test\nX = X.replace([np.inf, -np.inf], np.nan).fillna(0)\nX_test = X_test.replace([np.inf, -np.inf], np.nan).fillna(0)\n\n# Force garbage collection before training\ngc.collect()\n\nprint(f\"Training data shape: {X.shape}\")\nprint(f\"Test data shape: {X_test.shape}\")\nprint(f\"Number of features: {X.shape[1]}\")\nprint(\"\\n\")\n","metadata":{"papermill":{"duration":94.918518,"end_time":"2025-05-23T19:05:13.406243","exception":false,"start_time":"2025-05-23T19:03:38.487725","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T07:53:04.760506Z","iopub.execute_input":"2025-05-29T07:53:04.760834Z","iopub.status.idle":"2025-05-29T07:54:59.666295Z","shell.execute_reply.started":"2025-05-29T07:53:04.760811Z","shell.execute_reply":"2025-05-29T07:54:59.665296Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training base models","metadata":{"papermill":{"duration":0.004515,"end_time":"2025-05-23T19:05:13.415995","exception":false,"start_time":"2025-05-23T19:05:13.41148","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class Trainer:\n    def __init__(self, model):\n        self.model = model\n\n    def fit_predict(self, X, y, X_test):\n        print(f\"Training {self.model.__class__.__name__}\\n\")\n\n        fold_scores = []\n        oof_preds = np.zeros(X.shape[0])\n        test_preds = np.zeros(X_test.shape[0])\n\n        split = KFold(n_splits=CFG.n_folds, shuffle=False).split(X, y)\n        for fold_idx, (train_idx, val_idx) in enumerate(split):\n\n            X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n            y_train, y_val = y[train_idx], y[val_idx]\n\n            model = clone(self.model)\n            model.fit(X_train, y_train)\n\n            y_preds = model.predict(X_val)\n            oof_preds[val_idx] = y_preds\n            \n            temp_test_preds = model.predict(X_test)\n            test_preds += temp_test_preds / CFG.n_folds\n\n            fold_score = pearsonr(y_val, y_preds)[0]\n            fold_scores.append(fold_score)\n\n            print(f\"--- Fold {fold_idx} - Score: {fold_score:.6f}\")\n\n            del X_train, y_train, X_val, y_val, y_preds, model, temp_test_preds\n            gc.collect()\n\n        overall_score = pearsonr(y, oof_preds)[0]\n        mean_score = np.mean(fold_scores)\n        std_score = np.std(fold_scores)\n        \n        print(f\"\\n------ Overall Score: {overall_score:.6f} - Mean Score: {mean_score:.6f} ± {std_score:.6f}\")\n        \n        return oof_preds, test_preds, fold_scores\n    \n    def tune(self, X, y):\n        fold_scores = []\n        \n        split = KFold(n_splits=CFG.n_folds, shuffle=False).split(X, y)\n        for train_idx, val_idx in split:\n            X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n            y_train, y_val = y[train_idx], y[val_idx]\n\n            model = clone(self.model)\n            model.fit(X_train, y_train)\n\n            y_preds = model.predict(X_val)\n            fold_score = pearsonr(y_val, y_preds)[0]\n            fold_scores.append(fold_score)\n\n            del X_train, y_train, X_val, y_val, y_preds, model\n            gc.collect()\n\n        return np.mean(fold_scores)","metadata":{"papermill":{"duration":0.01697,"end_time":"2025-05-23T19:05:13.437714","exception":false,"start_time":"2025-05-23T19:05:13.420744","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-05-29T07:54:59.667184Z","iopub.execute_input":"2025-05-29T07:54:59.667421Z","iopub.status.idle":"2025-05-29T07:54:59.680214Z","shell.execute_reply.started":"2025-05-29T07:54:59.667402Z","shell.execute_reply":"2025-05-29T07:54:59.679226Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgbm_goss_params = {\n    \"boosting_type\": \"goss\",\n    \"colsample_bytree\": 0.34695458228489784,\n    \"learning_rate\": 0.031023014900595287,\n    \"min_child_samples\": 30,\n    \"min_child_weight\": 0.4727729225033618,\n    \"n_estimators\": 220,\n    \"n_jobs\": -1,\n    \"num_leaves\": 58,\n    \"random_state\": 42,\n    \"reg_alpha\": 38.665994901468224,\n    \"reg_lambda\": 92.76991677464294,\n    \"subsample\": 0.4810891284493255,\n    \"verbose\": -1\n}\n\nxgb_params = {\n    \"colsample_bylevel\": 0.4778015829774066,\n    \"colsample_bynode\": 0.362764358742407,\n    \"colsample_bytree\": 0.7107423488010493,\n    \"gamma\": 1.7094857725240398,\n    \"learning_rate\": 0.02213323588455387,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1667,\n    \"n_jobs\": -1,\n    \"random_state\": 42,\n    \"reg_alpha\": 39.352415706891264,\n    \"reg_lambda\": 75.44843704068275,\n    \"subsample\": 0.06566669853471274,\n    \"verbosity\": 0\n}\n\n\nlasso_params = {\n    \"alpha\": 0.01,\n    \"fit_intercept\": True,\n    \"max_iter\": 1000,\n    \"tol\": 0.0001,\n    \"selection\": \"cyclic\",\n    \"random_state\": 42\n}\n\nelasticnet_params = {\n    \"alpha\": 0.01,           \n    \"l1_ratio\": 0.5,         \n    \"fit_intercept\": True,\n    \"max_iter\": 1000,\n    \"tol\": 0.0001,\n    \"random_state\": 42\n}","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":0.014435,"end_time":"2025-05-23T19:05:13.457052","exception":false,"start_time":"2025-05-23T19:05:13.442617","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T07:54:59.681355Z","iopub.execute_input":"2025-05-29T07:54:59.681669Z","iopub.status.idle":"2025-05-29T07:54:59.711485Z","shell.execute_reply.started":"2025-05-29T07:54:59.681646Z","shell.execute_reply":"2025-05-29T07:54:59.710285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scores = {}\noof_preds = {}\ntest_preds = {}","metadata":{"papermill":{"duration":0.011866,"end_time":"2025-05-23T19:05:13.473876","exception":false,"start_time":"2025-05-23T19:05:13.46201","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T07:54:59.712357Z","iopub.execute_input":"2025-05-29T07:54:59.712600Z","iopub.status.idle":"2025-05-29T07:54:59.736932Z","shell.execute_reply.started":"2025-05-29T07:54:59.712582Z","shell.execute_reply":"2025-05-29T07:54:59.735972Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LightGBM (goss)","metadata":{"papermill":{"duration":0.004953,"end_time":"2025-05-23T19:35:43.665794","exception":false,"start_time":"2025-05-23T19:35:43.660841","status":"completed"},"tags":[]}},{"cell_type":"code","source":"lgbm_goss_trainer = Trainer(LGBMRegressor(**lgbm_goss_params))\n\noof_preds[\"LightGBM (goss)\"], test_preds[\"LightGBM (goss)\"], scores[\"LightGBM (goss)\"] = lgbm_goss_trainer.fit_predict(X, y, X_test)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":false,"start_time":"2025-05-23T19:35:43.670873","status":"running"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T07:54:59.738116Z","iopub.execute_input":"2025-05-29T07:54:59.738471Z","iopub.status.idle":"2025-05-29T08:05:20.325546Z","shell.execute_reply.started":"2025-05-29T07:54:59.738444Z","shell.execute_reply":"2025-05-29T08:05:20.324266Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## XGBoost","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"xgb_trainer = Trainer(XGBRegressor(**xgb_params))\n\noof_preds[\"XGBoost\"], test_preds[\"XGBoost\"], scores[\"XGBoost\"] = xgb_trainer.fit_predict(X, y, X_test)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T08:05:20.326862Z","iopub.execute_input":"2025-05-29T08:05:20.327245Z","iopub.status.idle":"2025-05-29T09:52:37.751507Z","shell.execute_reply.started":"2025-05-29T08:05:20.327217Z","shell.execute_reply":"2025-05-29T09:52:37.750324Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Lasso","metadata":{}},{"cell_type":"code","source":"lasso_trainer = Trainer(Lasso(**lasso_params))\n\noof_preds[\"Lasso\"], test_preds[\"Lasso\"], scores[\"Lasso\"] = lasso_trainer.fit_predict(X, y, X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T09:52:37.752989Z","iopub.execute_input":"2025-05-29T09:52:37.753375Z","iopub.status.idle":"2025-05-29T10:06:59.448039Z","shell.execute_reply.started":"2025-05-29T09:52:37.753349Z","shell.execute_reply":"2025-05-29T10:06:59.446843Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ElasticNet","metadata":{}},{"cell_type":"code","source":"elasticnet_trainer = Trainer(ElasticNet(**elasticnet_params))\n\noof_preds[\"ElasticNet\"], test_preds[\"ElasticNet\"], scores[\"ElasticNet\"] = elasticnet_trainer.fit_predict(X, y, X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T10:06:59.449287Z","iopub.execute_input":"2025-05-29T10:06:59.449709Z","iopub.status.idle":"2025-05-29T10:24:17.601991Z","shell.execute_reply.started":"2025-05-29T10:06:59.449686Z","shell.execute_reply":"2025-05-29T10:24:17.600976Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ensembling with Ridge","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"X = pd.DataFrame(oof_preds)\nX_test = pd.DataFrame(test_preds)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T10:24:17.606294Z","iopub.execute_input":"2025-05-29T10:24:17.606852Z","iopub.status.idle":"2025-05-29T10:24:17.723796Z","shell.execute_reply.started":"2025-05-29T10:24:17.606828Z","shell.execute_reply":"2025-05-29T10:24:17.722916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective(trial):    \n    params = {\n        \"random_state\": CFG.seed,\n        \"alpha\": trial.suggest_float(\"alpha\", 0, 100),\n        \"tol\": trial.suggest_float(\"tol\", 1e-6, 1e-2)\n    }\n    \n    trainer = Trainer(Ridge(**params))\n    return trainer.tune(X, y)\n\nif CFG.run_optuna:\n    sampler = optuna.samplers.TPESampler(seed=CFG.seed, multivariate=True)\n    study = optuna.create_study(direction=\"maximize\", sampler=sampler)\n    study.optimize(objective, n_trials=CFG.n_optuna_trials, n_jobs=-1, catch=(ValueError,))\n    best_params = study.best_params\n\n    ridge_params = {\n        \"random_state\": CFG.seed,\n        \"alpha\": best_params[\"alpha\"],\n        \"tol\": best_params[\"tol\"]\n    }\nelse:\n    ridge_params = {\n        \"random_state\": CFG.seed\n    }","metadata":{"trusted":true,"_kg_hide-output":true,"scrolled":true,"execution":{"iopub.status.busy":"2025-05-29T10:24:17.724948Z","iopub.execute_input":"2025-05-29T10:24:17.725309Z","iopub.status.idle":"2025-05-29T10:27:33.783037Z","shell.execute_reply.started":"2025-05-29T10:24:17.725278Z","shell.execute_reply":"2025-05-29T10:27:33.782158Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ridge_trainer = Trainer(Ridge(**ridge_params))\n\n_, ridge_test_preds, scores[\"Ridge (ensemble)\"] = ridge_trainer.fit_predict(X, y, X_test)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T10:27:33.784121Z","iopub.execute_input":"2025-05-29T10:27:33.784430Z","iopub.status.idle":"2025-05-29T10:27:34.920879Z","shell.execute_reply.started":"2025-05-29T10:27:33.784410Z","shell.execute_reply":"2025-05-29T10:27:34.919872Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"sub = pd.read_csv(CFG.sample_sub_path)\nsub[\"prediction\"] = ridge_test_preds\nsub.to_csv(\"submission.csv\", index=False)\nsub.head()","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T10:27:34.921960Z","iopub.execute_input":"2025-05-29T10:27:34.922236Z","iopub.status.idle":"2025-05-29T10:27:36.650064Z","shell.execute_reply.started":"2025-05-29T10:27:34.922215Z","shell.execute_reply":"2025-05-29T10:27:36.649078Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Results","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"scores = pd.DataFrame(scores)\nmean_scores = scores.mean().sort_values(ascending=False)\norder = scores.mean().sort_values(ascending=False).index.tolist()\n\nmin_score = mean_scores.min()\nmax_score = mean_scores.max()\npadding = (max_score - min_score) * 0.5\nlower_limit = min_score - padding\nupper_limit = max_score + padding\n\nfig, axs = plt.subplots(1, 2, figsize=(15, scores.shape[1] * 0.5))\n\nboxplot = sns.boxplot(data=scores, order=order, ax=axs[0], orient=\"h\", color=\"grey\")\naxs[0].set_title(f\"Fold Score\")\naxs[0].set_xlabel(\"\")\naxs[0].set_ylabel(\"\")\n\nbarplot = sns.barplot(x=mean_scores.values, y=mean_scores.index, ax=axs[1], color=\"grey\")\naxs[1].set_title(f\"Average Score\")\naxs[1].set_xlabel(\"\")\naxs[1].set_xlim(left=lower_limit, right=upper_limit)\naxs[1].set_ylabel(\"\")\n\nfor i, (score, model) in enumerate(zip(mean_scores.values, mean_scores.index)):\n    color = \"cyan\" if \"ensemble\" in model.lower() else \"grey\"\n    barplot.patches[i].set_facecolor(color)\n    boxplot.patches[i].set_facecolor(color)\n    barplot.text(score, i, round(score, 6), va=\"center\")\n\nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T10:27:36.651168Z","iopub.execute_input":"2025-05-29T10:27:36.652100Z","iopub.status.idle":"2025-05-29T10:27:37.319733Z","shell.execute_reply.started":"2025-05-29T10:27:36.652063Z","shell.execute_reply":"2025-05-29T10:27:37.318659Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Acknowledgements: ","metadata":{}},{"cell_type":"markdown","source":"[https://www.kaggle.com/code/ravaghi/drw-crypto-market-prediction-ensemble](https://www.kaggle.com/code/ravaghi/drw-crypto-market-prediction-ensemble)\n\n[https://www.kaggle.com/code/tayloramarel/drw-crypto-market-prediction-ensemble](https://www.kaggle.com/code/tayloramarel/drw-crypto-market-prediction-ensemble)\n","metadata":{}}]}