{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.11"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":5456.26661,"end_time":"2025-05-24T10:59:14.597955","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-05-24T09:28:18.331345","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install scikit-learn==1.5.2","metadata":{"_kg_hide-output":true,"papermill":{"duration":9.948495,"end_time":"2025-05-24T09:28:33.60533","exception":false,"start_time":"2025-05-24T09:28:23.656835","status":"completed"},"scrolled":true,"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T17:38:14.914108Z","iopub.execute_input":"2025-05-26T17:38:14.914697Z","iopub.status.idle":"2025-05-26T17:38:19.420465Z","shell.execute_reply.started":"2025-05-26T17:38:14.914660Z","shell.execute_reply":"2025-05-26T17:38:19.419318Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Imports and configs","metadata":{"papermill":{"duration":0.004584,"end_time":"2025-05-24T09:28:33.615072","exception":false,"start_time":"2025-05-24T09:28:33.610488","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn.model_selection import TimeSeriesSplit\nfrom sklearn.linear_model import Ridge\nfrom tqdm import tqdm\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom scipy.stats import pearsonr\nfrom xgboost import XGBRegressor\nfrom sklearn.base import clone\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nimport warnings\nimport optuna\nimport gc\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"_kg_hide-output":true,"papermill":{"duration":10.023958,"end_time":"2025-05-24T09:28:43.643567","exception":false,"start_time":"2025-05-24T09:28:33.619609","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T17:38:19.422413Z","iopub.execute_input":"2025-05-26T17:38:19.422683Z","iopub.status.idle":"2025-05-26T17:38:23.374194Z","shell.execute_reply.started":"2025-05-26T17:38:19.422658Z","shell.execute_reply":"2025-05-26T17:38:23.373355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n    target = \"label\"\n    n_folds = 5\n    seed = 42\n\n    run_optuna = True\n    n_optuna_trials = 250","metadata":{"papermill":{"duration":0.012751,"end_time":"2025-05-24T09:28:43.661332","exception":false,"start_time":"2025-05-24T09:28:43.648581","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T17:38:23.375096Z","iopub.execute_input":"2025-05-26T17:38:23.375741Z","iopub.status.idle":"2025-05-26T17:38:23.381533Z","shell.execute_reply.started":"2025-05-26T17:38:23.375716Z","shell.execute_reply":"2025-05-26T17:38:23.380076Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data loading and preprocessing","metadata":{"papermill":{"duration":0.0045,"end_time":"2025-05-24T09:28:43.670566","exception":false,"start_time":"2025-05-24T09:28:43.666066","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def reduce_mem_usage(dataframe, dataset):    \n    print('Reducing memory usage for:', dataset)\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    \n    for col in dataframe.columns:\n        col_type = dataframe[col].dtype\n\n        c_min = dataframe[col].min()\n        c_max = dataframe[col].max()\n        if str(col_type)[:3] == 'int':\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                dataframe[col] = dataframe[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                dataframe[col] = dataframe[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                dataframe[col] = dataframe[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                dataframe[col] = dataframe[col].astype(np.int64)\n        else:\n            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                dataframe[col] = dataframe[col].astype(np.float16)\n            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                dataframe[col] = dataframe[col].astype(np.float32)\n            else:\n                dataframe[col] = dataframe[col].astype(np.float64)\n\n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    print('--- Memory usage before: {:.2f} MB'.format(initial_mem_usage))\n    print('--- Memory usage after: {:.2f} MB'.format(final_mem_usage))\n    print('--- Decreased memory usage by {:.1f}%\\n'.format(100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage))\n\n    return dataframe","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.017076,"end_time":"2025-05-24T09:28:43.692076","exception":false,"start_time":"2025-05-24T09:28:43.675","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T17:38:23.383857Z","iopub.execute_input":"2025-05-26T17:38:23.384178Z","iopub.status.idle":"2025-05-26T17:38:23.402398Z","shell.execute_reply.started":"2025-05-26T17:38:23.384151Z","shell.execute_reply":"2025-05-26T17:38:23.401411Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_parquet(CFG.train_path).reset_index(drop=True)\ntest = pd.read_parquet(CFG.test_path).reset_index(drop=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T17:38:23.403425Z","iopub.execute_input":"2025-05-26T17:38:23.403702Z","iopub.status.idle":"2025-05-26T17:39:24.714232Z","shell.execute_reply.started":"2025-05-26T17:38:23.403680Z","shell.execute_reply":"2025-05-26T17:39:24.713376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_to_drop = [\n    'X697', 'X698', 'X699', 'X700', 'X701', 'X702', 'X703', 'X704', 'X705', 'X706', \n    'X707', 'X708', 'X709', 'X710', 'X711', 'X712', 'X713', 'X714', 'X715', 'X716',\n    'X717', 'X864', 'X867', 'X869', 'X870', 'X871', 'X872', 'X104', 'X110', 'X116',\n    'X122', 'X128', 'X134', 'X140', 'X146', 'X152', 'X158', 'X164', 'X170', 'X176',\n    'X182', 'X351', 'X357', 'X363', 'X369', 'X375', 'X381', 'X387', 'X393', 'X399',\n    'X405', 'X411', 'X417', 'X423', 'X429'\n]","metadata":{"papermill":{"duration":0.012692,"end_time":"2025-05-24T09:28:43.709591","exception":false,"start_time":"2025-05-24T09:28:43.696899","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T17:39:24.715314Z","iopub.execute_input":"2025-05-26T17:39:24.716002Z","iopub.status.idle":"2025-05-26T17:39:24.721557Z","shell.execute_reply.started":"2025-05-26T17:39:24.715964Z","shell.execute_reply":"2025-05-26T17:39:24.720576Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.drop(columns=cols_to_drop)\ntest = test.drop(columns=[\"label\"] + cols_to_drop)\n\ndef clip_outliers_to_bounds(df, lower_q=0.05, upper_q=0.95):\n    df = df.copy()\n    numeric_cols = df.select_dtypes(include=[np.number]).columns\n\n    # Compute lower and upper quantiles\n    lower_bounds = df[numeric_cols].quantile(lower_q)\n    upper_bounds = df[numeric_cols].quantile(upper_q)\n\n    # Clip values to within bounds\n    for col in numeric_cols:\n        df[col] = df[col].clip(lower=lower_bounds[col], upper=upper_bounds[col])\n\n    print(f\"Outliers beyond {lower_q:.3f}–{upper_q:.3f} quantiles clipped to boundary values.\")\n    return df\n\n# train = clip_outliers_to_bounds(train)\n# test = clip_outliers_to_bounds(test)\n\ntrain = reduce_mem_usage(train, \"train\")\ntest = reduce_mem_usage(test, \"test\")\n\nX = train.drop(CFG.target, axis=1)\ny = train[CFG.target]\nX_test = test","metadata":{"papermill":{"duration":74.723597,"end_time":"2025-05-24T09:29:58.438005","exception":false,"start_time":"2025-05-24T09:28:43.714408","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T17:39:24.722369Z","iopub.execute_input":"2025-05-26T17:39:24.723410Z","iopub.status.idle":"2025-05-26T17:39:42.911432Z","shell.execute_reply.started":"2025-05-26T17:39:24.723375Z","shell.execute_reply":"2025-05-26T17:39:42.910293Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training base models","metadata":{"papermill":{"duration":0.004536,"end_time":"2025-05-24T09:29:58.44762","exception":false,"start_time":"2025-05-24T09:29:58.443084","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class Trainer:\n    def __init__(self, model):\n        self.model = model\n\n    def fit_predict(self, X, y, X_test, split_mode=\"kfold\"):\n        print(f\"Training {self.model.__class__.__name__}\\n\")\n\n        fold_scores = []\n        oof_preds   = np.zeros(X.shape[0])\n        valid_mask  = np.zeros((CFG.n_folds, X.shape[0]), dtype=bool)\n        test_preds = np.zeros(X_test.shape[0])\n\n        fold_oof_preds = []\n        fold_valid_mask = []\n        fold_test_preds = []\n\n        if split_mode == \"kfold\":\n            split = KFold(n_splits=CFG.n_folds, shuffle=False).split(X, y)\n        elif split_mode == \"timefold\":\n            split = TimeSeriesSplit(n_splits=CFG.n_folds).split(X, y)\n            \n        for fold_idx, (train_idx, val_idx) in enumerate(split):\n\n            # if val_idx[-1] < X.shape[0]-1:\n            #    val_idx = np.concatenate((val_idx, np.array(range(val_idx[-1]+1, X.shape[0]))))\n\n            X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n            y_train, y_val = y[train_idx], y[val_idx]\n\n            print(f\"Training with train idx: {train_idx[0]}-{train_idx[-1]}\")\n            print(f\"Training size: {X_train.shape[0]}\")\n            print(f\"Validating with val idx: {val_idx[0]}-{val_idx[-1]}\")\n            print(f\"Validation size: {X_val.shape[0]}\")\n\n            model = clone(self.model)\n\n            model.fit(\n                X_train, \n                y_train, \n                # Prevent overfitting\n                eval_set=[(X_val, y_val)]\n            )\n\n            y_preds = model.predict(X_val)\n            fold_score = pearsonr(y_val, y_preds)[0]\n            fold_scores.append(fold_score)\n\n            oof_preds[val_idx] = y_preds\n            valid_mask[fold_idx, val_idx] = True\n\n            # Additional validation slicing analysis:\n            print(\"Validation time-slice PearsonR scores:\")\n            num_slices = 5\n            val_size = len(val_idx)\n            step = val_size // num_slices\n            for i in range(num_slices):\n                start = i * step\n                end = val_size if i == num_slices - 1 else (i + 1) * step\n                y_slice_true = y_val[start:end]\n                y_slice_pred = y_preds[start:end]\n                slice_score = pearsonr(y_slice_true, y_slice_pred)[0] if len(y_slice_true) > 1 else float('nan')\n                print(f\"  Slice {i+1}: {slice_score:.6f} (idx {val_idx[start]}–{val_idx[end-1]})\")\n\n            temp_test_preds = model.predict(X_test)\n            test_preds += temp_test_preds / CFG.n_folds\n            \n            print(f\"--- Fold {fold_idx} - Score: {fold_score:.6f}\")\n\n            del X_train, y_train, X_val, y_val, y_preds, model, temp_test_preds\n            gc.collect()\n\n        # Filter to rows where all models gave OOF prediction\n        \n        overall_score = pearsonr(y, oof_preds)[0]\n        mean_score = np.mean(fold_scores)\n        std_score = np.std(fold_scores)\n        \n        print(f\"\\n------ Overall Score: {overall_score:.6f} - Mean Score: {mean_score:.6f} ± {std_score:.6f}\")\n        return oof_preds, test_preds, fold_scores, valid_mask \n        \n    def tune(self, X, y):\n        fold_scores = []\n        \n        if split_mode == \"kfold\":\n            split = KFold(n_splits=CFG.n_folds, shuffle=False).split(X, y)\n        elif split_mode == \"timefold\":\n            split = TimeSeriesSplit(n_splits=CFG.n_folds).split(X, y)\n            \n        for train_idx, val_idx in split:\n            X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n            y_train, y_val = y[train_idx], y[val_idx]\n\n            model = clone(self.model)\n            \n            model.fit(\n                X_train, y_train\n            )\n\n            y_preds = model.predict(X_val)\n            fold_score = pearsonr(y_val, y_preds)[0]\n            fold_scores.append(fold_score)\n\n            del X_train, y_train, X_val, y_val, y_preds, model\n            gc.collect()\n\n        return np.mean(fold_scores)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.018079,"end_time":"2025-05-24T09:29:58.470467","exception":false,"start_time":"2025-05-24T09:29:58.452388","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T17:39:42.912340Z","iopub.execute_input":"2025-05-26T17:39:42.912616Z","iopub.status.idle":"2025-05-26T17:39:42.928700Z","shell.execute_reply.started":"2025-05-26T17:39:42.912593Z","shell.execute_reply":"2025-05-26T17:39:42.927635Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgbm_params = {\n    \"boosting_type\": \"gbdt\",\n    \"colsample_bytree\": 0.5242042724303907,\n    \"learning_rate\": 0.014470794293130388,\n    \"min_child_samples\": 47,\n    \"min_child_weight\": 0.1936457311991661,\n    \"n_estimators\": 441,\n    \"n_jobs\": -1,\n    \"num_leaves\": 65,\n    \"random_state\": 42,\n    \"reg_alpha\": 76.69015407123774,\n    \"reg_lambda\": 78.57981723239948,\n    \"subsample\": 0.35497610282716086,\n    \"verbose\": -1,\n    \"early_stopping_rounds\": 100\n}\n\nlgbm_goss_params = {\n    \"boosting_type\": \"goss\",\n    \"colsample_bytree\": 0.32266516869045214,\n    \"learning_rate\": 0.013684657681610528,\n    \"min_child_samples\": 47,\n    \"min_child_weight\": 0.652800548618323,\n    \"n_estimators\": 268,\n    \"n_jobs\": -1,\n    \"num_leaves\": 25,\n    \"random_state\": 42,\n    \"reg_alpha\": 24.43093150663448,\n    \"reg_lambda\": 39.81794248056326,\n    \"subsample\": 0.21026644887863555,\n    \"verbose\": -1,\n    \"early_stopping_rounds\": 100\n}\n\nxgb_params = {\n    \"colsample_bylevel\": 0.4634967322919854,\n    \"colsample_bynode\": 0.6046331585629835,\n    \"colsample_bytree\": 0.11495541333509408,\n    \"gamma\": 1.0397769239502863,\n    \"learning_rate\": 0.09622196913585954,\n    \"max_depth\": 40,\n    \"max_leaves\": 19,\n    \"min_child_weight\": 76,\n    \"n_estimators\": 679,\n    \"n_jobs\": -1,\n    \"random_state\": 42,\n    \"reg_alpha\": 65.41659225037377,\n    \"reg_lambda\": 19.907991015311545,\n    \"subsample\": 0.014465324175810368,\n    \"verbosity\": 0,\n    \"early_stopping_rounds\": 100\n}","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":0.015831,"end_time":"2025-05-24T09:29:58.491399","exception":false,"start_time":"2025-05-24T09:29:58.475568","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T17:39:42.929775Z","iopub.execute_input":"2025-05-26T17:39:42.930105Z","iopub.status.idle":"2025-05-26T17:39:42.951741Z","shell.execute_reply.started":"2025-05-26T17:39:42.930069Z","shell.execute_reply":"2025-05-26T17:39:42.950811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scores = {}\noof_preds = {}\ntest_preds = {}\nvalid_mask = {}\n\nsplit_mode = \"timefold\"","metadata":{"papermill":{"duration":0.011644,"end_time":"2025-05-24T09:29:58.508252","exception":false,"start_time":"2025-05-24T09:29:58.496608","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T17:39:42.954571Z","iopub.execute_input":"2025-05-26T17:39:42.954939Z","iopub.status.idle":"2025-05-26T17:39:42.976650Z","shell.execute_reply.started":"2025-05-26T17:39:42.954916Z","shell.execute_reply":"2025-05-26T17:39:42.975751Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LightGBM (gbdt)","metadata":{"papermill":{"duration":0.004707,"end_time":"2025-05-24T09:29:58.518113","exception":false,"start_time":"2025-05-24T09:29:58.513406","status":"completed"},"tags":[]}},{"cell_type":"code","source":"lgbm_trainer = Trainer(\n    LGBMRegressor(**lgbm_params)\n)\n\noof_preds[\"LightGBM (gbdt)\"], test_preds[\"LightGBM (gbdt)\"], scores[\"LightGBM (gbdt)\"], valid_mask[\"LightGBM (gbdt)\"] = lgbm_trainer.fit_predict(X, y, X_test, split_mode)","metadata":{"papermill":{"duration":1385.087591,"end_time":"2025-05-24T09:53:03.610994","exception":false,"start_time":"2025-05-24T09:29:58.523403","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T17:39:42.977769Z","iopub.execute_input":"2025-05-26T17:39:42.978018Z","iopub.status.idle":"2025-05-26T17:45:56.465446Z","shell.execute_reply.started":"2025-05-26T17:39:42.978001Z","shell.execute_reply":"2025-05-26T17:45:56.464346Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LightGBM (goss)","metadata":{"papermill":{"duration":0.0058,"end_time":"2025-05-24T09:53:03.623067","exception":false,"start_time":"2025-05-24T09:53:03.617267","status":"completed"},"tags":[]}},{"cell_type":"code","source":"lgbm_goss_trainer = Trainer(LGBMRegressor(**lgbm_goss_params))\n\noof_preds[\"LightGBM (goss)\"], test_preds[\"LightGBM (goss)\"], scores[\"LightGBM (goss)\"], valid_mask[\"LightGBM (goss)\"] = lgbm_goss_trainer.fit_predict(X, y, X_test, split_mode)","metadata":{"papermill":{"duration":550.377529,"end_time":"2025-05-24T10:02:14.006266","exception":false,"start_time":"2025-05-24T09:53:03.628737","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T17:45:56.466593Z","iopub.execute_input":"2025-05-26T17:45:56.466948Z","execution_failed":"2025-05-26T17:46:07.050Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## XGBoost","metadata":{"papermill":{"duration":0.006518,"end_time":"2025-05-24T10:02:14.020317","exception":false,"start_time":"2025-05-24T10:02:14.013799","status":"completed"},"tags":[]}},{"cell_type":"code","source":"xgb_trainer = Trainer(XGBRegressor(**xgb_params))\n\noof_preds[\"XGBoost\"], test_preds[\"XGBoost\"], scores[\"XGBoost\"], valid_mask[\"XGBoost\"] = xgb_trainer.fit_predict(X, y, X_test, split_mode)","metadata":{"papermill":{"duration":3174.791446,"end_time":"2025-05-24T10:55:08.817375","exception":false,"start_time":"2025-05-24T10:02:14.025929","status":"completed"},"tags":[],"trusted":true,"execution":{"execution_failed":"2025-05-26T17:46:07.050Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ensembling with Ridge","metadata":{"papermill":{"duration":0.006336,"end_time":"2025-05-24T10:55:08.830352","exception":false,"start_time":"2025-05-24T10:55:08.824016","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# After training the base models\nX = pd.DataFrame(oof_preds)\nX_test = pd.DataFrame(test_preds)\n\n# Filter to rows where all models gave OOF predictions\nmeta_mask = (\n    valid_mask[\"LightGBM (gbdt)\"].any(axis=0) &\n    valid_mask[\"LightGBM (goss)\"].any(axis=0) &\n    valid_mask[\"XGBoost\"].any(axis=0) \n)\n\n# These are your clean stacking datasets\nX_meta = X.loc[meta_mask].reset_index(drop=True)\ny_meta = y.loc[meta_mask].reset_index(drop=True)","metadata":{"papermill":{"duration":0.026911,"end_time":"2025-05-24T10:55:08.863541","exception":false,"start_time":"2025-05-24T10:55:08.83663","status":"completed"},"tags":[],"trusted":true,"execution":{"execution_failed":"2025-05-26T17:46:07.050Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Optuna objective for tuning Ridge\ndef objective(trial):    \n    params = {\n        \"random_state\": CFG.seed,\n        \"alpha\": trial.suggest_float(\"alpha\", 0, 1000),\n        \"tol\": trial.suggest_float(\"tol\", 1e-6, 1e-2),\n        \"fit_intercept\": trial.suggest_categorical(\"fit_intercept\", [True, False]),\n        \"positive\": trial.suggest_categorical(\"positive\", [True, False])\n    }\n    \n    trainer = Trainer(Ridge(**params))\n    # Use the filtered meta data\n    return trainer.tune(X_meta, y_meta)\n\n# Run Optuna tuning\nif CFG.run_optuna:\n    sampler = optuna.samplers.TPESampler(seed=CFG.seed, multivariate=True)\n    study = optuna.create_study(direction=\"maximize\", sampler=sampler)\n    study.optimize(objective, n_trials=CFG.n_optuna_trials, n_jobs=-1, catch=(ValueError,))\n    best_params = study.best_params\n\n    ridge_params = {\n        \"random_state\": CFG.seed,\n        \"alpha\": best_params[\"alpha\"],\n        \"tol\": best_params[\"tol\"],\n        \"fit_intercept\": best_params[\"fit_intercept\"],\n        \"positive\": best_params[\"positive\"]\n    }\nelse:\n    ridge_params = {\n        \"random_state\": CFG.seed\n    }\n","metadata":{"_kg_hide-output":true,"papermill":{"duration":238.312261,"end_time":"2025-05-24T10:59:07.182198","exception":false,"start_time":"2025-05-24T10:55:08.869937","status":"completed"},"scrolled":true,"tags":[],"trusted":true,"execution":{"execution_failed":"2025-05-26T17:46:07.050Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Final training and test prediction\nridge_trainer = Trainer(Ridge(**ridge_params))\noof_ridge_preds, ridge_test_preds, ridge_scores, _ = ridge_trainer.fit_predict(X_meta, y_meta, X_test)\n\nscores[\"Ridge (ensemble)\"] = ridge_scores","metadata":{"papermill":{"duration":1.166491,"end_time":"2025-05-24T10:59:08.368681","exception":false,"start_time":"2025-05-24T10:59:07.20219","status":"completed"},"tags":[],"trusted":true,"execution":{"execution_failed":"2025-05-26T17:46:07.050Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.01791,"end_time":"2025-05-24T10:59:08.404893","exception":false,"start_time":"2025-05-24T10:59:08.386983","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sub = pd.read_csv(CFG.sample_sub_path)\nsub[\"prediction\"] = ridge_test_preds\nsub.to_csv(f\"sub_ridge_{np.mean(scores['Ridge (ensemble)']):.6f}.csv\", index=False)\nsub.head()","metadata":{"papermill":{"duration":1.778356,"end_time":"2025-05-24T10:59:10.201329","exception":false,"start_time":"2025-05-24T10:59:08.422973","status":"completed"},"tags":[],"trusted":true,"execution":{"execution_failed":"2025-05-26T17:46:07.050Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Results","metadata":{"papermill":{"duration":0.018014,"end_time":"2025-05-24T10:59:10.238696","exception":false,"start_time":"2025-05-24T10:59:10.220682","status":"completed"},"tags":[]}},{"cell_type":"code","source":"scores = pd.DataFrame(scores)\nmean_scores = scores.mean().sort_values(ascending=False)\norder = scores.mean().sort_values(ascending=False).index.tolist()\n\nmin_score = mean_scores.min()\nmax_score = mean_scores.max()\npadding = (max_score - min_score) * 0.5\nlower_limit = min_score - padding\nupper_limit = max_score + padding\n\nfig, axs = plt.subplots(1, 2, figsize=(15, scores.shape[1] * 0.5))\n\nboxplot = sns.boxplot(data=scores, order=order, ax=axs[0], orient=\"h\", color=\"grey\")\naxs[0].set_title(f\"Fold Score\")\naxs[0].set_xlabel(\"\")\naxs[0].set_ylabel(\"\")\n\nbarplot = sns.barplot(x=mean_scores.values, y=mean_scores.index, ax=axs[1], color=\"grey\")\naxs[1].set_title(f\"Average Score\")\naxs[1].set_xlabel(\"\")\naxs[1].set_xlim(left=lower_limit, right=upper_limit)\naxs[1].set_ylabel(\"\")\n\nfor i, (score, model) in enumerate(zip(mean_scores.values, mean_scores.index)):\n    color = \"cyan\" if \"ensemble\" in model.lower() else \"grey\"\n    barplot.patches[i].set_facecolor(color)\n    boxplot.patches[i].set_facecolor(color)\n    barplot.text(score, i, round(score, 6), va=\"center\")\n\nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":0.474663,"end_time":"2025-05-24T10:59:10.731667","exception":false,"start_time":"2025-05-24T10:59:10.257004","status":"completed"},"tags":[],"trusted":true,"execution":{"execution_failed":"2025-05-26T17:46:07.050Z"}},"outputs":[],"execution_count":null}]}