{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.11"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"},{"sourceId":243952999,"sourceType":"kernelVersion"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":9319.126687,"end_time":"2025-05-28T09:27:09.132450","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-05-28T06:51:50.005763","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install koolbox scikit-learn==1.5.2","metadata":{"_kg_hide-output":true,"papermill":{"duration":9.880461,"end_time":"2025-05-28T06:52:05.221127","exception":false,"start_time":"2025-05-28T06:51:55.340666","status":"completed"},"scrolled":true,"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Imports and configs","metadata":{"papermill":{"duration":0.004954,"end_time":"2025-05-28T06:52:05.231905","exception":false,"start_time":"2025-05-28T06:52:05.226951","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn.linear_model import Ridge\nfrom lightgbm import LGBMRegressor\nfrom scipy.stats import pearsonr as pr\nfrom xgboost import XGBRegressor\nfrom sklearn.base import clone\nfrom koolbox import Trainer\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nimport warnings\nimport optuna\nimport joblib\nimport glob\nimport gc\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"_kg_hide-output":true,"papermill":{"duration":9.349738,"end_time":"2025-05-28T06:52:14.586711","exception":false,"start_time":"2025-05-28T06:52:05.236973","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n    target = \"label\"\n    n_folds = 5\n    seed = 42\n\n    run_optuna = True\n    n_optuna_trials = 500","metadata":{"papermill":{"duration":0.012421,"end_time":"2025-05-28T06:52:14.604593","exception":false,"start_time":"2025-05-28T06:52:14.592172","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data loading and preprocessing","metadata":{"papermill":{"duration":0.004888,"end_time":"2025-05-28T06:52:14.614896","exception":false,"start_time":"2025-05-28T06:52:14.610008","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def reduce_mem_usage(dataframe, dataset):    \n    print('Reducing memory usage for:', dataset)\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    \n    for col in dataframe.columns:\n        col_type = dataframe[col].dtype\n\n        c_min = dataframe[col].min()\n        c_max = dataframe[col].max()\n        if str(col_type)[:3] == 'int':\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                dataframe[col] = dataframe[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                dataframe[col] = dataframe[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                dataframe[col] = dataframe[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                dataframe[col] = dataframe[col].astype(np.int64)\n        else:\n            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                dataframe[col] = dataframe[col].astype(np.float16)\n            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                dataframe[col] = dataframe[col].astype(np.float32)\n            else:\n                dataframe[col] = dataframe[col].astype(np.float64)\n\n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    print('--- Memory usage before: {:.2f} MB'.format(initial_mem_usage))\n    print('--- Memory usage after: {:.2f} MB'.format(final_mem_usage))\n    print('--- Decreased memory usage by {:.1f}%\\n'.format(100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage))\n\n    return dataframe","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.016341,"end_time":"2025-05-28T06:52:14.636335","exception":false,"start_time":"2025-05-28T06:52:14.619994","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_parquet(CFG.train_path).reset_index(drop=True)\ntest = pd.read_parquet(CFG.test_path).reset_index(drop=True)\n\ntrain = reduce_mem_usage(train, \"train\")\ntest = reduce_mem_usage(test, \"test\")\n\nX = train.drop(CFG.target, axis=1)\ny = train[CFG.target]\nX_test = test.drop(CFG.target, axis=1)","metadata":{"papermill":{"duration":70.181095,"end_time":"2025-05-28T06:53:24.866518","exception":false,"start_time":"2025-05-28T06:52:14.685423","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features = [\n    # https://www.kaggle.com/code/sadettinamilverdil/yat-r-m-tavsiyesi-de-ildir\n    \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n    \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n    \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"\n]\n\nX = X[features]\nX_test = X_test[features]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training base models","metadata":{"papermill":{"duration":0.005181,"end_time":"2025-05-28T06:53:31.106911","exception":false,"start_time":"2025-05-28T06:53:31.101730","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def pearsonr(y_true, y_pred):\n    return pr(y_true, y_pred)[0]","metadata":{"papermill":{"duration":0.012196,"end_time":"2025-05-28T06:53:31.124498","exception":false,"start_time":"2025-05-28T06:53:31.112302","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgbm_params = {\n    \"boosting_type\": \"gbdt\",\n    \"colsample_bytree\": 0.5625888953382505,\n    \"learning_rate\": 0.029312951475451557,\n    \"min_child_samples\": 63,\n    \"min_child_weight\": 0.11456572852335424,\n    \"n_estimators\": 126,\n    \"n_jobs\": -1,\n    \"num_leaves\": 37,\n    \"random_state\": 42,\n    \"reg_alpha\": 85.2476527854083,\n    \"reg_lambda\": 99.38305361388907,\n    \"subsample\": 0.450669817684892,\n    \"verbose\": -1\n}\n\nlgbm_goss_params = {\n    \"boosting_type\": \"goss\",\n    \"colsample_bytree\": 0.34695458228489784,\n    \"learning_rate\": 0.031023014900595287,\n    \"min_child_samples\": 30,\n    \"min_child_weight\": 0.4727729225033618,\n    \"n_estimators\": 220,\n    \"n_jobs\": -1,\n    \"num_leaves\": 58,\n    \"random_state\": 42,\n    \"reg_alpha\": 38.665994901468224,\n    \"reg_lambda\": 92.76991677464294,\n    \"subsample\": 0.4810891284493255,\n    \"verbose\": -1\n}\n\nxgb_params = {\n    \"colsample_bylevel\": 0.4778015829774066,\n    \"colsample_bynode\": 0.362764358742407,\n    \"colsample_bytree\": 0.7107423488010493,\n    \"gamma\": 1.7094857725240398,\n    \"learning_rate\": 0.02213323588455387,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1667,\n    \"n_jobs\": -1,\n    \"random_state\": 42,\n    \"reg_alpha\": 39.352415706891264,\n    \"reg_lambda\": 75.44843704068275,\n    \"subsample\": 0.06566669853471274,\n    \"verbosity\": 0\n}","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":0.016101,"end_time":"2025-05-28T06:53:31.146099","exception":false,"start_time":"2025-05-28T06:53:31.129998","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fold_scores = {}\noverall_scores = {}\n\noof_preds = {}\ntest_preds = {}","metadata":{"papermill":{"duration":0.011971,"end_time":"2025-05-28T06:53:31.163714","exception":false,"start_time":"2025-05-28T06:53:31.151743","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LightGBM (gbdt)","metadata":{"papermill":{"duration":0.005429,"end_time":"2025-05-28T06:53:31.175035","exception":false,"start_time":"2025-05-28T06:53:31.169606","status":"completed"},"tags":[]}},{"cell_type":"code","source":"lgbm_trainer = Trainer(\n    LGBMRegressor(**lgbm_params),\n    cv=KFold(n_splits=5, shuffle=False),\n    metric=pearsonr,\n    task=\"regression\",\n    metric_precision=6\n)\n\nlgbm_trainer.fit(X, y)\n\nfold_scores[\"LightGBM (gbdt)\"] = lgbm_trainer.fold_scores\noverall_scores[\"LightGBM (gbdt)\"] = [pearsonr(lgbm_trainer.oof_preds, y)]\noof_preds[\"LightGBM (gbdt)\"] = lgbm_trainer.oof_preds\ntest_preds[\"LightGBM (gbdt)\"] = lgbm_trainer.predict(X_test)","metadata":{"papermill":{"duration":546.840654,"end_time":"2025-05-28T07:02:38.021118","exception":false,"start_time":"2025-05-28T06:53:31.180464","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LightGBM (goss)","metadata":{"papermill":{"duration":0.005909,"end_time":"2025-05-28T07:02:38.035090","exception":false,"start_time":"2025-05-28T07:02:38.029181","status":"completed"},"tags":[]}},{"cell_type":"code","source":"lgbm_goss_trainer = Trainer(\n    LGBMRegressor(**lgbm_goss_params),\n    cv=KFold(n_splits=5, shuffle=False),\n    metric=pearsonr,\n    task=\"regression\",\n    metric_precision=6\n)\n\nlgbm_goss_trainer.fit(X, y)\n\nfold_scores[\"LightGBM (goss)\"] = lgbm_goss_trainer.fold_scores\noverall_scores[\"LightGBM (goss)\"] = [pearsonr(lgbm_goss_trainer.oof_preds, y)]\noof_preds[\"LightGBM (goss)\"] = lgbm_goss_trainer.oof_preds\ntest_preds[\"LightGBM (goss)\"] = lgbm_goss_trainer.predict(X_test)","metadata":{"papermill":{"duration":601.870551,"end_time":"2025-05-28T07:12:39.911607","exception":false,"start_time":"2025-05-28T07:02:38.041056","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## XGBoost","metadata":{"papermill":{"duration":0.006095,"end_time":"2025-05-28T07:12:39.924720","exception":false,"start_time":"2025-05-28T07:12:39.918625","status":"completed"},"tags":[]}},{"cell_type":"code","source":"xgb_trainer = Trainer(\n    XGBRegressor(**xgb_params),\n    cv=KFold(n_splits=5, shuffle=False),\n    metric=pearsonr,\n    task=\"regression\",\n    metric_precision=6\n)\n\nxgb_trainer.fit(X, y)\n\nfold_scores[\"XGBoost\"] = xgb_trainer.fold_scores\noverall_scores[\"XGBoost\"] = [pearsonr(xgb_trainer.oof_preds, y)]\noof_preds[\"XGBoost\"] = xgb_trainer.oof_preds\ntest_preds[\"XGBoost\"] = xgb_trainer.predict(X_test)","metadata":{"papermill":{"duration":7831.931128,"end_time":"2025-05-28T09:23:11.862150","exception":false,"start_time":"2025-05-28T07:12:39.931022","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## AutoGluon","metadata":{}},{"cell_type":"code","source":"oof_preds_files = glob.glob(f'/kaggle/input/drw-crypto-market-prediction-autogluon/*_oof_preds_*.pkl')\ntest_preds_files = glob.glob(f'/kaggle/input/drw-crypto-market-prediction-autogluon/*_test_preds_*.pkl')\n\nag_oof_preds = joblib.load(oof_preds_files[0])\nag_test_preds = joblib.load(test_preds_files[0])\n\nag_score = [pearsonr(ag_oof_preds, y)]\n\nag_scores = []\nsplit = KFold(n_splits=CFG.n_folds).split(X, y)\nfor _, val_idx in split:\n    y_val = y[val_idx]\n    y_preds = ag_oof_preds[val_idx]   \n    score = pearsonr(y_preds, y_val)\n    ag_scores.append(score)\n    \noof_preds[\"AutoGluon\"], test_preds[\"AutoGluon\"], overall_scores[\"AutoGluon\"], fold_scores[\"AutoGluon\"] = ag_oof_preds, ag_test_preds, ag_score, ag_scores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ensembling with Ridge","metadata":{"papermill":{"duration":0.006948,"end_time":"2025-05-28T09:23:11.877930","exception":false,"start_time":"2025-05-28T09:23:11.870982","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def plot_weights(weights, title):\n    sorted_indices = np.argsort(weights[0])[::-1]\n    sorted_coeffs = np.array(weights[0])[sorted_indices]\n    sorted_model_names = np.array(list(oof_preds.keys()))[sorted_indices]\n\n    plt.figure(figsize=(10, weights.shape[1] * 0.5))\n    ax = sns.barplot(x=sorted_coeffs, y=sorted_model_names, palette=\"RdYlGn_r\")\n\n    for i, (value, name) in enumerate(zip(sorted_coeffs, sorted_model_names)):\n        if value >= 0:\n            ax.text(value, i, f\"{value:.3f}\", va=\"center\", ha=\"left\", color=\"black\")\n        else:\n            ax.text(value, i, f\"{value:.3f}\", va=\"center\", ha=\"right\", color=\"black\")\n\n    xlim = ax.get_xlim()\n    ax.set_xlim(xlim[0] - 0.1 * abs(xlim[0]), xlim[1] + 0.1 * abs(xlim[1]))\n\n    plt.title(title)\n    plt.xlabel(\"\")\n    plt.ylabel(\"\")\n    plt.tight_layout()\n    plt.show()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.019582,"end_time":"2025-05-28T09:23:11.904119","exception":false,"start_time":"2025-05-28T09:23:11.884537","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = pd.DataFrame(oof_preds)\nX_test = pd.DataFrame(test_preds)","metadata":{"papermill":{"duration":0.115877,"end_time":"2025-05-28T09:23:12.026966","exception":false,"start_time":"2025-05-28T09:23:11.911089","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"joblib.dump(X, \"oof_preds.pkl\")\njoblib.dump(X_test, \"test_preds.pkl\")","metadata":{"papermill":{"duration":0.13148,"end_time":"2025-05-28T09:23:12.165729","exception":false,"start_time":"2025-05-28T09:23:12.034249","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective(trial):    \n    params = {\n        \"random_state\": CFG.seed,\n        \"alpha\": trial.suggest_float(\"alpha\", 0, 1),\n        \"tol\": trial.suggest_float(\"tol\", 1e-6, 1e-2),\n        \"fit_intercept\": trial.suggest_categorical(\"fit_intercept\", [True, False]),\n        \"positive\": trial.suggest_categorical(\"positive\", [True, False])\n    }\n\n    trainer = Trainer(\n        Ridge(**params),\n        cv=KFold(n_splits=5, shuffle=False),\n        metric=pearsonr,\n        task=\"regression\",\n        verbose=False\n    )\n    trainer.fit(X, y)\n    \n    return pearsonr(trainer.oof_preds, y)\n\nif CFG.run_optuna:\n    sampler = optuna.samplers.TPESampler(seed=CFG.seed, multivariate=True, n_startup_trials=CFG.n_optuna_trials // 10)\n    study = optuna.create_study(direction=\"maximize\", sampler=sampler)\n    study.optimize(objective, n_trials=CFG.n_optuna_trials, n_jobs=-1, catch=(ValueError,))\n    best_params = study.best_params\n\n    ridge_params = {\n        \"random_state\": CFG.seed,\n        \"alpha\": best_params[\"alpha\"],\n        \"tol\": best_params[\"tol\"],\n        \"fit_intercept\": best_params[\"fit_intercept\"],\n        \"positive\": best_params[\"positive\"]\n    }\nelse:\n    ridge_params = {\n        \"random_state\": CFG.seed\n    }","metadata":{"_kg_hide-output":true,"papermill":{"duration":229.358176,"end_time":"2025-05-28T09:27:01.531060","exception":false,"start_time":"2025-05-28T09:23:12.172884","status":"completed"},"scrolled":true,"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ridge_trainer = Trainer(\n    Ridge(**ridge_params),\n    cv=KFold(n_splits=5, shuffle=False),\n    metric=pearsonr,\n    task=\"regression\",\n    metric_precision=6\n)\n\nridge_trainer.fit(X, y)\n\nfold_scores[\"Ridge (ensemble)\"] = ridge_trainer.fold_scores\noverall_scores[\"Ridge (ensemble)\"] = [pearsonr(ridge_trainer.oof_preds, y)]\nridge_test_preds = ridge_trainer.predict(X_test)","metadata":{"papermill":{"duration":1.116192,"end_time":"2025-05-28T09:27:02.667165","exception":false,"start_time":"2025-05-28T09:27:01.550973","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ridge_coeffs = np.zeros((1, X.shape[1]))\nfor m in ridge_trainer.estimators:\n    ridge_coeffs += m.coef_\nridge_coeffs = ridge_coeffs / len(ridge_trainer.estimators)\n\nplot_weights(ridge_coeffs, \"Ridge Coefficients\")","metadata":{"papermill":{"duration":0.381603,"end_time":"2025-05-28T09:27:03.067767","exception":false,"start_time":"2025-05-28T09:27:02.686164","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.019284,"end_time":"2025-05-28T09:27:03.107042","exception":false,"start_time":"2025-05-28T09:27:03.087758","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sub = pd.read_csv(CFG.sample_sub_path)\nsub[\"prediction\"] = ridge_test_preds\nsub.to_csv(f\"sub_ridge_{overall_scores['Ridge (ensemble)'][0]:.6f}.csv\", index=False)\nsub.head()","metadata":{"papermill":{"duration":1.91128,"end_time":"2025-05-28T09:27:05.038651","exception":false,"start_time":"2025-05-28T09:27:03.127371","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Results","metadata":{"papermill":{"duration":0.020011,"end_time":"2025-05-28T09:27:05.079631","exception":false,"start_time":"2025-05-28T09:27:05.059620","status":"completed"},"tags":[]}},{"cell_type":"code","source":"fold_scores = pd.DataFrame(fold_scores)\noverall_scores = pd.DataFrame(overall_scores).transpose().sort_values(by=0, ascending=False)\norder = overall_scores.index.tolist()\n\nmin_score = overall_scores.values.flatten().min()\nmax_score = overall_scores.values.flatten().max()\npadding = (max_score - min_score) * 0.5\nlower_limit = min_score - padding\nupper_limit = max_score + padding\n\nfig, axs = plt.subplots(1, 2, figsize=(15, fold_scores.shape[1] * 0.5))\n\nboxplot = sns.boxplot(data=fold_scores, order=order, ax=axs[0], orient=\"h\", color=\"grey\")\naxs[0].set_title(f\"Fold Score\")\naxs[0].set_xlabel(\"\")\naxs[0].set_ylabel(\"\")\n\nbarplot = sns.barplot(x=overall_scores.values.flatten(), y=overall_scores.index, ax=axs[1], color=\"grey\")\naxs[1].set_title(f\"Overall Score\")\naxs[1].set_xlabel(\"\")\naxs[1].set_xlim(left=lower_limit, right=upper_limit)\naxs[1].set_ylabel(\"\")\n\nfor i, (score, model) in enumerate(zip(overall_scores.values.flatten(), overall_scores.index)):\n    color = \"cyan\" if \"ensemble\" in model.lower() else \"grey\"\n    barplot.patches[i].set_facecolor(color)\n    boxplot.patches[i].set_facecolor(color)\n    barplot.text(score, i, round(score, 6), va=\"center\")\n\nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":0.356656,"end_time":"2025-05-28T09:27:05.456995","exception":false,"start_time":"2025-05-28T09:27:05.100339","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}