{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":88.072356,"end_time":"2025-06-14T11:26:22.99696","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-06-14T11:24:54.924604","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":" # Run Necessary imports\n ","metadata":{}},{"cell_type":"code","source":"import sys\nimport pandas as pd\nimport numpy as np\nfrom xgboost import XGBRegressor\nfrom scipy.stats import pearsonr\nfrom sklearn.linear_model import SGDRegressor\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import StandardScaler","metadata":{"papermill":{"duration":7.230706,"end_time":"2025-06-14T11:25:06.375184","exception":false,"start_time":"2025-06-14T11:24:59.144478","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:24:07.318695Z","iopub.execute_input":"2025-08-21T05:24:07.318962Z","iopub.status.idle":"2025-08-21T05:24:08.246410Z","shell.execute_reply.started":"2025-08-21T05:24:07.318944Z","shell.execute_reply":"2025-08-21T05:24:08.245622Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering ","metadata":{}},{"cell_type":"code","source":"def feature_engineering(df):\n\n    df = df.copy()\n    \n    df['bid_ask_interaction'] = df['ask_qty'] * df['ask_qty']\n    df['bid_buy_interaction'] = df['ask_qty'] * df['buy_qty']\n    df['bid_sell_interaction'] = df['ask_qty'] * df['sell_qty']\n    df['ask_buy_interaction'] = df['ask_qty'] * df['buy_qty']\n    df['ask_sell_interaction'] = df['ask_qty'] * df['sell_qty']\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n\n    df['volume_weighted_sell'] = df['sell_qty'] * df['volume']\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-10)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-10)\n    df['log_volume'] = np.log1p(df['volume'])\n    \n    # Price Pressure Indicators\n    df['net_order_flow'] = df['buy_qty'] - df['sell_qty']\n    df['normalized_net_flow'] = df['net_order_flow'] / (df['volume'] + 1e-10)\n    df['buying_pressure'] = df['buy_qty'] / (df['volume'] + 1e-10)\n    df['volume_weighted_buy'] = df['buy_qty'] * df['volume']\n\n    # Liquidity Depth Measures\n    df['total_depth'] = df['ask_qty'] + df['ask_qty']\n    df['depth_imbalance'] = (df['ask_qty'] - df['ask_qty']) / (df['total_depth'] + 1e-10)\n    df['relative_spread'] = np.abs(df['ask_qty'] - df['ask_qty']) / (df['total_depth'] + 1e-10)\n    df['log_depth'] = np.log1p(df['total_depth'])\n\n     # Order Flow Toxicity Proxies\n    df['kyle_lambda'] = np.abs(df['net_order_flow']) / (df['volume'] + 1e-10)\n    df['flow_toxicity'] = np.abs(df['order_flow_imbalance']) * df['volume']\n    df['aggressive_flow_ratio'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n\n    # Market Activity Indicators\n    df['volume_depth_ratio'] = df['volume'] / (df['total_depth'] + 1e-10)\n    df['activity_intensity'] = (df['buy_qty'] + df['sell_qty']) / (df['volume'] + 1e-10)\n    df['log_buy_qty'] = np.log1p(df['buy_qty'])\n    df['log_sell_qty'] = np.log1p(df['sell_qty'])\n    df['log_bid_qty'] = np.log1p(df['ask_qty'])\n    df['log_ask_qty'] = np.log1p(df['ask_qty'])\n    \n    # Microstructure Volatility Proxies\n    df['realized_spread_proxy'] = 2 * np.abs(df['net_order_flow']) / (df['volume'] + 1e-10)\n    df['price_impact_proxy'] = df['net_order_flow'] / (df['total_depth'] + 1e-10)\n    df['quote_volatility_proxy'] = np.abs(df['depth_imbalance'])\n    \n    # Complex Interaction Terms\n    df['flow_depth_interaction'] = df['net_order_flow'] * df['total_depth']\n    df['imbalance_volume_interaction'] = df['order_flow_imbalance'] * df['volume']\n    df['depth_volume_interaction'] = df['total_depth'] * df['volume']\n    df['buy_sell_spread'] = np.abs(df['buy_qty'] - df['sell_qty'])\n    df['bid_ask_spread'] = np.abs(df['ask_qty'] - df['ask_qty'])\n\n    \n    df = df.fillna(0)\n    df = df.replace([np.inf, -np.inf], np.nan)\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:24:12.098392Z","iopub.execute_input":"2025-08-21T05:24:12.098872Z","iopub.status.idle":"2025-08-21T05:24:12.109686Z","shell.execute_reply.started":"2025-08-21T05:24:12.098848Z","shell.execute_reply":"2025-08-21T05:24:12.108790Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_parquet(\"/kaggle/input/drw-crypto-market-prediction/train.parquet\")\ntrain_df.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:24:15.478651Z","iopub.execute_input":"2025-08-21T05:24:15.479355Z","iopub.status.idle":"2025-08-21T05:24:21.433744Z","shell.execute_reply.started":"2025-08-21T05:24:15.479332Z","shell.execute_reply":"2025-08-21T05:24:21.432959Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Checking the shapes of training data and testing data\n","metadata":{}},{"cell_type":"code","source":"train_df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:24:21.434954Z","iopub.execute_input":"2025-08-21T05:24:21.435179Z","iopub.status.idle":"2025-08-21T05:24:21.439876Z","shell.execute_reply.started":"2025-08-21T05:24:21.435162Z","shell.execute_reply":"2025-08-21T05:24:21.439295Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:24:21.440689Z","iopub.execute_input":"2025-08-21T05:24:21.440942Z","iopub.status.idle":"2025-08-21T05:24:22.685494Z","shell.execute_reply.started":"2025-08-21T05:24:21.440921Z","shell.execute_reply":"2025-08-21T05:24:22.684563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_parquet(\"/kaggle/input/drw-crypto-market-prediction/test.parquet\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:24:23.748318Z","iopub.execute_input":"2025-08-21T05:24:23.749050Z","iopub.status.idle":"2025-08-21T05:24:28.665333Z","shell.execute_reply.started":"2025-08-21T05:24:23.749026Z","shell.execute_reply":"2025-08-21T05:24:28.664734Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:24:28.666340Z","iopub.execute_input":"2025-08-21T05:24:28.666639Z","iopub.status.idle":"2025-08-21T05:24:28.671567Z","shell.execute_reply.started":"2025-08-21T05:24:28.666609Z","shell.execute_reply":"2025-08-21T05:24:28.671059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#importing optuna and train test split\n\nimport optuna\nimport xgboost as xgb\nfrom sklearn.model_selection import train_test_split\nfrom scipy.stats import pearsonr\n\nbase_feature = [\"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"]\n\ntop_100_feature = [\"X683\", \"X140\", \"X758\", \"X425\", \"X752\", \"X344\", \"X292\", \"X646\",\n                   \"X287\", \"X134\", \"X385\", \"X647\", \"X682\", \"X279\", \"X345\", \"X466\",\n                   \"X381\", \"X778\", \"X283\", \"X739\", \"X427\", \"X272\", \"X684\", \"X301\",\n                   \"X198\", \"X465\", \"X608\", \"X738\", \"X384\", \"X137\", \"X386\", \"X581\",\n                   \"X734\", \"X180\", \"X589\", \"X421\", \"X610\", \"X780\", \"X387\", \"X772\",\n                   \"X654\", \"X428\", \"X96\", \"X779\", \"X426\", \"X98\", \"X591\", \"X650\",\n                   \"X613\", \"X566\", \"X605\", \"X181\", \"X750\", \"X174\", \"X288\", \"X607\",\n                   \"X579\", \"X176\", \"X508\", \"X178\", \"X419\", \"X219\", \"X343\", \"X89\",\n                   \"X678\", \"X588\", \"X40\", \"X293\", \"X411\", \"X757\", \"X337\", \"X285\",\n                   \"X295\", \"X341\", \"X443\", \"X179\", \"X575\", \"X751\", \"X92\", \"X562\",\n                   \"X769\", \"X776\", \"X501\", \"X298\", \"X375\", \"X95\", \"X590\", \"X611\",\n                   \"X94\", \"X270\", \"X424\", \"X86\", \"X587\", \"X434\", \"X638\", \"X170\",\n                   \"X297\", \"X136\", \"X97\", \"X572\"]\n\n\ntop_30_feature = top_100_feature[:30]\n\nFEATURES = [*base_feature, *top_30_feature]\n\nLABEL_COLUMN = \"label\"\n\n#X=train_df[FEATURES]\n\n#X_test = test_df[FEATURES]\n\nX =  feature_engineering(train_df[FEATURES])\nX_test =  feature_engineering(test_df[FEATURES])\ny = train_df[LABEL_COLUMN]\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:24:28.672081Z","iopub.execute_input":"2025-08-21T05:24:28.672274Z","iopub.status.idle":"2025-08-21T05:24:30.854497Z","shell.execute_reply.started":"2025-08-21T05:24:28.672259Z","shell.execute_reply":"2025-08-21T05:24:30.853688Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfrom sklearn.model_selection import train_test_split\n\n# Split data \nX_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=42)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:24:38.278983Z","iopub.execute_input":"2025-08-21T05:24:38.279317Z","iopub.status.idle":"2025-08-21T05:24:38.598938Z","shell.execute_reply.started":"2025-08-21T05:24:38.279293Z","shell.execute_reply":"2025-08-21T05:24:38.598297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:24:40.718395Z","iopub.execute_input":"2025-08-21T05:24:40.718996Z","iopub.status.idle":"2025-08-21T05:24:40.723874Z","shell.execute_reply.started":"2025-08-21T05:24:40.718973Z","shell.execute_reply":"2025-08-21T05:24:40.723297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_valid.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:24:42.508716Z","iopub.execute_input":"2025-08-21T05:24:42.508994Z","iopub.status.idle":"2025-08-21T05:24:42.514152Z","shell.execute_reply.started":"2025-08-21T05:24:42.508973Z","shell.execute_reply":"2025-08-21T05:24:42.513306Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:24:44.499639Z","iopub.execute_input":"2025-08-21T05:24:44.499947Z","iopub.status.idle":"2025-08-21T05:24:44.504848Z","shell.execute_reply.started":"2025-08-21T05:24:44.499926Z","shell.execute_reply":"2025-08-21T05:24:44.504243Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_valid.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:24:46.438766Z","iopub.execute_input":"2025-08-21T05:24:46.439622Z","iopub.status.idle":"2025-08-21T05:24:46.444189Z","shell.execute_reply.started":"2025-08-21T05:24:46.439586Z","shell.execute_reply":"2025-08-21T05:24:46.443480Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Hyperparameter tuning\n\ndef objective(trial):\n    \n    params = {\n        \"tree_method\": \"gpu_hist\",\n        \"device\": \"cuda\",\n        \"verbosity\": 0,\n        \"random_state\": 42,\n        \"n_jobs\": -1,\n        \n        # Learning parameters\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.3, log=True),\n        \"n_estimators\": trial.suggest_int(\"n_estimators\", 500, 3000), \n        \n        # Tree structure\n        \"max_depth\": trial.suggest_int(\"max_depth\", 4, 12),  # avoid too shallow\n        \"max_leaves\": trial.suggest_int(\"max_leaves\", 16, 64),  \n        \"min_child_weight\": trial.suggest_int(\"min_child_weight\", 1, 10),  # smaller range\n        \n        # Subsampling (avoid starving trees)\n        \"subsample\": trial.suggest_float(\"subsample\", 0.6, 1.0),\n        \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.6, 1.0),\n        \"colsample_bylevel\": trial.suggest_float(\"colsample_bylevel\", 0.6, 1.0),\n        \"colsample_bynode\": trial.suggest_float(\"colsample_bynode\", 0.6, 1.0),\n        \n        # Regularization (smaller ranges to prevent over-penalizing)\n        \"gamma\": trial.suggest_float(\"gamma\", 0, 3.0),\n        \"reg_alpha\": trial.suggest_float(\"reg_alpha\", 0, 10.0),\n        \"reg_lambda\": trial.suggest_float(\"reg_lambda\", 0, 10.0)\n    }\n    \n\n    \n    \n\n    model = xgb.XGBRegressor(early_stopping_rounds=50,**params)    \n    model.fit(\n            X_train, \n            y_train,\n            eval_set=[(X_valid, y_valid)],\n            verbose=False)\n        \n    preds = model.predict(X_valid)\n    r, _ = pearsonr(y_valid, preds)\n\n    return r\n\n\n# Run Optuna\nstudy = optuna.create_study(direction=\"maximize\")\nstudy.optimize(objective, n_trials=20)\n\nprint(\"Best Pearson correlation:\", study.best_value)\nprint(\"Best Parameters:\", study.best_params)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:24:49.808549Z","iopub.execute_input":"2025-08-21T05:24:49.809149Z","iopub.status.idle":"2025-08-21T05:30:10.166619Z","shell.execute_reply.started":"2025-08-21T05:24:49.809123Z","shell.execute_reply":"2025-08-21T05:30:10.165801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_params = study.best_params\n\n# Add required non-tuned params\n\nbest_params.update({\n    \"tree_method\": \"hist\",\n    \"device\": \"cuda\",\n    \"random_state\": 42,\n    \"n_jobs\": -1,\n    \"early_stopping_rounds\": 50,\n    \"eval_metric\": \"rmse\"\n})\n\nfinal_model = xgb.XGBRegressor(**best_params)\nfinal_model.fit(\n    X_train,\n    y_train,\n    eval_set=[(X_valid, y_valid)],\n    verbose=False\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:32:39.304847Z","iopub.execute_input":"2025-08-21T05:32:39.305580Z","iopub.status.idle":"2025-08-21T05:33:06.243019Z","shell.execute_reply.started":"2025-08-21T05:32:39.305554Z","shell.execute_reply":"2025-08-21T05:33:06.242292Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n\n# predictions on validation data\ny_pred = final_model.predict(X_valid)\n\n# evaluating accuracy metrics\nrmse = np.sqrt(mean_squared_error(y_valid, y_pred))\nmae = mean_absolute_error(y_valid, y_pred)\nr2 = r2_score(y_valid, y_pred)\n\nprint(f\"RMSE: {rmse:.4f}\")\nprint(f\"MAE: {mae:.4f}\")\nprint(f\"R² Score: {r2:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:33:15.698859Z","iopub.execute_input":"2025-08-21T05:33:15.699440Z","iopub.status.idle":"2025-08-21T05:33:16.115461Z","shell.execute_reply.started":"2025-08-21T05:33:15.699416Z","shell.execute_reply":"2025-08-21T05:33:16.114801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# actual vs predicted graph \n\nimport matplotlib.pyplot as plt\n\nplt.scatter(y_valid, y_pred, alpha=0.6)\nplt.plot([y_valid.min(), y_valid.max()], [y_valid.min(), y_valid.max()], 'r--')\nplt.xlabel(\"Actual\")\nplt.ylabel(\"Predicted\")\nplt.title(\"Predicted vs Actual\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:35:07.729259Z","iopub.execute_input":"2025-08-21T05:35:07.730003Z","iopub.status.idle":"2025-08-21T05:35:08.289584Z","shell.execute_reply.started":"2025-08-21T05:35:07.729976Z","shell.execute_reply":"2025-08-21T05:35:08.288796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# predicting on test data\n\npreds = final_model.predict(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:35:20.148899Z","iopub.execute_input":"2025-08-21T05:35:20.149794Z","iopub.status.idle":"2025-08-21T05:35:22.172463Z","shell.execute_reply.started":"2025-08-21T05:35:20.149767Z","shell.execute_reply":"2025-08-21T05:35:22.171835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# gain: total gain of splits using the feature (better measure of usefulness)\nimportance_gain = final_model.get_booster().get_score(importance_type='gain')\n\n# cover: relative number of observations impacted by splits on the feature\nimportance_cover = final_model.get_booster().get_score(importance_type='cover')\n\nimportance_gain_df = pd.DataFrame(list(importance_gain.items()), columns=['Feature', 'Gain']).sort_values('Gain', ascending=False)\nimportance_cover_df = pd.DataFrame(list(importance_cover.items()), columns=['Feature', 'Cover']).sort_values('Cover', ascending=False)\n\nprint(\"Top 20 Features by Gain:\")\ndisplay(importance_gain_df.head(20))\n\nprint(\"Top 20 Features by Cover:\")\ndisplay(importance_cover_df.head(20))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T06:02:07.484650Z","iopub.execute_input":"2025-08-21T06:02:07.484999Z","iopub.status.idle":"2025-08-21T06:02:07.504075Z","shell.execute_reply.started":"2025-08-21T06:02:07.484975Z","shell.execute_reply":"2025-08-21T06:02:07.503233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df = pd.read_csv(\"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\")\nif 'label' in submission_df.columns:\n    submission_df = submission_df.drop(columns=['label'])\nsubmission_df[\"prediction\"] = preds\nsubmission_df.to_csv(\"submission.csv\", index=False)\n    \nprint(\"\\nSubmission file 'submission.csv' created successfully.\")\nprint(submission_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:35:28.259337Z","iopub.execute_input":"2025-08-21T05:35:28.259659Z","iopub.status.idle":"2025-08-21T05:35:29.527431Z","shell.execute_reply.started":"2025-08-21T05:35:28.259636Z","shell.execute_reply":"2025-08-21T05:35:29.526749Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Running the same experiment with time series split ","metadata":{}},{"cell_type":"code","source":"#timeseriessplit \n\nfrom sklearn.model_selection import TimeSeriesSplit\ntscv = TimeSeriesSplit(n_splits=3)\nsplits = list(tscv.split(X))\ntrain_idx, valid_idx = splits[-1]\nX_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\ny_train, y_valid = y.iloc[train_idx], y.iloc[valid_idx]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:37:20.354513Z","iopub.execute_input":"2025-08-21T05:37:20.355020Z","iopub.status.idle":"2025-08-21T05:37:20.477366Z","shell.execute_reply.started":"2025-08-21T05:37:20.354996Z","shell.execute_reply":"2025-08-21T05:37:20.476426Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Hyperparameter tuning\ndef objective1(trial):\n    \n\n    params = {\n        \"tree_method\": \"gpu_hist\",\n        \"device\": \"cuda\",\n        \"verbosity\": 0,\n        \"random_state\": 42,\n        \"n_jobs\": -1,\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.3, log=True),\n        \"max_depth\": trial.suggest_int(\"max_depth\", 3, 20),\n        \"max_leaves\": trial.suggest_int(\"max_leaves\", 8, 64),\n        \"min_child_weight\": trial.suggest_int(\"min_child_weight\", 1, 30),\n        \"subsample\": trial.suggest_float(\"subsample\", 0.3, 1.0),\n        \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.4, 1.0),\n        \"colsample_bylevel\": trial.suggest_float(\"colsample_bylevel\", 0.3, 1.0),\n        \"colsample_bynode\": trial.suggest_float(\"colsample_bynode\", 0.3, 1.0),\n        \"gamma\": trial.suggest_float(\"gamma\", 0, 5.0),\n        \"reg_alpha\": trial.suggest_float(\"reg_alpha\", 0, 50.0),\n        \"reg_lambda\": trial.suggest_float(\"reg_lambda\", 0, 80.0),\n        \"n_estimators\": 1667\n    }\n\n    scores = []\n    \n    for train_idx, valid_idx in tscv.split(X):\n        \n        X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n        y_train, y_valid = y.iloc[train_idx], y.iloc[valid_idx]\n\n        model = xgb.XGBRegressor(early_stopping_rounds=50, **params)\n        \n        model.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], verbose=False)\n        \n        preds = model.predict(X_valid)\n        r, _ = pearsonr(y_valid, preds)\n        scores.append(r)\n\n    return np.mean(scores)\n    \n\n\n# Create and run Optuna study\nstudy = optuna.create_study(direction=\"maximize\")\nstudy.optimize(objective1, n_trials=20)\n\n#  results\nprint(\"Best Pearson correlation:\", study.best_value)\nprint(\"Best Parameters:\", study.best_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:37:23.273906Z","iopub.execute_input":"2025-08-21T05:37:23.274197Z","iopub.status.idle":"2025-08-21T05:39:49.826889Z","shell.execute_reply.started":"2025-08-21T05:37:23.274175Z","shell.execute_reply":"2025-08-21T05:39:49.826117Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_params = study.best_params\n\n# Add required non-tuned params\nbest_params.update({\n    \"tree_method\": \"hist\",\n    \"device\": \"cuda\",\n    \"random_state\": 42,\n    \"n_jobs\": -1,\n    \"early_stopping_rounds\": 50,\n    \"eval_metric\": \"rmse\"\n})\n\n# Suppose last fold train indices:\ntscv = TimeSeriesSplit(n_splits=3)\nsplits = list(tscv.split(X))\ntrain_idx, val_idx = splits[-1]\n\nX_final_train = X.iloc[train_idx]\ny_final_train = y.iloc[train_idx]\n\nfinal_model = xgb.XGBRegressor(**study.best_params)\nfinal_model.fit(X_final_train, y_final_train, verbose=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:40:09.884724Z","iopub.execute_input":"2025-08-21T05:40:09.885026Z","iopub.status.idle":"2025-08-21T05:40:14.399279Z","shell.execute_reply.started":"2025-08-21T05:40:09.885002Z","shell.execute_reply":"2025-08-21T05:40:14.398715Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = final_model.predict(X_valid)\n\nrmse = np.sqrt(mean_squared_error(y_valid, y_pred))\nmae = mean_absolute_error(y_valid, y_pred)\nr2 = r2_score(y_valid, y_pred)\n\nprint(f\"RMSE: {rmse:.4f}\")\nprint(f\"MAE: {mae:.4f}\")\nprint(f\"R² Score: {r2:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:40:25.899543Z","iopub.execute_input":"2025-08-21T05:40:25.899831Z","iopub.status.idle":"2025-08-21T05:40:26.025249Z","shell.execute_reply.started":"2025-08-21T05:40:25.899812Z","shell.execute_reply":"2025-08-21T05:40:26.024397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.scatter(y_valid, y_pred, alpha=0.6)\nplt.plot([y_valid.min(), y_valid.max()], [y_valid.min(), y_valid.max()], 'r--')\nplt.xlabel(\"Actual\")\nplt.ylabel(\"Predicted\")\nplt.title(\"Predicted vs Actual\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:40:44.904395Z","iopub.execute_input":"2025-08-21T05:40:44.904744Z","iopub.status.idle":"2025-08-21T05:40:45.349607Z","shell.execute_reply.started":"2025-08-21T05:40:44.904719Z","shell.execute_reply":"2025-08-21T05:40:45.348891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preds = final_model.predict(X_test)\npreds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:40:50.772873Z","iopub.execute_input":"2025-08-21T05:40:50.773169Z","iopub.status.idle":"2025-08-21T05:40:51.219934Z","shell.execute_reply.started":"2025-08-21T05:40:50.773147Z","shell.execute_reply":"2025-08-21T05:40:51.219321Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df = pd.read_csv(\"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\")\nif 'label' in submission_df.columns:\n    submission_df = submission_df.drop(columns=['label'])\nsubmission_df[\"prediction\"] = preds\nsubmission_df.to_csv(\"submission1.csv\", index=False)\n    \nprint(\"\\nSubmission file 'submission1.csv' created successfully.\")\nprint(submission_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-21T05:40:53.944573Z","iopub.execute_input":"2025-08-21T05:40:53.945221Z","iopub.status.idle":"2025-08-21T05:40:54.929493Z","shell.execute_reply.started":"2025-08-21T05:40:53.945200Z","shell.execute_reply":"2025-08-21T05:40:54.928828Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Private leaderboard**\n\n\n# train test split : 0.02264\n\n# time series split : 0.06136\n\n","metadata":{}},{"cell_type":"markdown","source":"Observations:\n\n1) Adding more technical indicators significantly deteroriates performance. Relationships between indicators and price action change quickly in crypto (what worked in a bull run fails in a bear).\n2) Time series split gives a better performance on unseen data compared to train test split because it maintains the temporal order ","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}