{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\nimport polars.selectors as cs\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.base import BaseEstimator, RegressorMixin\n\n\nimport optuna\n\nimport xgboost as xgb\nimport time\nfrom scipy.stats import pearsonr\n\nimport gc \nimport functools\n\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nfrom typing import Union, List","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-24T09:49:10.265595Z","iopub.execute_input":"2025-07-24T09:49:10.265908Z","iopub.status.idle":"2025-07-24T09:49:12.719791Z","shell.execute_reply.started":"2025-07-24T09:49:10.265879Z","shell.execute_reply":"2025-07-24T09:49:12.718869Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Custom Evaluation Metrics","metadata":{}},{"cell_type":"code","source":"def _pearsonr(y_true, y_pred):\n    return pearsonr(y_true, y_pred)[0]\n    \ndef pearsonr_coeff(preds, data):\n    y_true = data.get_label()\n    valid_score = _pearsonr(y_true, preds)\n    return 'pearsonr_coeff_score', valid_score\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T04:32:59.167474Z","iopub.execute_input":"2025-07-24T04:32:59.167740Z","iopub.status.idle":"2025-07-24T04:32:59.180398Z","shell.execute_reply.started":"2025-07-24T04:32:59.167725Z","shell.execute_reply":"2025-07-24T04:32:59.179618Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Custom Ensemble Regressor","metadata":{}},{"cell_type":"code","source":"class VotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n\n    def fit(self, X, y=None):\n        # No training in VotingModel since estimators are pre-trained\n        return self\n\n    def predict(self, X):\n        y_preds = []\n        for estimator in self.estimators:\n            # If the estimator is an XGBoost Booster, convert DataFrame to DMatrix\n            if isinstance(estimator, xgb.core.Booster):\n                X_converted = xgb.DMatrix(X)\n            else:\n                X_converted = X\n            y_preds.append(estimator.predict(X_converted))\n        return np.mean(y_preds, axis=0)\n\n    def predict_proba(self, X):\n        y_preds = []\n        for estimator in self.estimators:\n            if hasattr(estimator, \"predict_proba\"):\n                y_preds.append(estimator.predict_proba(X))\n            else:\n                raise AttributeError(\"One of the estimators does not support predict_proba.\")\n        return np.mean(y_preds, axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T04:32:59.181278Z","iopub.execute_input":"2025-07-24T04:32:59.181782Z","iopub.status.idle":"2025-07-24T04:32:59.198810Z","shell.execute_reply.started":"2025-07-24T04:32:59.181757Z","shell.execute_reply":"2025-07-24T04:32:59.198096Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Memory Optimization","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\"\n    Optimizes the memory usage of a DataFrame by downcasting numeric columns to smaller data types.\n\n    Parameters:\n    - df: DataFrame to be optimized\n\n    Returns:\n    - df: Optimized DataFrame\n    \"\"\"\n\n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type) == \"category\":\n            continue\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if (\n                    c_min > np.finfo(np.float16).min\n                    and c_max < np.finfo(np.float16).max\n                ):\n                    df[col] = df[col].astype(np.float16)\n                elif (\n                    c_min > np.finfo(np.float32).min\n                    and c_max < np.finfo(np.float32).max\n                ):\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            continue\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T04:32:59.200103Z","iopub.execute_input":"2025-07-24T04:32:59.200278Z","iopub.status.idle":"2025-07-24T04:32:59.216276Z","shell.execute_reply.started":"2025-07-24T04:32:59.200265Z","shell.execute_reply":"2025-07-24T04:32:59.215504Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Configuration","metadata":{}},{"cell_type":"code","source":"class Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    FEATURES = ['X363', 'X321', 'X405', 'X730', 'X523', 'X756', 'X589', 'X462', 'X779',\n                'X25', 'X532', 'X520', 'X329', 'X383', 'X751', 'X535', 'X639', 'X596', 'X761',\n                'X752', 'X287', 'X298', 'X759', 'X302', 'X55', 'X56', 'X52', 'X303', 'X51',\n                'X598', 'X385', 'X603', 'X674', 'X415', 'X345', 'X174', 'X178', 'X168', 'X612',\n                'bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\n\n    LABEL_COLUMN = \"label\"\n    RANDOM_STATE = 42","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T04:32:59.217166Z","iopub.execute_input":"2025-07-24T04:32:59.217456Z","iopub.status.idle":"2025-07-24T04:32:59.233157Z","shell.execute_reply.started":"2025-07-24T04:32:59.217440Z","shell.execute_reply":"2025-07-24T04:32:59.232587Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"def feature_engineering(df):\n    # Create a stable working copy to avoid modifying the original DataFrame\n    df = df.copy()\n    epsilon = np.finfo(float).eps  # Very small constant to prevent divide-by-zero errors\n\n    # Replace NaNs with zeros to prevent issues during feature construction\n    df.fillna(0, inplace=True)\n\n    # Interaction Features\n    # Capture multiplicative effects between different order quantities\n    new_features = {\n        'bid_ask_interaction': df['bid_qty'] * df['ask_qty'],\n        'bid_buy_interaction': df['bid_qty'] * df['buy_qty'],\n        'bid_sell_interaction': df['bid_qty'] * df['sell_qty'],\n        'ask_buy_interaction': df['ask_qty'] * df['buy_qty'],\n        'ask_sell_interaction': df['ask_qty'] * df['sell_qty'],\n        'buy_sell_interaction': df['buy_qty'] * df['sell_qty'],\n\n        # Spread Indicator\n        # Measures the difference between ask and bid sizes normalized by their total\n        'spread_indicator': (df['ask_qty'] - df['bid_qty']) / (df['ask_qty'] + df['bid_qty'] + epsilon),\n\n        # Volume-Weighted Quantities\n        'volume_weighted_buy': df['buy_qty'] * df['volume'],\n        'volume_weighted_sell': df['sell_qty'] * df['volume'],\n        'volume_weighted_bid': df['bid_qty'] * df['volume'],\n        'volume_weighted_ask': df['ask_qty'] * df['volume'],\n\n        # Ratio Features\n        'buy_sell_ratio': df['buy_qty'] / (df['sell_qty'] + epsilon),\n        'bid_ask_ratio': df['bid_qty'] / (df['ask_qty'] + epsilon),\n\n        # Order Flow & Market Pressure\n        'order_flow_imbalance': (df['buy_qty'] - df['sell_qty']) / (df['volume'] + epsilon),\n        'buying_pressure': df['buy_qty'] / (df['volume'] + epsilon),\n        'selling_pressure': df['sell_qty'] / (df['volume'] + epsilon)\n    }\n\n    # Add new features into the DataFrame efficiently\n    df = df.assign(**new_features)\n\n    # Liquidity Measures\n    df['total_liquidity'] = df['bid_qty'] + df['ask_qty']\n    df['liquidity_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_liquidity'] + epsilon)\n    df['relative_spread'] = (df['ask_qty'] - df['bid_qty']) / (df['volume'] + epsilon)\n\n    # Trade Activity & Intensity\n    df['trade_intensity'] = (df['buy_qty'] + df['sell_qty']) / (df['volume'] + epsilon)\n    df['avg_trade_size'] = df['volume'] / (df['buy_qty'] + df['sell_qty'] + epsilon)\n    df['net_trade_flow'] = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + epsilon)\n\n    # Market Depth & Engagement\n    df['depth_ratio'] = df['total_liquidity'] / (df['volume'] + epsilon)\n    df['volume_participation'] = (df['buy_qty'] + df['sell_qty']) / (df['total_liquidity'] + epsilon)\n    df['market_activity'] = df['volume'] * df['total_liquidity']\n\n    # Execution Quality Metrics\n    df['effective_spread_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + epsilon)\n    df['realized_volatility_proxy'] = np.abs(df['order_flow_imbalance']) * df['volume']\n\n    # Normalized Volumes\n    df['normalized_buy_volume'] = df['buy_qty'] / (df['bid_qty'] + epsilon)\n    df['normalized_sell_volume'] = df['sell_qty'] / (df['ask_qty'] + epsilon)\n\n    # Advanced Interactions\n    df['liquidity_adjusted_imbalance'] = df['order_flow_imbalance'] * df['depth_ratio']\n    df['pressure_spread_interaction'] = df['buying_pressure'] * df['spread_indicator']\n\n    # Depth Ratio & Imbalance\n    df['bid_depth_ratio'] = df['bid_qty'] / (df['volume'] + 1e-8)\n    df['ask_depth_ratio'] = df['ask_qty'] / (df['volume'] + 1e-8)\n    df['depth_imbalance'] = (df['bid_depth_ratio'] - df['ask_depth_ratio']) / (df['depth_ratio'] + 1e-8)\n\n    # Illiquidity Metrics\n    df['net_order_flow'] = df['buy_qty'] - df['sell_qty']\n    df['kyle_lambda'] = np.abs(df['net_order_flow']) / (df['volume'] + 1e-8)\n    df['net_pressure'] = df['buying_pressure'] - df['selling_pressure']\n    df['amihud_illiquidity'] = np.abs(df['net_pressure']) / (df['volume'] + 1e-8)\n    df['liquidity_consumption'] = df['volume'] / (df['total_liquidity'] + 1e-8)\n\n    # Price & Execution Efficiency\n    df['bid_ask_spread'] = df['ask_qty'] - df['bid_qty']\n    df['price_efficiency'] = 1 / (1 + df['amihud_illiquidity'])\n    df['execution_quality'] = df['volume'] / (df['bid_ask_spread'] + 1)\n\n    # Toxicity Proxies\n    df['pin_proxy'] = np.abs(df['order_flow_imbalance']) * df['amihud_illiquidity']\n    df['order_toxicity'] = np.abs(df['order_flow_imbalance']) * df['kyle_lambda']\n\n    # Market Momentum Features\n    df['bid_momentum'] = df['bid_qty'] * df['buy_qty'] / (df['volume'] + 1e-8)\n    df['ask_momentum'] = df['ask_qty'] * df['sell_qty'] / (df['volume'] + 1e-8)\n    df['liquidity_adjusted_volume'] = df['volume'] / np.sqrt(df['total_liquidity'] + 1)\n\n    # Log-Transformed Stability Features\n    df['log_volume'] = np.log1p(df['volume'])\n    df['log_liquidity'] = np.log1p(df['total_liquidity'])\n    df['log_spread'] = np.log1p(np.abs(df['bid_ask_spread']))\n\n    # Final Sanity Cleanup\n    # Replace infinities with NaNs, then backfill with zero\n    df.replace([np.inf, -np.inf], np.nan, inplace=True)\n    df.fillna(0, inplace=True)\n\n    return df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T04:32:59.292810Z","iopub.execute_input":"2025-07-24T04:32:59.293627Z","iopub.status.idle":"2025-07-24T04:32:59.307142Z","shell.execute_reply.started":"2025-07-24T04:32:59.293591Z","shell.execute_reply":"2025-07-24T04:32:59.306332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_time_weights(n_samples, decay_factor=0.95):\n    \"\"\"\n    Create exponentially decaying weights based on sample position.\n    More recent samples (higher indices) get higher weights.\n    decay_factor controls the rate of decay (0.95 = 5% decay per time unit)\n    \"\"\"\n    positions = np.arange(n_samples)\n    # Normalize positions to [0, 1] range\n    normalized_positions = positions / (n_samples - 1)\n    # Apply exponential weighting\n    weights = decay_factor ** (1 - normalized_positions)\n    # Normalize weights to sum to n_samples (maintains scale)\n    weights = weights * n_samples / weights.sum()\n    return weights","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T04:32:59.308388Z","iopub.execute_input":"2025-07-24T04:32:59.308639Z","iopub.status.idle":"2025-07-24T04:32:59.323997Z","shell.execute_reply.started":"2025-07-24T04:32:59.308618Z","shell.execute_reply":"2025-07-24T04:32:59.323413Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Loading Data and Fold Assignment","metadata":{}},{"cell_type":"code","source":"train_df = reduce_mem_usage(pd.read_parquet(Config.TRAIN_PATH))\ntest_df = reduce_mem_usage(pd.read_parquet(Config.TEST_PATH))\nsubmission_df = pd.read_csv(Config.SUBMISSION_PATH)\n\n# Apply feature engineering\ntrain_df = feature_engineering(train_df)\ntest_df = feature_engineering(test_df)\n\n# Assign folds\ntrain_df.loc['2023-03-01 00:00:00':'2023-05-01 00:00:00', 'Fold'] = 1\ntrain_df.loc['2023-05-01 00:00:00':'2023-07-01 00:00:00', 'Fold'] = 2\ntrain_df.loc['2023-07-01 00:00:00':'2023-09-01 00:00:00', 'Fold'] = 3\ntrain_df.loc['2023-09-01 00:00:00':'2023-11-01 00:00:00', 'Fold'] = 4\ntrain_df.loc['2023-11-01 00:00:00':'2024-01-01 00:00:00', 'Fold'] = 5\ntrain_df.loc['2024-01-01 00:00:00':'2024-03-01 00:00:00', 'Fold'] = 6\n\n# Apply time weights\ntrain_df['weight'] = create_time_weights(len(train_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T04:32:59.324669Z","iopub.execute_input":"2025-07-24T04:32:59.324915Z","iopub.status.idle":"2025-07-24T04:33:44.296685Z","shell.execute_reply.started":"2025-07-24T04:32:59.324894Z","shell.execute_reply":"2025-07-24T04:33:44.296011Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Hyperparameter Tuning","metadata":{}},{"cell_type":"code","source":"models = []\nvalid_scores = []\n\n# Objective function for hyperparameter tuning\ndef objective(trial):\n    fold_scores = []\n    \n    for fold in range(1, 6):\n        # Split data into train/validation based on fold\n        X_train = train_df[train_df[\"Fold\"] != fold][Config.FEATURES]\n        w_train = train_df[train_df[\"Fold\"] != fold][\"weight\"]\n        y_train = train_df[train_df[\"Fold\"] != fold][Config.LABEL_COLUMN]\n\n        X_valid = train_df[train_df[\"Fold\"] == fold][Config.FEATURES]\n        w_valid = train_df[train_df[\"Fold\"] == fold][\"weight\"]\n        y_valid = train_df[train_df[\"Fold\"] == fold][Config.LABEL_COLUMN]\n\n        dtrain = xgb.DMatrix(data=X_train, label=y_train, weight=w_train)\n        dvalid = xgb.DMatrix(data=X_valid, label=y_valid, weight=w_valid)\n\n        # Suggest hyperparameters\n        params = {\n            \"objective\": \"reg:squarederror\",\n            \"seed\": Config.RANDOM_STATE,\n            \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.3, log=True),\n            \"max_depth\": trial.suggest_int(\"max_depth\", 3, 10),\n            \"min_child_weight\": trial.suggest_float(\"min_child_weight\", 1, 10),\n            \"subsample\": trial.suggest_float(\"subsample\", 0.5, 1, step=0.1),\n            \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.5, 1.0, step=0.1),\n            \"gamma\": trial.suggest_float(\"gamma\", 0, 10),\n            \"lambda\": trial.suggest_float(\"lambda\", 1e-3, 10.0, log=True),\n            \"alpha\": trial.suggest_float(\"alpha\", 1e-3, 10.0, log=True),\n        }\n\n        num_boost_round = 150\n        evals = [(dtrain, \"train\"), (dvalid, \"valid\")]\n\n        model = xgb.train(params, dtrain, num_boost_round=num_boost_round, evals=evals,custom_metric=pearsonr_coeff, verbose_eval=False)\n        valid_pred = model.predict(dvalid)\n\n        # Pearson correlation as performance metric\n        fold_score = _pearsonr(y_valid, valid_pred)\n        fold_scores.append(fold_score)\n        \n        models.append(model)\n\n    # Return average Pearson correlation across folds\n    return np.mean(fold_scores)\n\n# Run Optuna study\nstudy = optuna.create_study(direction=\"maximize\")\nstudy.optimize(objective, n_trials=200)\n\n# Print best parameters found\nprint(\"Best parameters:\", study.best_params)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T04:33:44.297537Z","iopub.execute_input":"2025-07-24T04:33:44.297864Z","iopub.status.idle":"2025-07-24T04:34:09.692346Z","shell.execute_reply.started":"2025-07-24T04:33:44.297837Z","shell.execute_reply":"2025-07-24T04:34:09.690749Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"# Initialize the VotingModel with trained estimators; notice the variable is 'voting_model'\nvoting_model = VotingModel(models)\n\n# Option 1: If you want to work manually with DMatrix, you can do:\n# dtest = xgb.DMatrix(test_df[Config.FEATURES])\n# However, if using VotingModel, it's better to let it handle conversion:\n# submission['prediction'] = voting_model.predict(dtest)  # if predict() expects DMatrix\n# Option 2: Use DataFrame directly, and let VotingModel automatically convert if needed:\nsubmission_df['prediction'] = voting_model.predict(test_df[Config.FEATURES])\n\n# Save predictions to CSV file\nsubmission_df.to_csv(r'submission.csv', index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T04:34:09.694771Z","iopub.execute_input":"2025-07-24T04:34:09.695439Z","iopub.status.idle":"2025-07-24T04:34:12.158333Z","shell.execute_reply.started":"2025-07-24T04:34:09.695419Z","shell.execute_reply":"2025-07-24T04:34:12.157492Z"}},"outputs":[],"execution_count":null}]}