{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"},{"sourceId":12566011,"sourceType":"datasetVersion","datasetId":7935367}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":679.589845,"end_time":"2025-07-04T10:52:52.29918","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-07-04T10:41:32.709335","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom sklearn.preprocessing import StandardScaler\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom scipy.stats import pearsonr\nimport optuna\nfrom optuna.samplers import TPESampler\nimport warnings\nfrom itertools import combinations\nimport random\nfrom typing import List, Dict, Tuple\nimport gc\nimport time\nfrom joblib import Parallel, delayed\nwarnings.filterwarnings('ignore')\n\ndef feature_engineering(df):\n    df['exp_856P868P855P289'] = np.exp(df['X446'] + df['X76'] + df['X37'] + df['X287'])\n    df['exp_860P868P855P289'] = np.exp(df['X3'] + df['X76'] + df['X37'] + df['X287'])\n    df['exp_598P868P855P289'] = np.exp(df['X66'] + df['X76'] + df['X37'] + df['X287'])\n    df['exp_612P868P855P289'] = np.exp(df['X1'] + df['X76'] + df['X37'] + df['X287'])\n    df['exp_289P855P21'] = np.exp(df['X287'] + df['X37'] + df['X21'])\n    df['868xexp_289M125'] = df['X76'] * np.exp(df['X287'] - df['X19'])\n    \n    df['exp_603P868P855P289'] = np.exp(df['X25'] + df['X76'] + df['X37'] + df['X287'])\n    df['exp_174P868P855P289'] = np.exp(df['X174'] + df['X76'] + df['X37'] + df['X287'])\n    df['exp_465P868P855P289'] = np.exp(df['X465'] + df['X76'] + df['X37'] + df['X287'])\n    df['exp_125P862P289M125'] = np.exp(df['X19'] + df['X123'] + df['X287'] - df['X19'])\n    df['exp_168P868P855P289'] = np.exp(df['X168'] + df['X76'] + df['X37'] + df['X287'])\n    df['exp_855P289M125'] = np.exp(df['X37'] + df['X287'] - df['X19'])\n    df['exp_302P289M125'] = np.exp(df['X298'] + df['X287'] - df['X19'])\n    df['289xexp_289M125'] = df['X287'] * np.exp(df['X287'] - df['X19'])\n    \n    df['exp_862P868P855P289'] = np.exp(df['X123'] + df['X76'] + df['X37'] + df['X287'])\n    df['868x868x855x289'] = df['X76'] * df['X76'] * df['X37'] * df['X287']\n    df['385xexp_289M125'] = df['X385'] * np.exp(df['X287'] - df['X19'])\n    df['exp_862P289M125'] = np.exp(df['X123'] + df['X287'] - df['X19'])\n    df['exp_786P289M125'] = np.exp(df['X453'] + df['X287'] - df['X19'])\n    df['exp_856P289M125'] = np.exp(df['X446'] + df['X287'] - df['X19'])\n    df['852x868x855x289'] = df['X594'] * df['X76'] * df['X37'] * df['X287']\n    df['465x862x465'] = df['X465'] * df['X465'] * df['X123']\n    df['540x881'] = df['X476'] * df['X443']\n    \n    df['bid_ask_interaction'] = df['ask_qty'] * df['ask_qty']\n    df['bid_buy_interaction'] = df['ask_qty'] * df['buy_qty']\n    df['bid_sell_interaction'] = df['ask_qty'] * df['sell_qty']\n    df['ask_buy_interaction'] = df['ask_qty'] * df['buy_qty']\n    df['ask_sell_interaction'] = df['ask_qty'] * df['sell_qty']\n    \n    df['volume_weighted_sell'] = df['sell_qty'] * df['volume']\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-10)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-10)\n    df['log_volume'] = np.log1p(df['volume'])\n    \n    df['effective_spread_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-10)\n    df['bid_ask_imbalance'] = (df['ask_qty'] - df['ask_qty']) / (df['ask_qty'] + df['ask_qty'] + 1e-10)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    df['liquidity_ratio'] = (df['ask_qty'] + df['ask_qty']) / (df['volume'] + 1e-10)\n    \n    df['ask_buy_interaction_x_X293'] = df['X293'] * df['ask_buy_interaction']\n    \n    df['net_order_flow'] = df['buy_qty'] - df['sell_qty']\n    df['normalized_net_flow'] = df['net_order_flow'] / (df['volume'] + 1e-10)\n    df['buying_pressure'] = df['buy_qty'] / (df['volume'] + 1e-10)\n    df['volume_weighted_buy'] = df['buy_qty'] * df['volume']\n    \n    df['total_depth'] = df['ask_qty'] + df['ask_qty']\n    df['depth_imbalance'] = (df['ask_qty'] - df['ask_qty']) / (df['total_depth'] + 1e-10)\n    df['relative_spread'] = np.abs(df['ask_qty'] - df['ask_qty']) / (df['total_depth'] + 1e-10)\n    df['log_depth'] = np.log1p(df['total_depth'])\n    \n    df['kyle_lambda'] = np.abs(df['net_order_flow']) / (df['volume'] + 1e-10)\n    df['flow_toxicity'] = np.abs(df['order_flow_imbalance']) * df['volume']\n    df['aggressive_flow_ratio'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    \n    df['volume_depth_ratio'] = df['volume'] / (df['total_depth'] + 1e-10)\n    df['activity_intensity'] = (df['buy_qty'] + df['sell_qty']) / (df['volume'] + 1e-10)\n    df['log_buy_qty'] = np.log1p(df['buy_qty'])\n    df['log_sell_qty'] = np.log1p(df['sell_qty'])\n    df['log_bid_qty'] = np.log1p(df['ask_qty'])\n    df['log_ask_qty'] = np.log1p(df['ask_qty'])\n    \n    df['realized_spread_proxy'] = 2 * np.abs(df['net_order_flow']) / (df['volume'] + 1e-10)\n    df['price_impact_proxy'] = df['net_order_flow'] / (df['total_depth'] + 1e-10)\n    df['quote_volatility_proxy'] = np.abs(df['depth_imbalance'])\n    \n    df['flow_depth_interaction'] = df['net_order_flow'] * df['total_depth']\n    df['imbalance_volume_interaction'] = df['order_flow_imbalance'] * df['volume']\n    df['depth_volume_interaction'] = df['total_depth'] * df['volume']\n    df['buy_sell_spread'] = np.abs(df['buy_qty'] - df['sell_qty'])\n    df['bid_ask_spread'] = np.abs(df['ask_qty'] - df['ask_qty'])\n    \n    df['trade_informativeness'] = df['net_order_flow'] / (df['ask_qty'] + df['ask_qty'] + 1e-10)\n    df['execution_shortfall_proxy'] = df['buy_sell_spread'] / (df['volume'] + 1e-10)\n    df['adverse_selection_proxy'] = df['net_order_flow'] / (df['total_depth'] + 1e-10) * df['volume']\n    \n    df['fill_probability'] = df['volume'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    df['execution_rate'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    df['market_efficiency'] = df['volume'] / (df['bid_ask_spread'] + 1e-10)\n    \n    df['sqrt_volume'] = np.sqrt(df['volume'])\n    df['sqrt_depth'] = np.sqrt(df['total_depth'])\n    df['volume_squared'] = df['volume'] ** 2\n    df['imbalance_squared'] = df['order_flow_imbalance'] ** 2\n    \n    df['bid_ratio'] = df['ask_qty'] / (df['total_depth'] + 1e-10)\n    df['ask_ratio'] = df['ask_qty'] / (df['total_depth'] + 1e-10)\n    df['buy_ratio'] = df['buy_qty'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    df['sell_ratio'] = df['sell_qty'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    \n    df['liquidity_consumption'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    df['market_stress'] = df['volume'] / (df['total_depth'] + 1e-10) * np.abs(df['order_flow_imbalance'])\n    df['depth_depletion'] = df['volume'] / (df['ask_qty'] + df['ask_qty'] + 1e-10)\n    \n    df['net_buying_ratio'] = df['net_order_flow'] / (df['volume'] + 1e-10)\n    df['directional_volume'] = df['net_order_flow'] * np.log1p(df['volume'])\n    df['signed_volume'] = np.sign(df['net_order_flow']) * df['volume']\n    \n    df['sqrt_volume_div_log_volume'] = df['sqrt_volume'] / (df['log_volume'] + 1e-6)\n    df['sqrt_volume_div_activity_intensity'] = df['sqrt_volume'] / (df['activity_intensity'] + 1e-6)\n    df['sqrt_volume_mul_fill_probability'] = df['sqrt_volume'] * df['fill_probability']\n    df['volume_div_sqrt_volume'] = df['volume'] / (df['sqrt_volume'] + 1e-6)\n    df['sqrt_volume_div_fill_probability'] = df['sqrt_volume'] / (df['fill_probability'] + 1e-6)\n    df['sqrt_volume_mul_activity_intensity'] = df['sqrt_volume'] * df['activity_intensity']\n    df['sqrt_volume_div_log_sell_qty'] = df['sqrt_volume'] / (df['log_sell_qty'] + 1e-6)\n    df['log_buy_qty_mul_sqrt_volume'] = df['log_buy_qty'] * df['sqrt_volume']\n    df['sqrt_volume_mul_log_buy_qty'] = df['sqrt_volume'] * df['log_buy_qty']\n    df['log_volume_mul_sqrt_volume'] = df['log_volume'] * df['sqrt_volume']\n    \n    df['log_sell_qty_mul_X598'] = df['log_sell_qty'] * df['X66']\n    df['log_buy_qty_mul_X598'] = df['log_buy_qty'] * df['X66']\n    df['log_volume_mul_X598'] = df['log_volume'] * df['X66']\n    df['sqrt_volume_mul_X856'] = df['sqrt_volume'] * df['X446']\n    df['log_sell_qty_mul_X302'] = df['log_sell_qty'] * df['X298']\n    df['log_volume_mul_X302'] = df['log_volume'] * df['X298']\n    df['log_buy_qty_mul_X302'] = df['log_buy_qty'] * df['X298']\n    df['log_sell_qty_mul_X292'] = df['log_sell_qty'] * df['X292']\n    \n    df = df.replace([np.inf, -np.inf], np.nan)\n    df = df.fillna(0)\n    \n    return df\n\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    BASE_FEATURES = [\n        \"X287\", \"X446\", \"X66\", \"X123\", \"X385\", \"X594\", \"X25\", \"X3\", \"X231\",\n        \"X415\", \"X345\", \"X37\", \"X174\", \"X298\", \"X178\", \"X168\", \"X1\",\n        \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\", \"X210\", \"X421\", \"X92\",\n        'X465', 'X105', 'X19', 'X21', \"X76\", \"X453\", \"X293\", \"X504\", \n        'X476', 'X486', 'X443', 'X329', 'X79', \"X292\", \"X244\"\n    ]\n    \n    SELECTED_FEATURES = [\n        \"X287\", \"X446\", \"X66\", \"X123\", \"X385\", \"X25\", \"X3\", \"X231\",\n        \"X415\", \"X345\", \"X37\", \"X174\", \"X298\", \"X178\", \"X168\", \"X1\",\n        \"buy_qty\", \"sell_qty\", \"volume\", \n        \"X210\", \"X421\", \"X92\", \"X292\", \"X244\", \"X329\",\n        'ask_buy_interaction_x_X293', '868xexp_289M125', 'exp_786P289M125', 'exp_856P289M125',\n        'exp_612P868P855P289', 'exp_598P868P855P289', 'exp_855P289M125',\n        '385xexp_289M125', '465x862x465', '540x881', 'exp_125P862P289M125',\n        'bid_ask_interaction', 'bid_buy_interaction', 'bid_sell_interaction', \n        'ask_buy_interaction', 'ask_sell_interaction', \"log_volume\", 'net_order_flow', \n        'normalized_net_flow', 'buying_pressure', 'volume_weighted_buy', 'total_depth', \n        'depth_imbalance', 'relative_spread', 'log_depth', 'kyle_lambda', 'flow_toxicity', \n        'aggressive_flow_ratio', 'volume_depth_ratio', 'activity_intensity', 'log_buy_qty', \n        'log_sell_qty', 'log_bid_qty', 'log_ask_qty', 'realized_spread_proxy', \n        'price_impact_proxy', 'quote_volatility_proxy', 'flow_depth_interaction', \n        'imbalance_volume_interaction', 'depth_volume_interaction', 'trade_informativeness',\n        'execution_shortfall_proxy', 'adverse_selection_proxy', 'fill_probability',\n        'execution_rate', 'market_efficiency', 'sqrt_volume', 'sqrt_depth', 'volume_squared',\n        'imbalance_squared', 'bid_ratio', 'ask_ratio', 'buy_ratio', 'sell_ratio',\n        'liquidity_consumption', 'market_stress', 'depth_depletion', 'net_buying_ratio',\n        'directional_volume', 'signed_volume',\n        \"sqrt_volume_mul_fill_probability\", \"volume_div_sqrt_volume\",\n        \"log_buy_qty_mul_sqrt_volume\", \"sqrt_volume_mul_log_buy_qty\",\n        \"log_volume_mul_sqrt_volume\", \"sqrt_volume_mul_X856\",\n        \"log_sell_qty_mul_X302\", \"log_volume_mul_X302\",\n        \"log_buy_qty_mul_X302\", \"log_sell_qty_mul_X292\"\n    ]\n    \n    LABEL_COLUMN = \"label\"\n    N_FOLDS = 5\n    RANDOM_STATE = 42\n    N_TRIALS = 150\n    \n    FEATURE_SUBSET_SIZE = 35\n    N_FEATURE_SUBSETS = 25\n    FEATURE_OVERLAP_RATIO = 0.45\n    \n    CHUNK_SIZE = 40000\n    CHUNK_OVERLAP = 15000\n    \n    EPOCHS_PER_SUBSET = 5\n    LEARNING_RATE_DECAY = 0.95\n    \n    CORE_FEATURES = [\n        \"X287\", \"X446\", \"X66\", \"X123\", \"volume\", \"buy_qty\", \"sell_qty\",\n        \"net_order_flow\", \"log_volume\", \"buying_pressure\", \"order_flow_imbalance\",\n        \"volume_weighted_buy\", \"effective_spread_proxy\"\n    ]\n    \n    ENSEMBLE_BLEND_MODELS = 3\n    META_LEARNING_FOLDS = 3\n    BAYESIAN_OPTIMIZATION_ROUNDS = 30\n    N_PARALLEL_JOBS = 4\n\ndef create_time_decay_weights(n: int, decay: float = 0.9) -> np.ndarray:\n    positions = np.arange(n)\n    normalized = positions / (n - 1)\n    weights = decay ** (1.0 - normalized)\n    return weights * n / weights.sum()\n\ndef create_adaptive_weights(n: int, volatility: float) -> np.ndarray:\n    base_decay = 0.9\n    adjusted_decay = base_decay - (volatility * 0.1)\n    adjusted_decay = np.clip(adjusted_decay, 0.7, 0.95)\n    return create_time_decay_weights(n, adjusted_decay)\n\ndef load_data():\n    train_df = pd.read_parquet(Config.TRAIN_PATH, columns=Config.BASE_FEATURES + [Config.LABEL_COLUMN])\n    test_df = pd.read_parquet(Config.TEST_PATH, columns=Config.BASE_FEATURES)\n    submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n    \n    train_df = feature_engineering(train_df)\n    test_df = feature_engineering(test_df)\n    \n    print(f\"Data loaded - Train: {train_df.shape}, Test: {test_df.shape}\")\n    return train_df.reset_index(drop=True), test_df.reset_index(drop=True), submission_df\n\ndef generate_diverse_feature_subsets(all_features: List[str], config: Config) -> List[List[str]]:\n    feature_subsets = []\n    \n    non_core_features = [f for f in all_features if f not in config.CORE_FEATURES]\n    \n    core_size = len(config.CORE_FEATURES)\n    non_core_size = config.FEATURE_SUBSET_SIZE - core_size\n    \n    feature_importance_groups = {\n        'market_microstructure': [f for f in non_core_features if any(x in f for x in ['volume', 'qty', 'flow', 'spread', 'depth'])],\n        'engineered_exponential': [f for f in non_core_features if 'exp_' in f],\n        'interactions': [f for f in non_core_features if any(x in f for x in ['times', 'mul', 'div', 'interaction'])],\n        'transformations': [f for f in non_core_features if any(x in f for x in ['log', 'sqrt', 'squared'])],\n        'raw_features': [f for f in non_core_features if f.startswith('X') and not any(x in f for x in ['_', 'times', 'mul', 'div'])]\n    }\n    \n    for i in range(config.N_FEATURE_SUBSETS):\n        if i == 0:\n            selected_non_core = random.sample(non_core_features, non_core_size)\n        else:\n            overlap_size = int(non_core_size * config.FEATURE_OVERLAP_RATIO)\n            new_size = non_core_size - overlap_size\n            \n            all_previous = []\n            for prev_subset in feature_subsets:\n                all_previous.extend([f for f in prev_subset if f not in config.CORE_FEATURES])\n            all_previous = list(set(all_previous))\n            \n            overlap_features = random.sample(all_previous, min(overlap_size, len(all_previous)))\n            \n            group_distribution = {\n                'market_microstructure': 0.3,\n                'engineered_exponential': 0.2,\n                'interactions': 0.2,\n                'transformations': 0.15,\n                'raw_features': 0.15\n            }\n            \n            new_features = []\n            for group_name, proportion in group_distribution.items():\n                group_features = feature_importance_groups[group_name]\n                available_group = [f for f in group_features if f not in overlap_features and f not in new_features]\n                n_select = int(new_size * proportion)\n                if available_group:\n                    new_features.extend(random.sample(available_group, min(n_select, len(available_group))))\n            \n            remaining_needed = new_size - len(new_features)\n            if remaining_needed > 0:\n                available_remaining = [f for f in non_core_features if f not in overlap_features and f not in new_features]\n                if available_remaining:\n                    new_features.extend(random.sample(available_remaining, min(remaining_needed, len(available_remaining))))\n            \n            selected_non_core = overlap_features + new_features\n        \n        subset = config.CORE_FEATURES + selected_non_core\n        feature_subsets.append(subset[:config.FEATURE_SUBSET_SIZE])\n    \n    return feature_subsets\n\ndef create_stratified_data_chunks(df: pd.DataFrame, config: Config) -> List[Tuple[int, int]]:\n    chunks = []\n    n_rows = len(df)\n    \n    target_volatility = df[config.LABEL_COLUMN].rolling(window=10000, min_periods=1000).std()\n    target_volatility = target_volatility.fillna(target_volatility.mean())\n    \n    high_volatility_mask = target_volatility > target_volatility.quantile(0.7)\n    \n    start = 0\n    while start < n_rows:\n        end = min(start + config.CHUNK_SIZE, n_rows)\n        \n        high_vol_ratio = high_volatility_mask[start:end].mean()\n        if high_vol_ratio > 0.5:\n            chunk_size = int(config.CHUNK_SIZE * 0.8)\n            end = min(start + chunk_size, n_rows)\n        \n        chunks.append((start, end))\n        \n        start = end - config.CHUNK_OVERLAP\n        if end == n_rows:\n            break\n    \n    return chunks\n\nclass EnhancedProgressiveEnsembleTrainer:\n    def __init__(self, config: Config):\n        self.config = config\n        self.models = []\n        self.feature_subsets = []\n        self.subset_weights = []\n        self.model_metadata = []\n        self.feature_importance_tracking = {}\n        \n    def train_subset_model(self, \n                          train_df: pd.DataFrame,\n                          feature_subset: List[str],\n                          chunk_indices: Tuple[int, int],\n                          params: Dict,\n                          epoch: int,\n                          model_type: str = 'xgboost') -> object:\n        start_idx, end_idx = chunk_indices\n        chunk_df = train_df.iloc[start_idx:end_idx]\n        \n        chunk_volatility = chunk_df[self.config.LABEL_COLUMN].std()\n        weights = create_adaptive_weights(len(chunk_df), chunk_volatility)\n        \n        adjusted_params = params.copy()\n        adjusted_params['learning_rate'] *= (self.config.LEARNING_RATE_DECAY ** epoch)\n        \n        scaler = StandardScaler()\n        X_scaled = scaler.fit_transform(chunk_df[feature_subset])\n        y = chunk_df[self.config.LABEL_COLUMN]\n        \n        if model_type == 'xgboost':\n            model = XGBRegressor(**adjusted_params)\n            model.fit(X_scaled, y, sample_weight=weights, verbose=False)\n        elif model_type == 'lightgbm':\n            lgb_params = {\n                'boosting_type': 'gbdt',\n                'objective': 'regression',\n                'metric': 'rmse',\n                'verbosity': -1,\n                'random_state': adjusted_params.get('random_state', 42),\n                'n_estimators': adjusted_params.get('n_estimators', 300),\n                'learning_rate': adjusted_params.get('learning_rate', 0.03),\n                'num_leaves': 31,\n                'max_depth': adjusted_params.get('max_depth', 5),\n                'subsample': adjusted_params.get('subsample', 0.8),\n                'colsample_bytree': adjusted_params.get('colsample_bytree', 0.8),\n                'reg_alpha': adjusted_params.get('reg_alpha', 0.1),\n                'reg_lambda': adjusted_params.get('reg_lambda', 0.1),\n            }\n            model = LGBMRegressor(**lgb_params)\n            model.fit(X_scaled, y, sample_weight=weights)\n        elif model_type == 'catboost':\n            cat_params = {\n                'iterations': adjusted_params.get('n_estimators', 300),\n                'learning_rate': adjusted_params.get('learning_rate', 0.03),\n                'depth': adjusted_params.get('max_depth', 5),\n                'l2_leaf_reg': adjusted_params.get('reg_lambda', 1.0),\n                'subsample': adjusted_params.get('subsample', 0.8),\n                'random_state': adjusted_params.get('random_state', 42),\n                'verbose': False\n            }\n            model = CatBoostRegressor(**cat_params)\n            model.fit(X_scaled, y, sample_weight=weights)\n        \n        return model, scaler\n    \n    def evaluate_model_comprehensive(self, \n                                   model: object,\n                                   scaler: StandardScaler,\n                                   valid_df: pd.DataFrame,\n                                   feature_subset: List[str]) -> Dict[str, float]:\n        X_valid_scaled = scaler.transform(valid_df[feature_subset])\n        predictions = model.predict(X_valid_scaled)\n        \n        y_true = valid_df[self.config.LABEL_COLUMN]\n        \n        pearson_corr = pearsonr(y_true, predictions)[0]\n        \n        residuals = y_true - predictions\n        residual_std = residuals.std()\n        \n        directional_accuracy = ((y_true > 0) == (predictions > 0)).mean()\n        \n        mae = np.mean(np.abs(residuals))\n        \n        metrics = {\n            'pearson': pearson_corr,\n            'residual_std': residual_std,\n            'directional_accuracy': directional_accuracy,\n            'mae': mae,\n            'combined_score': pearson_corr * 0.6 + directional_accuracy * 0.3 - residual_std * 0.1\n        }\n        \n        return metrics\n    \n    def train_progressive_ensemble_enhanced(self,\n                                          train_df: pd.DataFrame,\n                                          test_df: pd.DataFrame,\n                                          optimized_params: Dict) -> Tuple[np.ndarray, Dict]:\n        \n        all_features = [col for col in train_df.columns if col != self.config.LABEL_COLUMN]\n        self.feature_subsets = generate_diverse_feature_subsets(all_features, self.config)\n        \n        chunks = create_stratified_data_chunks(train_df, self.config)\n        \n        test_predictions = []\n        model_info = []\n        \n        print(f\"Training with {len(self.feature_subsets)} feature subsets and {len(chunks)} data chunks\")\n        print(f\"Total models to train: {len(self.feature_subsets) * self.config.EPOCHS_PER_SUBSET}\")\n        \n        model_types = ['xgboost', 'lightgbm', 'catboost']\n        \n        for epoch in range(self.config.EPOCHS_PER_SUBSET):\n            epoch_start_time = time.time()\n            print(f\"\\nEpoch {epoch + 1}/{self.config.EPOCHS_PER_SUBSET}\")\n            \n            epoch_models = []\n            \n            for subset_idx, feature_subset in enumerate(self.feature_subsets):\n                chunk_idx = (subset_idx + epoch * len(self.feature_subsets)) % len(chunks)\n                chunk = chunks[chunk_idx]\n                \n                model_type = model_types[(subset_idx + epoch) % len(model_types)]\n                \n                if subset_idx % 5 == 0:\n                    print(f\"  Training subset {subset_idx + 1}/{len(self.feature_subsets)} \"\n                          f\"on chunk {chunk_idx + 1}/{len(chunks)} with {model_type}\")\n                \n                model, scaler = self.train_subset_model(\n                    train_df, \n                    feature_subset, \n                    chunk, \n                    optimized_params, \n                    epoch,\n                    model_type\n                )\n                \n                valid_start = int(0.8 * len(train_df))\n                valid_df = train_df.iloc[valid_start:]\n                \n                valid_sample_size = min(20000, len(valid_df))\n                valid_sample = valid_df.sample(n=valid_sample_size, random_state=subset_idx)\n                \n                metrics = self.evaluate_model_comprehensive(model, scaler, valid_sample, feature_subset)\n                \n                self.models.append((model, scaler))\n                model_info.append({\n                    'epoch': epoch,\n                    'subset_idx': subset_idx,\n                    'chunk_idx': chunk_idx,\n                    'metrics': metrics,\n                    'features': feature_subset,\n                    'model_type': model_type\n                })\n                \n                test_pred = model.predict(scaler.transform(test_df[feature_subset]))\n                test_predictions.append(test_pred)\n                \n                if hasattr(model, 'feature_importances_'):\n                    for feat, imp in zip(feature_subset, model.feature_importances_):\n                        if feat not in self.feature_importance_tracking:\n                            self.feature_importance_tracking[feat] = []\n                        self.feature_importance_tracking[feat].append(imp)\n                \n                if (subset_idx + 1) % 10 == 0:\n                    gc.collect()\n            \n            epoch_duration = time.time() - epoch_start_time\n            print(f\"  Epoch completed in {epoch_duration:.1f} seconds\")\n        \n        scores = np.array([info['metrics']['combined_score'] for info in model_info])\n        \n        score_quantiles = np.percentile(scores, [25, 50, 75])\n        \n        adjusted_scores = scores.copy()\n        for i, score in enumerate(scores):\n            model_type = model_info[i]['model_type']\n            if model_type == 'xgboost':\n                adjusted_scores[i] *= 1.05\n            elif model_type == 'lightgbm':\n                adjusted_scores[i] *= 1.02\n        \n        exp_scores = np.exp((adjusted_scores - adjusted_scores.min()) * 5)\n        self.subset_weights = exp_scores / exp_scores.sum()\n        \n        test_predictions = np.array(test_predictions)\n        \n        top_model_indices = np.argsort(adjusted_scores)[-int(len(adjusted_scores) * 0.7):]\n        top_weights = self.subset_weights[top_model_indices]\n        top_weights = top_weights / top_weights.sum()\n        \n        final_predictions = np.average(test_predictions[top_model_indices], weights=top_weights, axis=0)\n        \n        self.model_metadata = model_info\n        \n        return final_predictions, model_info\n    \n    def create_meta_ensemble(self, test_df: pd.DataFrame, n_top_models: int = 50) -> np.ndarray:\n        scores = [info['metrics']['combined_score'] for info in self.model_metadata]\n        top_indices = np.argsort(scores)[-n_top_models:]\n        \n        meta_predictions = []\n        \n        for idx in top_indices:\n            model, scaler = self.models[idx]\n            features = self.model_metadata[idx]['features']\n            pred = model.predict(scaler.transform(test_df[features]))\n            meta_predictions.append(pred)\n        \n        meta_predictions = np.array(meta_predictions)\n        \n        correlation_matrix = np.corrcoef(meta_predictions)\n        \n        diversity_scores = 1 - np.mean(np.abs(correlation_matrix), axis=1)\n        performance_scores = np.array([scores[idx] for idx in top_indices])\n        \n        combined_weights = performance_scores * (1 + diversity_scores * 0.3)\n        combined_weights = combined_weights / combined_weights.sum()\n        \n        return np.average(meta_predictions, weights=combined_weights, axis=0)\n\ndef create_advanced_objective(train_df, feature_subsets, config):\n    \n    def objective(trial):\n        model_type = trial.suggest_categorical('model_type', ['xgboost', 'lightgbm'])\n        \n        if model_type == 'xgboost':\n            params = {\n                'tree_method': 'hist',\n                'device': 'gpu',\n                'verbosity': 0,\n                'random_state': config.RANDOM_STATE,\n                'n_jobs': -1,\n                \n                'n_estimators': trial.suggest_int('n_estimators', 100, 800),\n                'max_depth': trial.suggest_int('max_depth', 3, 10),\n                'min_child_weight': trial.suggest_int('min_child_weight', 10, 100),\n                'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3, log=True),\n                \n                'reg_alpha': trial.suggest_float('reg_alpha', 0.01, 100, log=True),\n                'reg_lambda': trial.suggest_float('reg_lambda', 0.01, 100, log=True),\n                'gamma': trial.suggest_float('gamma', 0.01, 10, log=True),\n                \n                'subsample': trial.suggest_float('subsample', 0.5, 1.0),\n                'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),\n                'colsample_bylevel': trial.suggest_float('colsample_bylevel', 0.5, 1.0),\n            }\n        else:\n            params = {\n                'boosting_type': 'gbdt',\n                'objective': 'regression',\n                'metric': 'rmse',\n                'verbosity': -1,\n                'random_state': config.RANDOM_STATE,\n                \n                'n_estimators': trial.suggest_int('n_estimators', 100, 800),\n                'num_leaves': trial.suggest_int('num_leaves', 20, 300),\n                'max_depth': trial.suggest_int('max_depth', 3, 10),\n                'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3, log=True),\n                \n                'reg_alpha': trial.suggest_float('reg_alpha', 0.01, 100, log=True),\n                'reg_lambda': trial.suggest_float('reg_lambda', 0.01, 100, log=True),\n                \n                'subsample': trial.suggest_float('subsample', 0.5, 1.0),\n                'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),\n                'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 10, 100),\n            }\n        \n        subset_scores = []\n        \n        eval_subsets = random.sample(feature_subsets, min(5, len(feature_subsets)))\n        \n        tscv = TimeSeriesSplit(n_splits=3, test_size=30000)\n        \n        for feature_subset in eval_subsets:\n            fold_scores = []\n            \n            for fold_idx, (train_idx, valid_idx) in enumerate(tscv.split(train_df)):\n                if fold_idx > 1:\n                    break\n                \n                if len(train_idx) > 80000:\n                    train_idx = train_idx[-80000:]\n                \n                X_train = train_df.iloc[train_idx][feature_subset]\n                y_train = train_df.iloc[train_idx][config.LABEL_COLUMN]\n                X_valid = train_df.iloc[valid_idx][feature_subset]\n                y_valid = train_df.iloc[valid_idx][config.LABEL_COLUMN]\n                \n                scaler = StandardScaler()\n                X_train_scaled = scaler.fit_transform(X_train)\n                X_valid_scaled = scaler.transform(X_valid)\n                \n                if model_type == 'xgboost':\n                    model = XGBRegressor(**params)\n                    model.fit(X_train_scaled, y_train, \n                             eval_set=[(X_valid_scaled, y_valid)], \n                             early_stopping_rounds=30,\n                             verbose=False)\n                else:\n                    model = LGBMRegressor(**params)\n                    model.fit(X_train_scaled, y_train)\n                \n                valid_pred = model.predict(X_valid_scaled)\n                \n                pearson = pearsonr(y_valid, valid_pred)[0]\n                directional_acc = ((y_valid > 0) == (valid_pred > 0)).mean()\n                \n                combined_score = pearson * 0.7 + directional_acc * 0.3\n                fold_scores.append(combined_score)\n            \n            subset_scores.append(np.mean(fold_scores))\n        \n        return np.mean(subset_scores)\n    \n    return objective\n\ndef perform_final_ensemble_calibration(predictions: np.ndarray, \n                                     train_df: pd.DataFrame,\n                                     config: Config) -> np.ndarray:\n    \n    train_percentiles = np.percentile(train_df[config.LABEL_COLUMN], [5, 25, 50, 75, 95])\n    pred_percentiles = np.percentile(predictions, [5, 25, 50, 75, 95])\n    \n    calibration_factor = np.std(train_df[config.LABEL_COLUMN]) / np.std(predictions)\n    calibration_factor = np.clip(calibration_factor, 0.8, 1.2)\n    \n    calibrated_predictions = predictions * calibration_factor\n    \n    calibrated_predictions = np.clip(\n        calibrated_predictions,\n        np.percentile(train_df[config.LABEL_COLUMN], 0.5),\n        np.percentile(train_df[config.LABEL_COLUMN], 99.5)\n    )\n    \n    return calibrated_predictions\n\ndef main():\n    start_time = time.time()\n    \n    print(\"Loading data...\")\n    train_df = pd.read_parquet(Config.TRAIN_PATH, columns=Config.BASE_FEATURES + [Config.LABEL_COLUMN])\n    test_df = pd.read_parquet(Config.TEST_PATH, columns=Config.BASE_FEATURES)\n    submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n    \n    train_df = feature_engineering(train_df)\n    test_df = feature_engineering(test_df)\n    \n    print(f\"Data loaded - Train: {train_df.shape}, Test: {test_df.shape}\")\n    \n    all_features = [col for col in train_df.columns if col != Config.LABEL_COLUMN]\n    feature_subsets = generate_diverse_feature_subsets(all_features, Config)\n    \n    print(f\"\\nOptimizing hyperparameters with {Config.N_TRIALS} trials...\")\n    study = optuna.create_study(\n        direction='maximize',\n        sampler=TPESampler(seed=Config.RANDOM_STATE),\n        pruner=optuna.pruners.MedianPruner(n_startup_trials=20, n_warmup_steps=10)\n    )\n    \n    objective = create_advanced_objective(train_df, feature_subsets, Config)\n    study.optimize(objective, n_trials=Config.N_TRIALS, show_progress_bar=True)\n    \n    print(f\"\\nBest score: {study.best_value:.4f}\")\n    print(\"Best parameters:\")\n    for key, value in study.best_params.items():\n        print(f\"  {key}: {value}\")\n    \n    print(\"\\nTraining enhanced progressive ensemble...\")\n    trainer = EnhancedProgressiveEnsembleTrainer(Config)\n    \n    optimized_params = study.best_params.copy()\n    optimized_params.update({\n        'tree_method': 'hist',\n        'device': 'gpu',\n        'verbosity': 0,\n        'random_state': Config.RANDOM_STATE,\n        'n_jobs': -1\n    })\n    \n    final_predictions, model_info = trainer.train_progressive_ensemble_enhanced(\n        train_df, \n        test_df, \n        optimized_params\n    )\n    \n    print(\"\\nCreating meta-ensemble...\")\n    meta_predictions = trainer.create_meta_ensemble(test_df, n_top_models=50)\n    \n    ensemble_predictions = 0.7 * final_predictions + 0.3 * meta_predictions\n    \n    calibrated_predictions = perform_final_ensemble_calibration(\n        ensemble_predictions, train_df, Config\n    )\n    \n    print(\"\\n=== Model Performance Summary ===\")\n    scores_by_epoch = {}\n    for info in model_info:\n        epoch = info['epoch']\n        if epoch not in scores_by_epoch:\n            scores_by_epoch[epoch] = []\n        scores_by_epoch[epoch].append(info['metrics']['combined_score'])\n    \n    for epoch, scores in scores_by_epoch.items():\n        print(f\"Epoch {epoch + 1}: Mean Score = {np.mean(scores):.4f} (±{np.std(scores):.4f})\")\n    \n    valid_start = int(0.8 * len(train_df))\n    valid_df = train_df.iloc[valid_start:]\n    \n    valid_predictions = []\n    for i, (model, scaler) in enumerate(trainer.models[:50]):\n        features = model_info[i]['features']\n        pred = model.predict(scaler.transform(valid_df[features]))\n        valid_predictions.append(pred)\n    \n    valid_predictions = np.array(valid_predictions)\n    ensemble_valid_pred = np.average(valid_predictions, weights=trainer.subset_weights[:50], axis=0)\n    final_score = pearsonr(valid_df[Config.LABEL_COLUMN], ensemble_valid_pred)[0]\n    \n    print(f\"\\nFinal ensemble validation score: {final_score:.4f}\")\n    \n    print(\"\\nTop 20 most important features:\")\n    feature_importance_avg = {\n        feat: np.mean(imps) for feat, imps in trainer.feature_importance_tracking.items()\n    }\n    sorted_features = sorted(feature_importance_avg.items(), key=lambda x: x[1], reverse=True)\n    for feat, imp in sorted_features[:20]:\n        print(f\"  {feat}: {imp:.4f}\")\n    \n    submission_df[\"prediction\"] = calibrated_predictions\n    submission_df.to_csv(\"submission_enhanced_ensemble.csv\", index=False)\n    print(\"\\nSubmission saved to submission_enhanced_ensemble.csv\")\n    \n    model_summary = {\n        'total_models': len(trainer.models),\n        'feature_subsets': len(trainer.feature_subsets),\n        'epochs': Config.EPOCHS_PER_SUBSET,\n        'final_score': final_score,\n        'model_type_distribution': {\n            model_type: sum(1 for info in model_info if info['model_type'] == model_type)\n            for model_type in ['xgboost', 'lightgbm', 'catboost']\n        }\n    }\n    \n    total_time = time.time() - start_time\n    print(f\"\\nTotal execution time: {total_time/60:.1f} minutes\")\n    \n    return submission_df, optimized_params, final_score, trainer, model_summary\n\nif __name__ == \"__main__\":\n    submission_df, best_params, score, trainer, summary = main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}