{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Core ML libraries\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, HistGradientBoostingRegressor\nfrom sklearn.linear_model import Ridge, Lasso, ElasticNet, HuberRegressor\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, QuantileTransformer\nfrom sklearn.feature_selection import SelectKBest, mutual_info_regression\nfrom sklearn.metrics import mean_squared_error\nfrom scipy.stats import pearsonr, rankdata\nfrom scipy.optimize import minimize, differential_evolution\nfrom scipy.special import softmax\nfrom scipy.ndimage import gaussian_filter1d\n\n# Deep learning\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset\n\n# Additional libraries\nimport gc\nimport os\nfrom datetime import datetime\nfrom typing import Dict, List, Tuple, Optional\nimport json\nimport pickle\nimport lightgbm as lgb\n\n# Check GPU availability\nUSE_GPU = torch.cuda.is_available()\nDEVICE = torch.device('cuda' if USE_GPU else 'cpu')\n\nprint(\"=\" * 100)\nprint(\"ULTIMATE ENHANCED CRYPTO PREDICTION SYSTEM - MERGED VERSION\")\nprint(\"=\" * 100)\nprint(f\"Device: {DEVICE}\")\nif USE_GPU:\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")\nprint(f\"Start Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")\nprint(\"=\" * 100)\n\nclass Config:\n    # Paths\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    # Training settings\n    n_folds = 3  # Reduced from 5 to prevent overfitting\n    random_state = 42\n    use_last_pct = 0.5  # Use last 50% of data\n    val_size = 0.15\n    \n    # Feature settings\n    max_features_before_transform = 200  # Limit features before transformations\n    max_features_final = 300  # Final feature limit\n    n_proprietary_features = 40\n    n_interaction_features = 50\n    \n    # Anti-overfitting settings\n    use_adversarial_validation = True\n    blend_with_previous = True\n    use_feature_selection = True\n    regularization_multiplier = 2.0\n    early_stopping_rounds = 50\n    \n    # Iteration settings\n    save_intermediate = True\n    verbose = True\n\nconfig = Config()\n\nclass AntiOverfittingWrapper:\n    \"\"\"Wrapper to add anti-overfitting measures to models\"\"\"\n    \n    def __init__(self, base_model, model_type='tree'):\n        self.base_model = base_model\n        self.model_type = model_type\n        self.train_score = None\n        self.val_score = None\n        \n    def fit(self, X_train, y_train, X_val=None, y_val=None):\n        \"\"\"Fit with overfitting detection\"\"\"\n        self.base_model.fit(X_train, y_train)\n        \n        # Calculate scores\n        train_pred = self.base_model.predict(X_train)\n        self.train_score = pearsonr(y_train, train_pred)[0]\n        \n        if X_val is not None and y_val is not None:\n            val_pred = self.base_model.predict(X_val)\n            self.val_score = pearsonr(y_val, val_pred)[0]\n            \n            # Check for overfitting\n            overfit_gap = self.train_score - self.val_score\n            if overfit_gap > 0.1:\n                print(f\"  Warning: Overfitting detected! Gap: {overfit_gap:.4f}\")\n    \n    def predict(self, X):\n        \"\"\"Predict with potential adjustments\"\"\"\n        predictions = self.base_model.predict(X)\n        \n        # Apply shrinkage if overfitting was detected\n        if self.train_score and self.val_score:\n            overfit_gap = self.train_score - self.val_score\n            if overfit_gap > 0.1:\n                # Shrink predictions towards mean\n                shrinkage_factor = 1 - min(0.3, overfit_gap)\n                pred_mean = predictions.mean()\n                predictions = shrinkage_factor * predictions + (1 - shrinkage_factor) * pred_mean\n        \n        return predictions\n\nclass UltimateFeatureBuilder:\n    \"\"\"Builds features iteratively with comprehensive feature engineering\"\"\"\n    \n    def __init__(self):\n        self.feature_sets = {\n            'baseline': [],\n            'level1': [],\n            'level2': [],\n            'level3': [],\n            'level4': [],\n            'level5': []\n        }\n        self.feature_importance = {}\n        \n    def get_baseline_features(self, df):\n        \"\"\"Level 0: Only raw market features\"\"\"\n        features = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\n        self.feature_sets['baseline'] = features\n        return features\n    \n    def add_level1_features(self, df):\n        \"\"\"Level 1: Basic ratios and spreads\"\"\"\n        print(\"  Adding Level 1 features (basic ratios)...\")\n        \n        eps = 1e-8\n        \n        # Bid-ask features\n        df['bid_ask_spread'] = df['ask_qty'] - df['bid_qty']\n        df['bid_ask_ratio'] = df['bid_qty'] / (df['ask_qty'] + eps)\n        df['ask_bid_ratio'] = df['ask_qty'] / (df['bid_qty'] + eps)\n        df['bid_ask_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] + eps)\n        \n        # Buy-sell features\n        df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + eps)\n        df['sell_buy_ratio'] = df['sell_qty'] / (df['buy_qty'] + eps)\n        df['buy_sell_diff'] = df['buy_qty'] - df['sell_qty']\n        \n        # Volume features\n        df['trade_volume'] = df['buy_qty'] + df['sell_qty']\n        df['volume_ratio'] = df['trade_volume'] / (df['volume'] + eps)\n        df['volume_imbalance'] = df['trade_volume'] / (df['volume'] + eps)\n        \n        # Log transforms\n        df['log_volume'] = np.log1p(df['volume'])\n        df['sqrt_volume'] = np.sqrt(df['volume'])\n        \n        new_features = [\n            'bid_ask_spread', 'bid_ask_ratio', 'ask_bid_ratio', 'bid_ask_imbalance',\n            'buy_sell_ratio', 'sell_buy_ratio', 'buy_sell_diff',\n            'trade_volume', 'volume_ratio', 'volume_imbalance',\n            'log_volume', 'sqrt_volume'\n        ]\n        \n        self.feature_sets['level1'] = new_features\n        return new_features\n    \n    def add_level2_features(self, df):\n        \"\"\"Level 2: Order flow and pressure indicators\"\"\"\n        print(\"  Adding Level 2 features (order flow & pressure)...\")\n        \n        eps = 1e-8\n        \n        # Order flow imbalance (critical feature)\n        df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['volume'] + eps)\n        df['net_order_flow'] = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + eps)\n        \n        # Pressure indicators\n        df['buying_pressure'] = df['buy_qty'] / (df['volume'] + eps)\n        df['selling_pressure'] = df['sell_qty'] / (df['volume'] + eps)\n        df['net_pressure'] = df['buying_pressure'] - df['selling_pressure']\n        df['bid_pressure'] = df['bid_qty'] / (df['bid_qty'] + df['ask_qty'] + eps)\n        df['ask_pressure'] = df['ask_qty'] / (df['bid_qty'] + df['ask_qty'] + eps)\n        df['pressure_difference'] = df['bid_pressure'] - df['ask_pressure']\n        \n        # Liquidity\n        df['total_liquidity'] = df['bid_qty'] + df['ask_qty']\n        df['liquidity_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_liquidity'] + eps)\n        df['liquidity_ratio'] = df['total_liquidity'] / (df['volume'] + eps)\n        df['liquidity_concentration'] = df['total_liquidity'] / (df['volume'] + eps)\n        df['depth_ratio'] = df['total_liquidity'] / (df['trade_volume'] + eps)\n        \n        new_features = [\n            'order_flow_imbalance', 'net_order_flow',\n            'buying_pressure', 'selling_pressure', 'net_pressure',\n            'bid_pressure', 'ask_pressure', 'pressure_difference',\n            'total_liquidity', 'liquidity_imbalance', 'liquidity_ratio',\n            'liquidity_concentration', 'depth_ratio'\n        ]\n        \n        self.feature_sets['level2'] = new_features\n        return new_features\n    \n    def add_level3_features(self, df):\n        \"\"\"Level 3: Advanced market microstructure\"\"\"\n        print(\"  Adding Level 3 features (microstructure)...\")\n        \n        eps = 1e-8\n        \n        # Advanced microstructure metrics\n        df['kyle_lambda'] = df['net_order_flow'] / (df['sqrt_volume'] + eps)\n        df['vpin'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + eps)\n        df['amihud_illiquidity'] = np.abs(df['net_order_flow']) / (df['log_volume'] + eps)\n        df['roll_measure'] = 2 * np.sqrt(np.abs(df['bid_ask_spread'] * df['order_flow_imbalance']))\n        \n        # Market quality indicators\n        df['effective_spread'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + eps)\n        df['realized_spread'] = np.abs(df['order_flow_imbalance']) * df['volume']\n        df['price_impact'] = df['net_order_flow'] * df['volume']\n        \n        # Composite features\n        df['market_stress'] = df['effective_spread'] * df['vpin']\n        df['liquidity_tightness'] = df['liquidity_ratio'] * df['depth_ratio']\n        df['order_toxicity'] = df['kyle_lambda'] * df['vpin']\n        df['execution_cost'] = df['bid_ask_spread'] * df['liquidity_ratio']\n        \n        # Additional metrics\n        df['avg_trade_size'] = df['volume'] / (df['trade_volume'] + eps)\n        df['trade_intensity'] = df['trade_volume'] / (df['total_liquidity'] + eps)\n        \n        new_features = [\n            'kyle_lambda', 'vpin', 'amihud_illiquidity', 'roll_measure',\n            'effective_spread', 'realized_spread', 'price_impact',\n            'market_stress', 'liquidity_tightness', 'order_toxicity',\n            'execution_cost', 'avg_trade_size', 'trade_intensity'\n        ]\n        \n        self.feature_sets['level3'] = new_features\n        return new_features\n    \n    def add_level4_features(self, df, top_x_features=None):\n        \"\"\"Level 4: Selected X features and interactions\"\"\"\n        print(\"  Adding Level 4 features (X features + interactions)...\")\n        \n        eps = 1e-8\n        new_features = []\n        \n        # Add top X features if provided\n        if top_x_features:\n            available_x = [f for f in top_x_features if f in df.columns]\n            new_features.extend(available_x[:50])\n            print(f\"    Added {len(available_x[:50])} X features\")\n        \n        # Key interactions\n        interactions = [\n            ('order_flow_imbalance', 'kyle_lambda', 'ofi_kyle_interaction'),\n            ('buying_pressure', 'liquidity_ratio', 'pressure_liquidity_interaction'),\n            ('vpin', 'bid_ask_spread', 'vpin_spread_interaction'),\n            ('market_stress', 'volume', 'stress_volume_interaction'),\n            ('net_order_flow', 'vpin', 'flow_vpin_interaction'),\n            ('liquidity_imbalance', 'effective_spread', 'liq_spread_interaction'),\n            ('kyle_lambda', 'amihud_illiquidity', 'kyle_amihud_interaction'),\n            ('bid_ask_ratio', 'order_flow_imbalance', 'ratio_ofi_interaction'),\n            ('buying_pressure', 'selling_pressure', 'pressure_product'),\n            ('total_liquidity', 'vpin', 'liquidity_vpin_interaction')\n        ]\n        \n        for feat1, feat2, interaction_name in interactions:\n            if feat1 in df.columns and feat2 in df.columns:\n                df[interaction_name] = df[feat1] * df[feat2]\n                new_features.append(interaction_name)\n        \n        # Non-linear transformations\n        if 'order_flow_imbalance' in df.columns:\n            df['ofi_squared'] = df['order_flow_imbalance'] ** 2\n            df['ofi_sqrt_abs'] = np.sqrt(np.abs(df['order_flow_imbalance']))\n            new_features.extend(['ofi_squared', 'ofi_sqrt_abs'])\n        \n        if 'kyle_lambda' in df.columns:\n            df['kyle_exp'] = np.exp(-np.abs(df['kyle_lambda']))\n            new_features.append('kyle_exp')\n        \n        self.feature_sets['level4'] = new_features\n        return new_features\n    \n    def add_level5_features(self, df):\n        \"\"\"Level 5: Proprietary features and robust transformations\"\"\"\n        print(\"  Adding Level 5 features (proprietary + robust)...\")\n        \n        new_features = []\n        \n        # Create proprietary features based on X feature combinations\n        x_features = [col for col in df.columns if col.startswith('X') and col[1:].isdigit()]\n        if len(x_features) >= 10:\n            # Select top features based on variance\n            x_data = df[x_features]\n            variances = x_data.var()\n            top_x = variances.nlargest(20).index.tolist()\n            \n            # Statistical aggregations\n            df['X_mean_top10'] = df[top_x[:10]].mean(axis=1)\n            df['X_std_top10'] = df[top_x[:10]].std(axis=1)\n            df['X_max_min_diff'] = df[top_x[:10]].max(axis=1) - df[top_x[:10]].min(axis=1)\n            df['X_median_top10'] = df[top_x[:10]].median(axis=1)\n            df['X_skew_top10'] = df[top_x[:10]].skew(axis=1)\n            \n            new_features.extend(['X_mean_top10', 'X_std_top10', 'X_max_min_diff', \n                               'X_median_top10', 'X_skew_top10'])\n            \n            # Create interactions with market features\n            if 'volume' in df.columns:\n                df['X_prop_vol_weighted'] = df[top_x[0]] * np.log1p(df['volume'])\n                new_features.append('X_prop_vol_weighted')\n            \n            if 'order_flow_imbalance' in df.columns:\n                df['X_prop_ofi_interaction'] = df[top_x[0]] * df['order_flow_imbalance']\n                new_features.append('X_prop_ofi_interaction')\n            \n            # Non-linear transformations\n            for i, feat in enumerate(top_x[:5]):\n                df[f'X_prop_sqrt_{i}'] = np.sqrt(np.abs(df[feat]))\n                df[f'X_prop_sq_{i}'] = df[feat] ** 2\n                new_features.extend([f'X_prop_sqrt_{i}', f'X_prop_sq_{i}'])\n        \n        # Advanced market combinations\n        eps = 1e-8\n        if all(col in df.columns for col in ['volume', 'order_flow_imbalance', 'kyle_lambda']):\n            df['volume_weighted_ofi'] = df['order_flow_imbalance'] * np.log1p(df['volume'])\n            df['kyle_volume_ratio'] = df['kyle_lambda'] / (df['sqrt_volume'] + eps)\n            new_features.extend(['volume_weighted_ofi', 'kyle_volume_ratio'])\n        \n        # Robust features (rank transformations)\n        rank_cols = ['volume', 'bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', \n                     'order_flow_imbalance', 'kyle_lambda', 'vpin']\n        for col in rank_cols:\n            if col in df.columns:\n                df[f'{col}_rank'] = rankdata(df[col], method='average') / len(df)\n                df[f'{col}_percentile'] = df[col].rank(pct=True)\n                new_features.extend([f'{col}_rank', f'{col}_percentile'])\n        \n        self.feature_sets['level5'] = new_features\n        return new_features\n    \n    def get_features_up_to_level(self, level):\n        \"\"\"Get all features up to specified level\"\"\"\n        all_features = []\n        levels = ['baseline', 'level1', 'level2', 'level3', 'level4', 'level5']\n        \n        for l in levels[:levels.index(level) + 1]:\n            all_features.extend(self.feature_sets.get(l, []))\n        \n        return all_features\n\nclass EnhancedModelIterator:\n    \"\"\"Iteratively builds and evaluates models with anti-overfitting measures\"\"\"\n    \n    def __init__(self, config):\n        self.config = config\n        self.results = {}\n        self.best_score = -np.inf\n        self.best_level = None\n        self.best_models = None\n        \n    def create_ensemble(self, complexity='low', use_gpu=False, reg_mult=1.0):\n        \"\"\"Create ensemble based on complexity level with adjustable regularization\"\"\"\n        models = {}\n        \n        if complexity == 'low':\n            # Simple, fast models with strong regularization\n            models['xgb_simple'] = XGBRegressor(\n                n_estimators=200,\n                max_depth=4,  # Reduced\n                learning_rate=0.03,\n                subsample=0.7,\n                colsample_bytree=0.7,\n                reg_alpha=5 * reg_mult,\n                reg_lambda=5 * reg_mult,\n                gamma=1,\n                random_state=42,\n                tree_method='gpu_hist' if use_gpu else 'hist',\n                verbosity=0\n            )\n            \n            models['lgb_simple'] = LGBMRegressor(\n                n_estimators=200,\n                max_depth=4,  # Reduced\n                num_leaves=31,\n                learning_rate=0.03,\n                feature_fraction=0.7,\n                bagging_fraction=0.7,\n                reg_alpha=5 * reg_mult,\n                reg_lambda=5 * reg_mult,\n                min_child_samples=50,\n                random_state=42,\n                device='gpu' if use_gpu else 'cpu',\n                verbose=-1\n            )\n            \n            models['ridge'] = Ridge(alpha=10.0 * reg_mult, random_state=42)\n            models['lasso'] = Lasso(alpha=0.1 * reg_mult, random_state=42)\n            \n        elif complexity == 'medium':\n            # Medium complexity with balanced regularization\n            models['xgb_medium'] = XGBRegressor(\n                n_estimators=400,\n                max_depth=6,  # Reduced\n                learning_rate=0.02,\n                subsample=0.6,\n                colsample_bytree=0.6,\n                reg_alpha=20 * reg_mult,\n                reg_lambda=20 * reg_mult,\n                gamma=2,\n                min_child_weight=20,\n                random_state=42,\n                tree_method='gpu_hist' if use_gpu else 'hist',\n                verbosity=0\n            )\n            \n            models['lgb_medium'] = LGBMRegressor(\n                n_estimators=400,\n                num_leaves=31,  # Reduced\n                max_depth=5,\n                learning_rate=0.02,\n                feature_fraction=0.6,\n                bagging_fraction=0.6,\n                lambda_l1=20 * reg_mult,\n                lambda_l2=20 * reg_mult,\n                min_child_samples=80,\n                random_state=42,\n                device='gpu' if use_gpu else 'cpu',\n                verbose=-1\n            )\n            \n            models['cat_medium'] = CatBoostRegressor(\n                iterations=300,\n                depth=6,  # Reduced\n                learning_rate=0.02,\n                l2_leaf_reg=20 * reg_mult,\n                min_data_in_leaf=50,\n                random_state=42,\n                task_type='GPU' if use_gpu else 'CPU',\n                verbose=False\n            )\n            \n            models['huber'] = HuberRegressor(epsilon=1.35, alpha=0.1 * reg_mult, max_iter=200)\n            models['elastic'] = ElasticNet(alpha=0.1 * reg_mult, l1_ratio=0.5, random_state=42)\n            \n        else:  # high\n            # Full complexity with strong regularization\n            models['xgb_complex'] = XGBRegressor(\n                n_estimators=600,  # Reduced\n                max_depth=8,  # Reduced\n                learning_rate=0.015,\n                subsample=0.5,\n                colsample_bytree=0.5,\n                reg_alpha=50 * reg_mult,\n                reg_lambda=50 * reg_mult,\n                gamma=3,\n                min_child_weight=30,\n                random_state=42,\n                tree_method='gpu_hist' if use_gpu else 'hist',\n                verbosity=0\n            )\n            \n            models['lgb_complex'] = LGBMRegressor(\n                n_estimators=600,  # Reduced\n                num_leaves=63,  # Reduced\n                max_depth=6,\n                learning_rate=0.015,\n                feature_fraction=0.5,\n                bagging_fraction=0.5,\n                lambda_l1=50 * reg_mult,\n                lambda_l2=50 * reg_mult,\n                min_child_samples=100,\n                random_state=42,\n                device='gpu' if use_gpu else 'cpu',\n                verbose=-1\n            )\n            \n            models['cat_complex'] = CatBoostRegressor(\n                iterations=500,  # Reduced\n                depth=8,  # Reduced\n                learning_rate=0.015,\n                l2_leaf_reg=50 * reg_mult,\n                min_data_in_leaf=80,\n                random_state=42,\n                task_type='GPU' if use_gpu else 'CPU',\n                verbose=False\n            )\n            \n            models['rf_complex'] = RandomForestRegressor(\n                n_estimators=200,  # Reduced\n                max_depth=15,  # Reduced\n                min_samples_split=50,\n                min_samples_leaf=20,\n                max_features='sqrt',\n                random_state=42,\n                n_jobs=-1\n            )\n            \n            models['extra_trees'] = ExtraTreesRegressor(\n                n_estimators=200,  # Reduced\n                max_depth=15,  # Reduced\n                min_samples_split=50,\n                min_samples_leaf=20,\n                max_features='sqrt',\n                random_state=42,\n                n_jobs=-1\n            )\n        \n        return models\n    \n    def train_and_evaluate(self, X_train, y_train, X_val, y_val, models, sample_weight=None):\n        \"\"\"Train models and return scores with anti-overfitting wrapper\"\"\"\n        scores = {}\n        predictions = {}\n        wrapped_models = {}\n        \n        for name, model in models.items():\n            try:\n                # Wrap model with anti-overfitting\n                wrapped_model = AntiOverfittingWrapper(\n                    model, \n                    model_type='tree' if name not in ['ridge', 'lasso', 'elastic', 'huber'] else 'linear'\n                )\n                \n                # Handle different model types\n                if name.startswith(('xgb', 'lgb', 'cat')) and sample_weight is not None:\n                    if name.startswith('lgb'):\n                        model.fit(\n                            X_train, y_train,\n                            sample_weight=sample_weight,\n                            eval_set=[(X_val, y_val)],\n                            callbacks=[lgb.early_stopping(config.early_stopping_rounds), lgb.log_evaluation(0)]\n                        )\n                    elif name.startswith('xgb'):\n                        model.fit(\n                            X_train, y_train,\n                            sample_weight=sample_weight,\n                            eval_set=[(X_val, y_val)],\n                            early_stopping_rounds=config.early_stopping_rounds,\n                            verbose=False\n                        )\n                    elif name.startswith('cat'):\n                        model.fit(\n                            X_train, y_train,\n                            sample_weight=sample_weight,\n                            eval_set=(X_val, y_val),\n                            early_stopping_rounds=config.early_stopping_rounds,\n                            verbose=False\n                        )\n                    else:\n                        model.fit(X_train, y_train, sample_weight=sample_weight)\n                else:\n                    model.fit(X_train, y_train)\n                \n                # Update wrapper\n                wrapped_model.base_model = model\n                wrapped_model.fit(X_train, y_train, X_val, y_val)\n                \n                # Predict with anti-overfitting adjustments\n                val_pred = wrapped_model.predict(X_val)\n                score = pearsonr(y_val, val_pred)[0]\n                \n                scores[name] = score\n                predictions[name] = val_pred\n                wrapped_models[name] = wrapped_model\n                \n                if self.config.verbose:\n                    print(f\"      {name}: {score:.4f}\")\n            \n            except Exception as e:\n                print(f\"      {name}: Failed - {str(e)}\")\n                scores[name] = 0\n                predictions[name] = np.zeros_like(y_val)\n        \n        # Ensemble prediction\n        valid_predictions = [pred for name, pred in predictions.items() if scores[name] > 0]\n        if valid_predictions:\n            ensemble_pred = np.mean(valid_predictions, axis=0)\n            ensemble_score = pearsonr(y_val, ensemble_pred)[0]\n        else:\n            ensemble_score = 0\n        \n        return scores, ensemble_score, wrapped_models, predictions\n    \n    def optimize_ensemble_weights(self, predictions, y_val):\n        \"\"\"Optimize weights with constraints to prevent overfitting\"\"\"\n        valid_preds = []\n        valid_names = []\n        \n        for name, pred in predictions.items():\n            if not np.isnan(pred).any():\n                valid_preds.append(pred)\n                valid_names.append(name)\n        \n        if len(valid_preds) == 0:\n            return np.array([1.0]), 0\n        \n        predictions_array = np.column_stack(valid_preds)\n        \n        def objective(weights):\n            weights = softmax(weights)\n            ensemble_pred = predictions_array @ weights\n            return -pearsonr(y_val, ensemble_pred)[0]\n        \n        n_models = len(valid_preds)\n        \n        # Constrained optimization with max weight limit\n        def constraint_sum(weights):\n            return np.sum(softmax(weights)) - 1.0\n        \n        def constraint_max_weight(weights, idx):\n            return 0.5 - softmax(weights)[idx]  # Max weight 0.5\n        \n        constraints = [{'type': 'eq', 'fun': constraint_sum}]\n        for i in range(n_models):\n            constraints.append({'type': 'ineq', 'fun': lambda w, idx=i: constraint_max_weight(w, idx)})\n        \n        # Try differential evolution for global optimization\n        bounds = [(-2, 2) for _ in range(n_models)]\n        result = differential_evolution(\n            objective, \n            bounds, \n            seed=42, \n            maxiter=50,\n            popsize=15\n        )\n        \n        optimal_weights = softmax(result.x)\n        optimal_score = -result.fun\n        \n        # Print weight distribution\n        if self.config.verbose and len(valid_names) > 1:\n            print(\"\\n  Optimal weights:\")\n            for name, weight in zip(valid_names, optimal_weights):\n                if weight > 0.05:\n                    print(f\"    {name}: {weight:.3f}\")\n        \n        return optimal_weights, optimal_score\n\nclass TimeBasedSampler:\n    \"\"\"Create time-based sample weights\"\"\"\n    \n    @staticmethod\n    def create_weights(n_samples, method='exponential', decay_factor=0.98):\n        \"\"\"Create sample weights based on time with stronger decay\"\"\"\n        positions = np.arange(n_samples)\n        normalized_positions = positions / (n_samples - 1)\n        \n        if method == 'exponential':\n            weights = decay_factor ** (1 - normalized_positions)\n        elif method == 'linear':\n            weights = 0.5 + 0.5 * normalized_positions\n        elif method == 'sqrt':\n            weights = np.sqrt(normalized_positions + 0.1)\n        else:\n            weights = np.ones(n_samples)\n        \n        # Normalize weights\n        weights = weights * n_samples / weights.sum()\n        return weights\n\ndef reduce_mem_usage(df, name=\"\"):\n    \"\"\"Optimize memory usage\"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n            else:\n                df[col] = df[col].astype(np.float32)\n    \n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage of {name}: {start_mem:.2f} MB -> {end_mem:.2f} MB ({100*(start_mem-end_mem)/start_mem:.1f}% reduction)')\n    return df\n\ndef select_features(X, y, feature_names, max_features=200):\n    \"\"\"Select most important features using mutual information\"\"\"\n    print(f\"\\nSelecting top {max_features} features...\")\n    \n    selector = SelectKBest(score_func=mutual_info_regression, k=min(max_features, X.shape[1]))\n    X_selected = selector.fit_transform(X, y)\n    \n    # Get selected feature indices\n    selected_indices = selector.get_support(indices=True)\n    selected_features = [feature_names[i] for i in selected_indices]\n    \n    # Print top features\n    scores = selector.scores_[selected_indices]\n    top_features = sorted(zip(selected_features, scores), key=lambda x: x[1], reverse=True)[:20]\n    \n    print(\"\\nTop 20 features by mutual information:\")\n    for feat, score in top_features:\n        print(f\"  {feat}: {score:.4f}\")\n    \n    return X_selected, selected_features, selector\n\ndef adversarial_validation(X_train, X_test, sample_size=50000):\n    \"\"\"Check if train and test distributions are similar\"\"\"\n    print(\"\\nPerforming adversarial validation...\")\n    \n    # Sample data\n    n_train = min(sample_size, len(X_train))\n    n_test = min(sample_size, len(X_test))\n    \n    train_sample = X_train[np.random.choice(len(X_train), n_train, replace=False)]\n    test_sample = X_test[np.random.choice(len(X_test), n_test, replace=False)]\n    \n    # Create labels\n    X_adv = np.vstack([train_sample, test_sample])\n    y_adv = np.concatenate([np.zeros(n_train), np.ones(n_test)])\n    \n    # Train classifier\n    from sklearn.model_selection import cross_val_score\n    clf = LGBMRegressor(n_estimators=100, num_leaves=31, random_state=42, verbose=-1)\n    scores = cross_val_score(clf, X_adv, y_adv, cv=3, scoring='roc_auc')\n    \n    adv_score = scores.mean()\n    print(f\"  Adversarial validation AUC: {adv_score:.4f}\")\n    \n    if adv_score > 0.7:\n        print(\"  Warning: Train and test distributions are significantly different!\")\n    \n    return adv_score\n\nclass UltimateMetaLearner:\n    \"\"\"Enhanced meta-learner with stacking and constraints\"\"\"\n    \n    def __init__(self, base_predictions, y_true, test_predictions):\n        self.base_predictions = base_predictions\n        self.y_true = y_true\n        self.test_predictions = test_predictions\n        self.n_models = len(base_predictions)\n        \n    def optimize_weights_with_constraints(self):\n        \"\"\"Optimize weights with additional constraints\"\"\"\n        def objective(weights):\n            weighted_pred = np.zeros_like(self.base_predictions[0])\n            for i, pred in enumerate(self.base_predictions):\n                weighted_pred += weights[i] * pred\n            return -pearsonr(self.y_true, weighted_pred)[0]\n        \n        # Constraints\n        constraints = [\n            {'type': 'eq', 'fun': lambda x: np.sum(x) - 1},  # Sum to 1\n            {'type': 'ineq', 'fun': lambda x: x}  # Non-negative\n        ]\n        \n        # Add constraint to limit max weight\n        for i in range(self.n_models):\n            constraints.append({\n                'type': 'ineq', \n                'fun': lambda x, idx=i: 0.5 - x[idx]  # Max weight 0.5\n            })\n        \n        bounds = [(0, 0.5) for _ in range(self.n_models)]\n        initial_weights = np.ones(self.n_models) / self.n_models\n        \n        result = minimize(objective, initial_weights, method='SLSQP', \n                         bounds=bounds, constraints=constraints)\n        \n        return result.x\n    \n    def create_stacking_features(self, predictions):\n        \"\"\"Create enhanced features for stacking\"\"\"\n        features = np.column_stack(predictions)\n        \n        # Add statistical features\n        features = np.column_stack([\n            features,\n            np.mean(features, axis=1),\n            np.std(features, axis=1),\n            np.max(features, axis=1),\n            np.min(features, axis=1),\n            np.median(features, axis=1),\n            np.percentile(features, 25, axis=1),\n            np.percentile(features, 75, axis=1),\n            # Add rank features\n            rankdata(np.mean(features, axis=1)) / len(features)\n        ])\n        \n        return features\n\nclass UltimateIterativePipeline:\n    \"\"\"Main pipeline with merged enhancements\"\"\"\n    \n    def __init__(self, config):\n        self.config = config\n        self.feature_builder = UltimateFeatureBuilder()\n        self.model_iterator = EnhancedModelIterator(config)\n        self.iteration_results = []\n        \n    def run(self):\n        \"\"\"Run the ultimate merged pipeline\"\"\"\n        print(\"\\n📂 LOADING DATA\")\n        print(\"=\"*80)\n        \n        # Load data\n        train = pd.read_parquet(self.config.train_path)\n        test = pd.read_parquet(self.config.test_path)\n        submission = pd.read_csv(self.config.sample_sub_path)\n        \n        # Use last portion of data\n        train_size = int(len(train) * (1 - self.config.use_last_pct))\n        train = train.iloc[train_size:].reset_index(drop=True)\n        print(f\"Using last {self.config.use_last_pct*100}% of training data: {train.shape}\")\n        \n        # Reduce memory\n        train = reduce_mem_usage(train, \"train\")\n        test = reduce_mem_usage(test, \"test\")\n        \n        # Store original data\n        y_train_full = train['label'].values\n        \n        # Create time-based sample weights\n        sample_weights = TimeBasedSampler.create_weights(len(train), method='exponential', decay_factor=0.98)\n        \n        # Iterative improvement\n        levels = [\n            ('baseline', 'low'),\n            ('level1', 'low'),\n            ('level2', 'medium'),\n            ('level3', 'medium'),\n            ('level4', 'high'),\n            ('level5', 'high')\n        ]\n        \n        best_submission = None\n        all_test_predictions = []\n        all_oof_predictions = []\n        all_scores = []\n        model_names = []\n        \n        # Multiple scalers\n        scalers = {\n            'robust': RobustScaler(),\n            'standard': StandardScaler(),\n            'quantile': QuantileTransformer(n_quantiles=1000, output_distribution='normal', random_state=42)\n        }\n        \n        # Define windows for diversity\n        windows = [\n            {'name': 'last_50', 'start': int(0.50 * len(train)), 'end': len(train)},\n            {'name': 'last_33', 'start': int(0.67 * len(train)), 'end': len(train)},\n            {'name': 'last_25', 'start': int(0.75 * len(train)), 'end': len(train)},\n        ]\n        \n        # Scaling preference map\n        scaling_map = {\n            'xgb': 'robust',\n            'lgb': 'robust',\n            'cat': 'standard',\n            'rf': 'robust',\n            'ridge': 'standard',\n            'lasso': 'standard',\n            'elastic': 'standard',\n            'huber': 'robust',\n            'extra': 'robust'\n        }\n        \n        prev_window_predictions = None\n        \n        for level_idx, (level_name, complexity) in enumerate(levels):\n            print(f\"\\n{'='*80}\")\n            print(f\"ITERATION {level_idx + 1}: {level_name.upper()}\")\n            print(f\"{'='*80}\")\n            \n            # Add features progressively\n            if level_name == 'baseline':\n                features = self.feature_builder.get_baseline_features(train)\n            else:\n                # Add new features\n                if level_name == 'level1':\n                    self.feature_builder.add_level1_features(train)\n                    self.feature_builder.add_level1_features(test)\n                elif level_name == 'level2':\n                    self.feature_builder.add_level2_features(train)\n                    self.feature_builder.add_level2_features(test)\n                elif level_name == 'level3':\n                    self.feature_builder.add_level3_features(train)\n                    self.feature_builder.add_level3_features(test)\n                elif level_name == 'level4':\n                    # Get top X features\n                    x_features = [col for col in train.columns if col.startswith('X') and col[1:].isdigit()]\n                    top_x_features = self._get_top_x_features(train[x_features + ['label']], n_top=100)\n                    self.feature_builder.add_level4_features(train, top_x_features)\n                    self.feature_builder.add_level4_features(test, top_x_features)\n                elif level_name == 'level5':\n                    self.feature_builder.add_level5_features(train)\n                    self.feature_builder.add_level5_features(test)\n                \n                features = self.feature_builder.get_features_up_to_level(level_name)\n            \n            # Ensure all features exist and handle infinities\n            features = [f for f in features if f in train.columns]\n            train[features] = train[features].replace([np.inf, -np.inf], np.nan).fillna(0)\n            test[features] = test[features].replace([np.inf, -np.inf], np.nan).fillna(0)\n            \n            # Clip extreme values\n            for col in features:\n                if col in train.columns:\n                    p99 = train[col].quantile(0.99)\n                    p1 = train[col].quantile(0.01)\n                    train[col] = train[col].clip(p1, p99)\n                    test[col] = test[col].clip(p1, p99)\n            \n            print(f\"\\nUsing {len(features)} features before selection\")\n            \n            # Feature selection\n            if self.config.use_feature_selection and len(features) > self.config.max_features_before_transform:\n                X_temp = train[features].values\n                y_temp = train['label'].values\n                \n                X_selected, selected_features, selector = select_features(\n                    X_temp, y_temp, features, self.config.max_features_before_transform\n                )\n                \n                features = selected_features\n                print(f\"Selected {len(features)} features\")\n            \n            # Prepare data\n            X = train[features].values\n            y = train['label'].values\n            X_test = test[features].values\n            \n            # Adversarial validation (only once per level)\n            if level_idx == 0 and self.config.use_adversarial_validation:\n                adv_score = adversarial_validation(X, X_test)\n            \n            # Time series cross-validation\n            tscv = TimeSeriesSplit(n_splits=self.config.n_folds)\n            \n            # Try different windows for this level\n            level_test_predictions = []\n            \n            for window_idx, window in enumerate(windows):\n                if level_idx < 3 and window_idx > 0:  # Only use multiple windows for later levels\n                    continue\n                    \n                print(f\"\\n  Window: {window['name']}\")\n                window_indices = list(range(window['start'], window['end']))\n                \n                # Get window data\n                X_window = X[window_indices]\n                y_window = y[window_indices]\n                window_weights = sample_weights[window_indices]\n                \n                # Try different scalers\n                best_scaler_score = -np.inf\n                best_scaler = None\n                best_predictions = None\n                best_oof = None\n                \n                for scaler_name, scaler in scalers.items():\n                    if level_idx < 2 and scaler_name == 'quantile':  # Skip quantile for early levels\n                        continue\n                        \n                    print(f\"\\n    Testing {scaler_name} scaler...\")\n                    \n                    # Scale data\n                    X_window_scaled = scaler.fit_transform(X_window)\n                    X_test_scaled = scaler.transform(X_test)\n                    \n                    # Clip scaled values\n                    X_window_scaled = np.clip(X_window_scaled, -10, 10)\n                    X_test_scaled = np.clip(X_test_scaled, -10, 10)\n                    \n                    # Create models with appropriate regularization\n                    models = self.model_iterator.create_ensemble(\n                        complexity, USE_GPU, \n                        reg_mult=self.config.regularization_multiplier\n                    )\n                    \n                    # Cross-validation\n                    window_oof = np.full(len(X_window), np.nan)\n                    fold_test_preds = []\n                    fold_scores = []\n                    \n                    for fold, (train_idx, valid_idx) in enumerate(tscv.split(X_window_scaled)):\n                        if fold >= self.config.n_folds:\n                            break\n                            \n                        print(f\"      Fold {fold + 1}/{self.config.n_folds}\")\n                        \n                        # Get fold data\n                        X_fold_train = X_window_scaled[train_idx]\n                        y_fold_train = y_window[train_idx]\n                        X_fold_valid = X_window_scaled[valid_idx]\n                        y_fold_valid = y_window[valid_idx]\n                        fold_weights = window_weights[train_idx]\n                        \n                        # Train and evaluate\n                        scores, ensemble_score, trained_models, predictions = self.model_iterator.train_and_evaluate(\n                            X_fold_train, y_fold_train, X_fold_valid, y_fold_valid, \n                            models, fold_weights\n                        )\n                        \n                        # Optimize weights if multiple valid models\n                        valid_models = [s for s in scores.values() if s > 0]\n                        if len(valid_models) > 1:\n                            weights, optimized_score = self.model_iterator.optimize_ensemble_weights(\n                                predictions, y_fold_valid\n                            )\n                            print(f\"        Ensemble: {ensemble_score:.4f} -> Optimized: {optimized_score:.4f}\")\n                        else:\n                            weights = np.array([1.0])\n                            optimized_score = ensemble_score\n                        \n                        # Store OOF predictions\n                        if optimized_score > 0:\n                            valid_preds = [pred for name, pred in predictions.items() if scores.get(name, 0) > 0]\n                            if len(weights) == len(valid_preds):\n                                fold_oof = np.average(valid_preds, axis=0, weights=weights)\n                            else:\n                                fold_oof = np.mean(valid_preds, axis=0)\n                            \n                            window_oof[valid_idx] = fold_oof\n                            fold_scores.append(optimized_score)\n                            \n                            # Test predictions\n                            test_preds = []\n                            for name, model in trained_models.items():\n                                if scores.get(name, 0) > 0:\n                                    test_preds.append(model.predict(X_test_scaled))\n                            \n                            if len(weights) == len(test_preds):\n                                fold_test = np.average(test_preds, axis=0, weights=weights)\n                            else:\n                                fold_test = np.mean(test_preds, axis=0)\n                            \n                            fold_test_preds.append(fold_test)\n                    \n                    if fold_scores:\n                        # Average test predictions\n                        test_predictions = np.mean(fold_test_preds, axis=0)\n                        \n                        # Calculate overall score\n                        valid_mask = ~np.isnan(window_oof)\n                        if np.sum(valid_mask) > 100:\n                            scaler_score = pearsonr(y_window[valid_mask], window_oof[valid_mask])[0]\n                            \n                            print(f\"    {scaler_name} overall score: {scaler_score:.4f}\")\n                            \n                            if scaler_score > best_scaler_score:\n                                best_scaler_score = scaler_score\n                                best_scaler = scaler_name\n                                best_predictions = test_predictions\n                                best_oof = window_oof\n                \n                # Blend with previous window if enabled\n                if self.config.blend_with_previous and prev_window_predictions is not None:\n                    blend_weight = 0.7  # Current window weight\n                    best_predictions = blend_weight * best_predictions + (1 - blend_weight) * prev_window_predictions\n                \n                prev_window_predictions = best_predictions.copy()\n                \n                # Store results\n                if best_scaler_score > 0:\n                    # Create full OOF array\n                    full_oof = np.full(len(train), np.nan)\n                    full_oof[window_indices] = best_oof\n                    \n                    all_oof_predictions.append(full_oof)\n                    all_test_predictions.append(best_predictions)\n                    all_scores.append(best_scaler_score)\n                    model_names.append(f\"{level_name}_{window['name']}_{best_scaler}\")\n                    \n                    print(f\"\\n  Window {window['name']} best: {best_scaler} (score: {best_scaler_score:.4f})\")\n                    \n                    level_test_predictions.append(best_predictions)\n            \n            # Blend predictions within level\n            if level_test_predictions:\n                level_ensemble = np.mean(level_test_predictions, axis=0)\n                \n                # Blend with previous levels for stability\n                if level_idx > 0 and all_test_predictions:\n                    level_ensemble = 0.7 * level_ensemble + 0.3 * np.mean(all_test_predictions[-3:], axis=0)\n                \n                # Store level results\n                self.iteration_results.append({\n                    'level': level_name,\n                    'n_features': len(features),\n                    'complexity': complexity,\n                    'best_score': best_scaler_score,\n                    'n_windows': len(level_test_predictions)\n                })\n                \n                # Save intermediate submission\n                if self.config.save_intermediate:\n                    iteration_submission = submission.copy()\n                    iteration_submission['prediction'] = level_ensemble\n                    \n                    # Clip predictions\n                    p95, p5 = np.percentile(y_train_full, [95, 5])\n                    iteration_submission['prediction'] = np.clip(iteration_submission['prediction'], p5, p95)\n                    \n                    filename = f'submission_{level_name}_merged.csv'\n                    iteration_submission.to_csv(filename, index=False)\n                    print(f\"  Saved: {filename}\")\n            \n            # Clean up memory\n            gc.collect()\n            if USE_GPU:\n                torch.cuda.empty_cache()\n        \n        # Final ensemble selection\n        print(\"\\n\" + \"=\"*80)\n        print(\"CREATING FINAL ENSEMBLE\")\n        print(\"=\"*80)\n        \n        # Select best models by score and diversity\n        sorted_indices = np.argsort(all_scores)[::-1]\n        \n        # Select diverse top models\n        selected_indices = []\n        selected_types = {}\n        \n        for idx in sorted_indices:\n            model_type = model_names[idx].split('_')[0]\n            \n            # Limit each level type\n            if selected_types.get(model_type, 0) < 2:\n                selected_indices.append(idx)\n                selected_types[model_type] = selected_types.get(model_type, 0) + 1\n            \n            if len(selected_indices) >= 12:  # Select top 12 diverse models\n                break\n        \n        print(f\"\\nTop {len(selected_indices)} models selected:\")\n        for i, idx in enumerate(selected_indices):\n            print(f\"{i+1:2d}. {model_names[idx]}: {all_scores[idx]:.4f}\")\n        \n        # Prepare selected predictions\n        selected_oof = []\n        selected_test = []\n        \n        for idx in selected_indices:\n            oof = all_oof_predictions[idx].copy()\n            # Fill NaN with mean\n            valid_mask = ~np.isnan(oof)\n            if valid_mask.sum() > 0:\n                oof[~valid_mask] = np.nanmean(oof)\n            else:\n                oof[:] = 0\n            \n            selected_oof.append(oof)\n            selected_test.append(all_test_predictions[idx])\n        \n        # Create meta-learner\n        meta_learner = UltimateMetaLearner(selected_oof, y_train_full, selected_test)\n        \n        # Multiple ensemble methods\n        print(\"\\nCreating ensemble predictions...\")\n        \n        # 1. Simple average\n        simple_avg = np.mean(selected_test, axis=0)\n        \n        # 2. Optimized weights\n        optimal_weights = meta_learner.optimize_weights_with_constraints()\n        weighted_avg = np.zeros_like(simple_avg)\n        for i, weight in enumerate(optimal_weights):\n            weighted_avg += weight * selected_test[i]\n        \n        print(\"\\nOptimal weights:\")\n        for i, (idx, weight) in enumerate(zip(selected_indices, optimal_weights)):\n            if weight > 0.05:\n                print(f\"  {model_names[idx]}: {weight:.3f}\")\n        \n        # 3. Median (robust to outliers)\n        median_pred = np.median(selected_test, axis=0)\n        \n        # 4. Trimmed mean (remove extremes)\n        sorted_preds = np.sort(selected_test, axis=0)\n        trim_count = max(1, len(selected_test) // 5)\n        trimmed_mean = np.mean(sorted_preds[trim_count:-trim_count], axis=0)\n        \n        # 5. Stacking\n        stacking_features_train = meta_learner.create_stacking_features(selected_oof)\n        stacking_features_test = meta_learner.create_stacking_features(selected_test)\n        \n        # Simple stacking model with strong regularization\n        stacker = Ridge(alpha=50.0)\n        stacker.fit(stacking_features_train, y_train_full)\n        stacking_pred = stacker.predict(stacking_features_test)\n        \n        # Final ensemble with conservative weights\n        final_predictions = (\n            0.35 * weighted_avg +      # Optimized weights\n            0.25 * median_pred +        # Robust median\n            0.20 * trimmed_mean +       # Trimmed mean\n            0.15 * simple_avg +         # Simple average\n            0.05 * stacking_pred        # Stacking (small weight due to potential overfit)\n        )\n        \n        print(\"\\nFinal ensemble weights:\")\n        print(\"  Weighted average: 35%\")\n        print(\"  Median: 25%\")\n        print(\"  Trimmed mean: 20%\")\n        print(\"  Simple average: 15%\")\n        print(\"  Stacking: 5%\")\n        \n        # Post-processing\n        print(\"\\n🔨 POST-PROCESSING\")\n        print(\"=\"*80)\n        \n        # Clip to training data range\n        p95, p5 = np.percentile(y_train_full, [95, 5])\n        final_predictions = np.clip(final_predictions, p5, p95)\n        \n        # Smooth predictions slightly\n        final_predictions = gaussian_filter1d(final_predictions, sigma=0.5)\n        \n        # Final clipping\n        final_predictions = np.clip(final_predictions, p5, p95)\n        \n        # Create submission\n        final_submission = submission.copy()\n        final_submission['prediction'] = final_predictions\n        final_submission.to_csv('submission_ultimate_merged.csv', index=False)\n        print(f\"\\n✅ Final ensemble submission saved!\")\n        \n        # Print statistics\n        print(f\"\\nPrediction statistics:\")\n        print(f\"  Mean: {final_predictions.mean():.4f}\")\n        print(f\"  Std: {final_predictions.std():.4f}\")\n        print(f\"  Min: {final_predictions.min():.4f}\")\n        print(f\"  Max: {final_predictions.max():.4f}\")\n        \n        # Also save best single model predictions\n        best_idx = sorted_indices[0]\n        best_submission = submission.copy()\n        best_submission['prediction'] = np.clip(all_test_predictions[best_idx], p5, p95)\n        best_submission.to_csv('submission_best_single_merged.csv', index=False)\n        print(f\"✅ Best single model saved ({model_names[best_idx]}, Score: {all_scores[best_idx]:.4f})\")\n        \n        # Print summary\n        self._print_summary()\n        \n        # Save detailed results\n        self._save_results()\n    \n    def _get_top_x_features(self, data, n_top=100):\n        \"\"\"Get top X features based on various metrics\"\"\"\n        correlations = data.corr()['label'].abs().sort_values(ascending=False)\n        top_features = [f for f in correlations.index if f.startswith('X')][:n_top]\n        \n        # Also consider features with high variance\n        x_cols = [col for col in data.columns if col.startswith('X')]\n        variances = data[x_cols].var().sort_values(ascending=False)\n        high_var_features = variances.index[:30].tolist()\n        \n        # Combine both lists (unique features)\n        combined_features = list(dict.fromkeys(top_features + high_var_features))\n        \n        return combined_features[:n_top]\n    \n    def _print_summary(self):\n        \"\"\"Print summary of all iterations\"\"\"\n        print(\"\\n\" + \"=\"*80)\n        print(\"ITERATION SUMMARY\")\n        print(\"=\"*80)\n        \n        print(\"\\n{:<10} {:<10} {:<15} {:<12} {:<10}\".format(\n            \"Level\", \"Features\", \"Complexity\", \"Best Score\", \"Windows\"\n        ))\n        print(\"-\" * 70)\n        \n        for result in self.iteration_results:\n            print(\"{:<10} {:<10} {:<15} {:<12.4f} {:<10}\".format(\n                result['level'],\n                result['n_features'],\n                result['complexity'],\n                result['best_score'],\n                result['n_windows']\n            ))\n    \n    def _save_results(self):\n        \"\"\"Save detailed results\"\"\"\n        clean_results = []\n        for result in self.iteration_results:\n            clean_result = {\n                'level': result['level'],\n                'n_features': int(result['n_features']),\n                'complexity': result['complexity'],\n                'best_score': float(result['best_score']),\n                'n_windows': int(result['n_windows'])\n            }\n            clean_results.append(clean_result)\n        \n        results = {\n            'iterations': clean_results,\n            'feature_sets': self.feature_builder.feature_sets,\n            'timestamp': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),\n            'config': {\n                'use_last_pct': self.config.use_last_pct,\n                'n_folds': self.config.n_folds,\n                'regularization_multiplier': self.config.regularization_multiplier,\n                'max_features_before_transform': self.config.max_features_before_transform\n            }\n        }\n        \n        with open('iteration_results_ultimate_merged.json', 'w') as f:\n            json.dump(results, f, indent=2)\n        \n        print(f\"\\n📊 Detailed results saved to iteration_results_ultimate_merged.json\")\n\n# Main execution\nif __name__ == \"__main__\":\n    pipeline = UltimateIterativePipeline(config)\n    pipeline.run()\n    \n    print(\"\\n\" + \"=\"*100)\n    print(\"✨ ULTIMATE MERGED PIPELINE COMPLETED!\")\n    print(\"=\"*100)\n    print(f\"End Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}