{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"},{"sourceId":12405352,"sourceType":"datasetVersion","datasetId":7823194}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Models\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, HistGradientBoostingRegressor\nfrom sklearn.linear_model import Ridge, Lasso, ElasticNet, HuberRegressor, BayesianRidge\nfrom sklearn.svm import SVR\nfrom sklearn.neural_network import MLPRegressor\n\n# Utilities\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, QuantileTransformer\nfrom scipy.stats import pearsonr, spearmanr\nfrom scipy.optimize import minimize, differential_evolution\nimport lightgbm as lgb\nimport xgboost as xgb\nimport os\nimport gc\nfrom typing import Dict, List, Tuple, Optional\nimport copy\n\nprint(\"Starting Working DRW Crypto Market Prediction Pipeline...\")\nprint(\"=\" * 100)\n\n# Configuration\nclass CFG:\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    n_folds = 5\n    random_state = 42\n    use_gpu = False  # Set to False for stability\n    \n    # Anti-overfitting parameters\n    use_discriminator = True\n    overfit_threshold = 0.15  # Gap between train and validation score\n    confidence_decay = 0.8  # Decay factor for overfit models\n    min_confidence = 0.3  # Minimum confidence for a model\n    \n    # Feature engineering - kept simple\n    use_lag_features = True\n    lag_periods = [1, 5, 10]\n    use_rolling_features = True\n    rolling_periods = [5, 10, 20]\n\n# Memory optimization\ndef reduce_mem_usage(df, name=\"\"):\n    print(f\"Optimizing memory for {name}...\")\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n    \n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage: {start_mem:.2f} MB -> {end_mem:.2f} MB ({100*(start_mem-end_mem)/start_mem:.1f}% reduction)')\n    return df\n\n# Enhanced feature engineering\ndef add_features(df):\n    \"\"\"Create comprehensive features for market microstructure\"\"\"\n    print(\"Engineering features...\")\n    \n    eps = 1e-8\n    \n    # Basic features from original\n    df['bid_ask_spread'] = df['ask_qty'] - df['bid_qty']\n    df['bid_ask_ratio'] = df['bid_qty'] / (df['ask_qty'] + eps)\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + eps)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['volume'] + eps)\n    \n    # Pressure indicators\n    df['buying_pressure'] = df['buy_qty'] / (df['volume'] + eps)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + eps)\n    df['net_pressure'] = df['buying_pressure'] - df['selling_pressure']\n    \n    # Liquidity features\n    df['total_liquidity'] = df['bid_qty'] + df['ask_qty']\n    df['liquidity_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_liquidity'] + eps)\n    df['liquidity_ratio'] = df['total_liquidity'] / (df['volume'] + eps)\n    \n    # Volume features\n    df['log_volume'] = np.log1p(df['volume'])\n    df['sqrt_volume'] = np.sqrt(df['volume'])\n    \n    # Market microstructure indicators\n    df['kyle_lambda'] = df['order_flow_imbalance'] / (df['sqrt_volume'] + eps)\n    df['vpin'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + eps)\n    \n    # Simple interaction features\n    df['spread_volume'] = df['bid_ask_spread'] * df['log_volume']\n    df['imbalance_volume'] = df['order_flow_imbalance'] * df['volume']\n    df['pressure_spread'] = df['net_pressure'] * df['bid_ask_spread']\n    \n    # Handle infinities and NaNs\n    numeric_cols = df.select_dtypes(include=[np.number]).columns\n    for col in numeric_cols:\n        df[col] = df[col].replace([np.inf, -np.inf], np.nan)\n        df[col] = df[col].fillna(0)\n        df[col] = np.clip(df[col], -1e8, 1e8)\n    \n    return df\n\n# Simple temporal features\ndef add_temporal_features(df):\n    \"\"\"Add simple lag and rolling features\"\"\"\n    print(\"Adding temporal features...\")\n    \n    # Key features to create lags/rolling for\n    key_features = ['volume', 'order_flow_imbalance', 'kyle_lambda', 'vpin']\n    \n    # Lag features\n    if CFG.use_lag_features:\n        for feature in key_features:\n            if feature in df.columns:\n                for lag in CFG.lag_periods:\n                    df[f'{feature}_lag_{lag}'] = df[feature].shift(lag)\n    \n    # Rolling features\n    if CFG.use_rolling_features:\n        for feature in key_features:\n            if feature in df.columns:\n                for period in CFG.rolling_periods:\n                    df[f'{feature}_roll_mean_{period}'] = df[feature].rolling(period, min_periods=1).mean()\n                    df[f'{feature}_roll_std_{period}'] = df[feature].rolling(period, min_periods=1).std()\n    \n    # Fill NaN values - use bfill then ffill then 0\n    df = df.bfill().ffill().fillna(0)\n    \n    return df\n\n# Feature selection based on analysis\ndef get_selected_features():\n    # Top anonymized features from analysis\n    top_x_features = [\n        \"X752\", \"X287\", \"X298\", \"X759\", \"X302\", \"X55\", \"X56\", \"X52\", \"X303\", \"X51\",\n        \"X344\", \"X598\", \"X385\", \"X603\", \"X674\", \"X415\", \"X345\", \"X137\", \"X174\", \"X178\",\n        \"X758\", \"X296\", \"X611\", \"X780\", \"X451\", \"X25\", \"X591\", \"X673\", \"X274\", \"X286\"\n    ]\n    \n    # Market microstructure features\n    market_features = [\n        'bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume',\n        'bid_ask_spread', 'bid_ask_ratio', 'buy_sell_ratio', 'order_flow_imbalance',\n        'buying_pressure', 'selling_pressure', 'net_pressure',\n        'total_liquidity', 'liquidity_imbalance', 'liquidity_ratio',\n        'log_volume', 'sqrt_volume', 'kyle_lambda', 'vpin',\n        'spread_volume', 'imbalance_volume', 'pressure_spread'\n    ]\n    \n    # Add temporal features if enabled\n    temporal_features = []\n    if CFG.use_lag_features:\n        for feat in ['volume', 'order_flow_imbalance', 'kyle_lambda', 'vpin']:\n            for lag in CFG.lag_periods:\n                temporal_features.append(f'{feat}_lag_{lag}')\n    \n    if CFG.use_rolling_features:\n        for feat in ['volume', 'order_flow_imbalance', 'kyle_lambda', 'vpin']:\n            for period in CFG.rolling_periods:\n                temporal_features.append(f'{feat}_roll_mean_{period}')\n                temporal_features.append(f'{feat}_roll_std_{period}')\n    \n    return top_x_features + market_features + temporal_features\n\n# Anti-Overfitting Discriminator\nclass AntiOverfittingDiscriminator:\n    def __init__(self, overfit_threshold=0.15, confidence_decay=0.8, min_confidence=0.3):\n        self.overfit_threshold = overfit_threshold\n        self.confidence_decay = confidence_decay\n        self.min_confidence = min_confidence\n        self.model_metrics = {}\n        \n    def evaluate_model_overfitting(self, model_name: str, train_scores: List[float], \n                                  valid_scores: List[float]) -> Dict:\n        train_mean = np.mean(train_scores)\n        valid_mean = np.mean(valid_scores)\n        \n        # Calculate overfitting metrics\n        overfit_gap = train_mean - valid_mean\n        score_variance = np.var(valid_scores)\n        score_stability = 1 / (1 + score_variance)\n        \n        # Determine if model is overfitting\n        is_overfitting = overfit_gap > self.overfit_threshold\n        \n        # Calculate confidence score\n        if is_overfitting:\n            confidence = max(self.min_confidence, \n                           1 - (overfit_gap / self.overfit_threshold) * (1 - self.confidence_decay))\n        else:\n            confidence = min(1.0, score_stability * (1 + valid_mean))\n        \n        metrics = {\n            'train_score': train_mean,\n            'valid_score': valid_mean,\n            'overfit_gap': overfit_gap,\n            'score_variance': score_variance,\n            'score_stability': score_stability,\n            'is_overfitting': is_overfitting,\n            'confidence': confidence\n        }\n        \n        self.model_metrics[model_name] = metrics\n        return metrics\n    \n    def get_adjusted_weights(self, model_names: List[str], base_weights: np.ndarray) -> np.ndarray:\n        adjusted_weights = base_weights.copy()\n        \n        for i, model_name in enumerate(model_names):\n            if model_name in self.model_metrics:\n                confidence = self.model_metrics[model_name]['confidence']\n                adjusted_weights[i] *= confidence\n        \n        # Renormalize\n        if adjusted_weights.sum() > 0:\n            adjusted_weights /= adjusted_weights.sum()\n        else:\n            adjusted_weights = base_weights\n            \n        return adjusted_weights\n\n# Improved model definitions\ndef get_models_with_antioverfit():\n    models = {\n        # LightGBM variants\n        'lgb_conservative': LGBMRegressor(\n            n_estimators=500,\n            learning_rate=0.01,\n            num_leaves=31,\n            max_depth=5,\n            min_child_samples=50,\n            subsample=0.6,\n            colsample_bytree=0.6,\n            reg_alpha=20,\n            reg_lambda=20,\n            random_state=42,\n            n_jobs=-1,\n            verbose=-1\n        ),\n        \n        'lgb_standard': LGBMRegressor(\n            n_estimators=400,\n            learning_rate=0.02,\n            num_leaves=63,\n            max_depth=7,\n            min_child_samples=30,\n            subsample=0.8,\n            colsample_bytree=0.8,\n            reg_alpha=10,\n            reg_lambda=10,\n            random_state=42,\n            n_jobs=-1,\n            verbose=-1\n        ),\n        \n        'lgb_aggressive': LGBMRegressor(\n            n_estimators=300,\n            learning_rate=0.03,\n            num_leaves=127,\n            max_depth=-1,\n            min_child_samples=20,\n            subsample=0.9,\n            colsample_bytree=0.9,\n            reg_alpha=5,\n            reg_lambda=5,\n            random_state=42,\n            n_jobs=-1,\n            verbose=-1\n        ),\n        \n        # XGBoost variants\n        'xgb_conservative': XGBRegressor(\n            n_estimators=800,\n            learning_rate=0.01,\n            max_depth=4,\n            subsample=0.6,\n            colsample_bytree=0.6,\n            gamma=2,\n            reg_alpha=5,\n            reg_lambda=5,\n            min_child_weight=25,\n            random_state=42,\n            tree_method='hist',\n            verbosity=0\n        ),\n        \n        'xgb_standard': XGBRegressor(\n            n_estimators=600,\n            learning_rate=0.015,\n            max_depth=6,\n            subsample=0.8,\n            colsample_bytree=0.8,\n            gamma=1,\n            reg_alpha=2,\n            reg_lambda=2,\n            min_child_weight=10,\n            random_state=42,\n            tree_method='hist',\n            verbosity=0\n        ),\n        \n        # CatBoost variants\n        'cat_conservative': CatBoostRegressor(\n            iterations=800,\n            learning_rate=0.02,\n            depth=4,\n            l2_leaf_reg=20,\n            random_state=42,\n            verbose=False,\n            task_type='CPU'\n        ),\n        \n        'cat_standard': CatBoostRegressor(\n            iterations=600,\n            learning_rate=0.03,\n            depth=6,\n            l2_leaf_reg=10,\n            random_state=42,\n            verbose=False,\n            task_type='CPU'\n        ),\n        \n        # Other models\n        'rf_conservative': RandomForestRegressor(\n            n_estimators=200,\n            max_depth=10,\n            min_samples_split=50,\n            min_samples_leaf=20,\n            max_features='sqrt',\n            random_state=42,\n            n_jobs=-1\n        ),\n        \n        'extra_trees': ExtraTreesRegressor(\n            n_estimators=200,\n            max_depth=12,\n            min_samples_split=40,\n            min_samples_leaf=15,\n            max_features='sqrt',\n            random_state=42,\n            n_jobs=-1\n        ),\n        \n        'hist_gb': HistGradientBoostingRegressor(\n            max_iter=300,\n            learning_rate=0.05,\n            max_depth=8,\n            min_samples_leaf=20,\n            l2_regularization=1.0,\n            random_state=42\n        ),\n        \n        # Linear models - Fixed HuberRegressor (no random_state parameter)\n        'ridge': Ridge(alpha=1.0, random_state=42),\n        'lasso': Lasso(alpha=0.01, random_state=42, max_iter=2000),\n        'elastic': ElasticNet(alpha=0.1, l1_ratio=0.5, random_state=42, max_iter=2000),\n        'huber': HuberRegressor(epsilon=1.35, alpha=1.0),  # No random_state\n        'bayesian_ridge': BayesianRidge(alpha_1=1e-6, alpha_2=1e-6, lambda_1=1e-6, lambda_2=1e-6),\n    }\n    \n    return models\n\n# Time-based sample weights\ndef create_time_weights(n_samples, decay_factor=0.95):\n    positions = np.arange(n_samples)\n    normalized_positions = positions / (n_samples - 1)\n    weights = decay_factor ** (1 - normalized_positions)\n    weights = weights * n_samples / weights.sum()\n    return weights\n\n# Training function with overfitting tracking\ndef train_model_with_tracking(model, X_train, y_train, X_valid, y_valid, \n                            sample_weight=None, model_name=\"\"):\n    if model_name.startswith('lgb'):\n        model.fit(\n            X_train, y_train,\n            sample_weight=sample_weight,\n            eval_set=[(X_valid, y_valid)],\n            callbacks=[lgb.early_stopping(50), lgb.log_evaluation(0)]\n        )\n    elif model_name.startswith('xgb'):\n        model.fit(\n            X_train, y_train,\n            sample_weight=sample_weight,\n            eval_set=[(X_valid, y_valid)],\n            early_stopping_rounds=50,\n            verbose=False\n        )\n    elif model_name.startswith('cat'):\n        model.fit(\n            X_train, y_train,\n            sample_weight=sample_weight,\n            eval_set=(X_valid, y_valid),\n            early_stopping_rounds=50,\n            verbose=False\n        )\n    else:\n        if hasattr(model, 'fit') and 'sample_weight' in model.fit.__code__.co_varnames:\n            model.fit(X_train, y_train, sample_weight=sample_weight)\n        else:\n            model.fit(X_train, y_train)\n    \n    # Get train and validation scores\n    train_pred = model.predict(X_train)\n    valid_pred = model.predict(X_valid)\n    \n    train_score = pearsonr(y_train, train_pred)[0]\n    valid_score = pearsonr(y_valid, valid_pred)[0]\n    \n    return model, train_score, valid_score, valid_pred\n\n# Enhanced Meta-learner\nclass EnhancedMetaLearner:\n    def __init__(self, base_predictions, y_true, test_predictions, discriminator=None):\n        self.base_predictions = base_predictions\n        self.y_true = y_true\n        self.test_predictions = test_predictions\n        self.n_models = len(base_predictions)\n        self.discriminator = discriminator\n        \n    def optimize_weights(self, method='SLSQP'):\n        \"\"\"Find optimal weights using scipy optimization\"\"\"\n        def objective(weights):\n            weighted_pred = np.zeros_like(self.base_predictions[0])\n            for i, pred in enumerate(self.base_predictions):\n                weighted_pred += weights[i] * pred\n            return -pearsonr(self.y_true, weighted_pred)[0]\n        \n        constraints = {'type': 'eq', 'fun': lambda x: np.sum(x) - 1}\n        bounds = [(0, 1) for _ in range(self.n_models)]\n        initial_weights = np.ones(self.n_models) / self.n_models\n        \n        # Try SLSQP optimization\n        best_weights = initial_weights\n        best_score = float('inf')\n        \n        try:\n            result = minimize(objective, initial_weights, method='SLSQP', \n                            bounds=bounds, constraints=constraints)\n            if result.success:\n                best_score = result.fun\n                best_weights = result.x\n        except:\n            print(\"  Warning: SLSQP optimization failed, using equal weights\")\n        \n        # Apply discriminator adjustments if available\n        if self.discriminator is not None:\n            model_names = [f\"model_{i}\" for i in range(self.n_models)]\n            best_weights = self.discriminator.get_adjusted_weights(model_names, best_weights)\n        \n        return best_weights\n    \n    def get_stacking_features(self, predictions):\n        \"\"\"Create features for meta-learner from base predictions\"\"\"\n        features = np.column_stack(predictions)\n        \n        # Add simple statistical features\n        features = np.column_stack([\n            features,\n            np.mean(features, axis=1),\n            np.std(features, axis=1),\n            np.max(features, axis=1),\n            np.min(features, axis=1),\n            np.median(features, axis=1)\n        ])\n        \n        return features\n    \n    def train_meta_models(self):\n        \"\"\"Train multiple meta-learners\"\"\"\n        train_features = self.get_stacking_features(self.base_predictions)\n        test_features = self.get_stacking_features(self.test_predictions)\n        \n        # Handle any remaining NaNs/Infs\n        train_features = np.nan_to_num(train_features, nan=0.0, posinf=1e8, neginf=-1e8)\n        test_features = np.nan_to_num(test_features, nan=0.0, posinf=1e8, neginf=-1e8)\n        \n        meta_models = {\n            'ridge': Ridge(alpha=1.0),\n            'huber': HuberRegressor(epsilon=1.35),\n            'bayesian_ridge': BayesianRidge(),\n            'elastic': ElasticNet(alpha=0.1, l1_ratio=0.5),\n            'lgb_meta': LGBMRegressor(\n                n_estimators=100,\n                learning_rate=0.05,\n                num_leaves=31,\n                max_depth=5,\n                random_state=42,\n                verbose=-1\n            )\n        }\n        \n        meta_predictions = {}\n        for name, model in meta_models.items():\n            try:\n                model.fit(train_features, self.y_true)\n                meta_predictions[name] = model.predict(test_features)\n            except Exception as e:\n                print(f\"  Warning: {name} meta-model failed: {str(e)}\")\n                meta_predictions[name] = np.mean(test_features[:, :self.n_models], axis=1)\n        \n        return meta_predictions\n\n# Main training pipeline\ndef train_and_predict():\n    print(\"\\nLoading data...\")\n    train = pd.read_parquet(CFG.train_path)\n    test = pd.read_parquet(CFG.test_path)\n    submission = pd.read_csv(CFG.sample_sub_path)\n    \n    print(f\"Train shape: {train.shape}\")\n    print(f\"Test shape: {test.shape}\")\n    \n    # Feature engineering\n    train = add_features(train)\n    test = add_features(test)\n    \n    # Add temporal features\n    train = add_temporal_features(train)\n    test = add_temporal_features(test)\n    \n    # Memory optimization\n    train = reduce_mem_usage(train, \"train\")\n    test = reduce_mem_usage(test, \"test\")\n    \n    # Feature selection\n    features = get_selected_features()\n    features = [f for f in features if f in train.columns]\n    print(f\"\\nUsing {len(features)} features\")\n    \n    # Handle potential missing columns in test set\n    for col in features:\n        if col not in test.columns:\n            test[col] = 0\n    \n    X = train[features]\n    y = train['label']\n    X_test = test[features]\n    \n    # Scaling\n    print(\"\\nScaling features...\")\n    scaler = RobustScaler()\n    X_scaled = scaler.fit_transform(X)\n    X_test_scaled = scaler.transform(X_test)\n    \n    # Clip extreme values\n    X_scaled = np.clip(X_scaled, -10, 10)\n    X_test_scaled = np.clip(X_test_scaled, -10, 10)\n    \n    # Convert back to DataFrame\n    X_scaled = pd.DataFrame(X_scaled, columns=features, index=X.index)\n    X_test_scaled = pd.DataFrame(X_test_scaled, columns=features, index=X_test.index)\n    \n    # Initialize discriminator\n    discriminator = AntiOverfittingDiscriminator(\n        overfit_threshold=CFG.overfit_threshold,\n        confidence_decay=CFG.confidence_decay,\n        min_confidence=CFG.min_confidence\n    ) if CFG.use_discriminator else None\n    \n    # Initialize storage\n    models = get_models_with_antioverfit()\n    all_oof_predictions = {}\n    all_test_predictions = {}\n    all_scores = {}\n    model_metrics = {}\n    \n    # Define data windows\n    windows = [\n        {'name': 'full', 'start': 0, 'end': len(train)},\n        {'name': 'last_70', 'start': int(0.30 * len(train)), 'end': len(train)},\n        {'name': 'last_50', 'start': int(0.50 * len(train)), 'end': len(train)},\n        {'name': 'last_30', 'start': int(0.70 * len(train)), 'end': len(train)},\n    ]\n    \n    # Cross-validation\n    print(\"\\nStarting cross-validation...\")\n    for window in windows:\n        print(f\"\\n{'='*50}\")\n        print(f\"Training on {window['name']} window\")\n        print(f\"{'='*50}\")\n        \n        # Get window data\n        X_window = X_scaled.iloc[window['start']:window['end']]\n        y_window = y.iloc[window['start']:window['end']]\n        \n        # Create time weights\n        sample_weights = create_time_weights(len(X_window), decay_factor=0.95)\n        \n        # Window-specific predictions\n        window_oof = {name: np.full(len(train), np.nan) for name in models}\n        window_test = {name: np.zeros(len(test)) for name in models}\n        window_train_scores = {name: [] for name in models}\n        window_valid_scores = {name: [] for name in models}\n        \n        # K-Fold cross-validation\n        kf = KFold(n_splits=CFG.n_folds, shuffle=True, random_state=CFG.random_state)\n        \n        for fold, (train_idx, valid_idx) in enumerate(kf.split(X_window)):\n            print(f\"\\nFold {fold + 1}/{CFG.n_folds}\")\n            \n            # Adjust indices to original dataframe\n            train_idx_global = train_idx + window['start']\n            valid_idx_global = valid_idx + window['start']\n            \n            X_train = X_window.iloc[train_idx]\n            y_train = y_window.iloc[train_idx]\n            X_valid = X_window.iloc[valid_idx]\n            y_valid = y_window.iloc[valid_idx]\n            \n            # Get weights for training\n            train_weights = sample_weights[train_idx]\n            \n            # Train each model\n            for model_name, model in models.items():\n                print(f\"  Training {model_name}...\", end='')\n                \n                try:\n                    # Clone model\n                    model_clone = copy.deepcopy(model)\n                    \n                    # Train with tracking\n                    trained_model, train_score, valid_score, valid_pred = train_model_with_tracking(\n                        model_clone, X_train, y_train, X_valid, y_valid,\n                        sample_weight=train_weights, model_name=model_name\n                    )\n                    \n                    # Predict on test\n                    test_pred = trained_model.predict(X_test_scaled)\n                    \n                    # Store predictions\n                    window_oof[model_name][valid_idx_global] = valid_pred\n                    window_test[model_name] += test_pred / CFG.n_folds\n                    \n                    # Track scores\n                    window_train_scores[model_name].append(train_score)\n                    window_valid_scores[model_name].append(valid_score)\n                    \n                    print(f\" Train: {train_score:.4f}, Valid: {valid_score:.4f}\")\n                    \n                except Exception as e:\n                    print(f\" Error: {str(e)}\")\n                    continue\n        \n        # Evaluate models with discriminator\n        if discriminator is not None and window['name'] != 'full':\n            print(f\"\\nAnti-Overfitting Analysis for {window['name']}:\")\n            for model_name in models:\n                if len(window_train_scores[model_name]) > 0:\n                    metrics = discriminator.evaluate_model_overfitting(\n                        f\"{model_name}_{window['name']}\",\n                        window_train_scores[model_name],\n                        window_valid_scores[model_name]\n                    )\n                    print(f\"  {model_name}: Gap={metrics['overfit_gap']:.3f}, Confidence={metrics['confidence']:.3f}\")\n        \n        # Calculate window scores\n        for model_name in models:\n            valid_mask = ~np.isnan(window_oof[model_name])\n            if np.sum(valid_mask) > 0:\n                window_score = pearsonr(y[valid_mask], window_oof[model_name][valid_mask])[0]\n                \n                key = f\"{model_name}_{window['name']}\"\n                all_scores[key] = window_score\n                all_oof_predictions[key] = window_oof[model_name]\n                all_test_predictions[key] = window_test[model_name]\n                model_metrics[key] = {\n                    'train_scores': window_train_scores[model_name],\n                    'valid_scores': window_valid_scores[model_name]\n                }\n    \n    # Filter valid scores\n    valid_scores = {k: v for k, v in all_scores.items() if not np.isnan(v) and v > 0}\n    \n    # Model selection\n    print(\"\\n\" + \"=\"*50)\n    print(\"MODEL SELECTION\")\n    print(\"=\"*50)\n    \n    # Apply discriminator confidence\n    if discriminator is not None:\n        adjusted_scores = {}\n        for model_name, score in valid_scores.items():\n            if model_name in discriminator.model_metrics:\n                confidence = discriminator.model_metrics[model_name]['confidence']\n                adjusted_scores[model_name] = score * confidence\n            else:\n                adjusted_scores[model_name] = score\n    else:\n        adjusted_scores = valid_scores\n    \n    # Sort models by adjusted score\n    sorted_scores = sorted(adjusted_scores.items(), key=lambda x: x[1], reverse=True)\n    top_n = min(20, len(sorted_scores))  # Top 20 models\n    top_models = [model[0] for model in sorted_scores[:top_n]]\n    \n    print(f\"\\nTop {top_n} models:\")\n    for i, (model, score) in enumerate(sorted_scores[:top_n]):\n        if discriminator and model in discriminator.model_metrics:\n            orig_score = valid_scores[model]\n            confidence = discriminator.model_metrics[model]['confidence']\n            print(f\"{i+1}. {model}: {orig_score:.4f} → {score:.4f} (conf: {confidence:.2f})\")\n        else:\n            print(f\"{i+1}. {model}: {score:.4f}\")\n    \n    # Prepare predictions for ensemble\n    selected_oof = []\n    selected_test = []\n    \n    for model_name in top_models:\n        oof_pred = all_oof_predictions[model_name].copy()\n        \n        # Handle NaN values\n        valid_mask = ~np.isnan(oof_pred)\n        if np.sum(valid_mask) > 0:\n            oof_pred[~valid_mask] = np.mean(oof_pred[valid_mask])\n        else:\n            oof_pred[:] = 0\n        \n        selected_oof.append(oof_pred)\n        selected_test.append(all_test_predictions[model_name])\n    \n    # Check if we have valid predictions\n    if len(selected_oof) == 0:\n        print(\"ERROR: No valid predictions found!\")\n        # Create a simple baseline prediction\n        submission['prediction'] = 0\n        submission.to_csv('submission.csv', index=False)\n        return\n    \n    # Ensemble methods\n    print(\"\\n\" + \"=\"*50)\n    print(\"ENSEMBLE CREATION\")\n    print(\"=\"*50)\n    \n    # Simple average\n    simple_avg_oof = np.mean(selected_oof, axis=0)\n    simple_avg_test = np.mean(selected_test, axis=0)\n    simple_score = pearsonr(y, simple_avg_oof)[0]\n    print(f\"Simple Average: {simple_score:.4f}\")\n    \n    # Weighted average\n    meta_learner = EnhancedMetaLearner(selected_oof, y.values, selected_test, discriminator)\n    optimal_weights = meta_learner.optimize_weights(method='SLSQP')\n    \n    weighted_oof = np.zeros_like(simple_avg_oof)\n    weighted_test = np.zeros_like(simple_avg_test)\n    \n    for i, weight in enumerate(optimal_weights):\n        weighted_oof += weight * selected_oof[i]\n        weighted_test += weight * selected_test[i]\n    \n    weighted_score = pearsonr(y, weighted_oof)[0]\n    print(f\"Weighted Average: {weighted_score:.4f}\")\n    \n    # Print top weights\n    print(\"\\nTop 10 weights:\")\n    weight_model_pairs = list(zip(optimal_weights, top_models))\n    weight_model_pairs.sort(reverse=True)\n    for weight, model in weight_model_pairs[:10]:\n        if weight > 0.01:\n            print(f\"  {model}: {weight:.4f}\")\n    \n    # Stacking\n    print(\"\\nTraining meta-learners...\")\n    meta_predictions = meta_learner.train_meta_models()\n    stacking_test = np.mean(list(meta_predictions.values()), axis=0)\n    print(f\"Stacking models trained: {len(meta_predictions)}\")\n    \n    # Final ensemble\n    print(\"\\n\" + \"=\"*50)\n    print(\"FINAL ENSEMBLE\")\n    print(\"=\"*50)\n    \n    # Combine methods\n    final_predictions = (\n        0.20 * simple_avg_test +\n        0.60 * weighted_test +\n        0.20 * stacking_test\n    )\n    print(\"Ensemble weights: Simple=20%, Weighted=60%, Stacking=20%\")\n    \n    # Post-processing: clip to reasonable range\n    p5, p95 = np.percentile(y, [5, 95])\n    final_predictions = np.clip(final_predictions, p5, p95)\n    \n    # Create submission\n    submission['prediction'] = final_predictions\n    submission.to_csv('submission.csv', index=False)\n    print(\"\\nSubmission saved to submission.csv\")\n    print(submission.head())\n    \n    # Save detailed results\n    if len(valid_scores) > 0:\n        results_df = pd.DataFrame({\n            'model': list(valid_scores.keys()),\n            'cv_score': list(valid_scores.values()),\n            'adjusted_score': [adjusted_scores.get(m, 0) for m in valid_scores.keys()]\n        }).sort_values('adjusted_score', ascending=False)\n        \n        results_df.to_csv('model_results.csv', index=False)\n        print(\"\\nModel results saved to model_results.csv\")\n    \n    # Summary\n    print(\"\\n\" + \"=\"*50)\n    print(\"SUMMARY\")\n    print(\"=\"*50)\n    print(f\"Total models trained: {len(valid_scores)}\")\n    print(f\"Models in ensemble: {len(top_models)}\")\n    if len(sorted_scores) > 0:\n        print(f\"Best individual model: {sorted_scores[0][0]} ({sorted_scores[0][1]:.4f})\")\n    print(f\"Simple average score: {simple_score:.4f}\")\n    print(f\"Weighted average score: {weighted_score:.4f}\")\n    \n    if discriminator is not None:\n        overfitting_models = sum(1 for m in discriminator.model_metrics.values() if m['is_overfitting'])\n        avg_confidence = np.mean([m['confidence'] for m in discriminator.model_metrics.values()])\n        print(f\"\\nOverfitting models: {overfitting_models}/{len(discriminator.model_metrics)}\")\n        print(f\"Average confidence: {avg_confidence:.3f}\")\n    \n    print(\"\\nPipeline completed successfully!\")\n\n# Execute pipeline\nif __name__ == \"__main__\":\n    train_and_predict()\n    print(\"\\nWorking Pipeline completed successfully!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}