{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!/usr/bin/env python\n# coding: utf-8\n\n# Install required packages\n!pip install -q lightgbm catboost xgboost\n!pip install -q scikit-learn scipy numpy pandas\n!pip install -q torch\n\nimport numpy as np\nimport pandas as pd\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Core imports\nimport gc\nimport os\nimport time\nfrom typing import Dict, List, Tuple\nimport copy\nimport random\n\n# ML imports\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, QuantileTransformer\nfrom sklearn.decomposition import PCA\nfrom sklearn.feature_selection import SelectKBest, f_regression\n\n# Models for diversity\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.linear_model import Ridge, Lasso, ElasticNet, HuberRegressor, BayesianRidge\nfrom sklearn.svm import SVR\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, GradientBoostingRegressor\nfrom sklearn.neural_network import MLPRegressor\nfrom sklearn.gaussian_process import GaussianProcessRegressor\nfrom sklearn.gaussian_process.kernels import DotProduct, WhiteKernel, RBF\nfrom sklearn.tree import DecisionTreeRegressor\n\nfrom scipy.stats import pearsonr\nfrom scipy.optimize import differential_evolution\n\n# Deep Learning\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\n# Set seeds\ndef set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\nset_seed(42)\n\nprint(\"=\" * 80)\nprint(\"DRW Crypto Market Prediction - Part 4: Maximum Diversity Models\")\nprint(\"=\" * 80)\nprint(f\"GPU Available: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n    print(f\"GPU Device: {torch.cuda.get_device_name(0)}\")\n\n# Configuration\nclass CFG:\n    # Data paths\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    # Model settings\n    n_folds = 5\n    random_state = 42\n    use_gpu = torch.cuda.is_available()\n    device = torch.device('cuda' if use_gpu else 'cpu')\n    \n    # Data settings\n    use_recent_ratio = 0.3  # Use last 30% of data\n    \n    # Neural network settings\n    batch_size = 2048\n    epochs = 30  # Reduced for speed\n    early_stopping_patience = 10\n    learning_rate = 0.001\n    weight_decay = 1e-5\n    \n    # Feature diversity\n    use_pca = True\n    pca_components = 50\n    use_feature_selection = True\n    top_k_features = 100\n    \n    # Feature lists\n    original_features = [\n        'X363', 'X405', 'X321', 'X175', 'X179', 'X137', 'X197', 'X22', 'X40', 'X181',\n        'X28', 'X169', 'X198', 'X173', 'X338', 'X288', 'X385', 'X344', 'X427', 'X587',\n        'X450', 'X97', 'X52', 'X444', 'X598', 'X379', 'X696', 'X297', 'X138', 'X572',\n        'X343', 'X586', 'X466', 'X438', 'X452', 'X459', 'X435', 'X386', 'X55', 'X341',\n        'X683', 'X428', 'X605', 'X445', 'X272', 'X180', 'X593', 'X680', 'X686', 'X692',\n        'X695', \"X603\", \"X674\", \"X421\", \"X333\", \"X415\", \"X345\", \"X174\", \"X302\", \"X178\",\n        \"X168\", \"X612\", 'X298', 'X45', 'X46', 'X39', 'X752', 'X759', 'X41', 'X42',\n        \"buy_qty\", \"sell_qty\", \"volume\", \"bid_qty\", \"ask_qty\"\n    ]\n    \n    new_features = [\n        'X758', 'X296', 'X611', 'X780', 'X451', 'X25', 'X591'\n    ]\n    \n    all_selected_features = original_features + new_features\n\n# Memory optimization\ndef reduce_mem_usage(df, verbose=True):\n    \"\"\"Optimize memory usage\"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose:\n        print(f'Memory usage: {start_mem:.2f} MB -> {end_mem:.2f} MB ({100*(start_mem-end_mem)/start_mem:.1f}% reduction)')\n    return df\n\ndef create_market_features(df):\n    \"\"\"Create essential market microstructure features\"\"\"\n    eps = 1e-8\n    \n    # Core microstructure features\n    df['bid_ask_spread'] = df['ask_qty'] - df['bid_qty']\n    df['bid_ask_ratio'] = df['bid_qty'] / (df['ask_qty'] + eps)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['volume'] + eps)\n    df['trade_volume'] = df['buy_qty'] + df['sell_qty']\n    df['total_liquidity'] = df['bid_qty'] + df['ask_qty']\n    \n    # Key microstructure metrics\n    df['kyle_lambda'] = df['order_flow_imbalance'] / (np.sqrt(df['volume']) + eps)\n    df['amihud_illiquidity'] = np.abs(df['order_flow_imbalance']) / (np.log1p(df['volume']) + eps)\n    df['pin_proxy'] = np.abs(df['order_flow_imbalance']) / (df['trade_volume'] + eps)\n    \n    # Simple transformations\n    df['volume_log'] = np.log1p(df['volume'])\n    df['volume_sqrt'] = np.sqrt(df['volume'])\n    \n    # Key ratios\n    df['bid_total_ratio'] = df['bid_qty'] / (df['total_liquidity'] + eps)\n    df['buy_volume_ratio'] = df['buy_qty'] / (df['volume'] + eps)\n    \n    # Rolling features (limited windows to save memory)\n    for w in [5, 10, 20]:\n        df[f'volume_ma_{w}'] = df['volume'].rolling(w, min_periods=1).mean()\n        df[f'ofi_ma_{w}'] = df['order_flow_imbalance'].rolling(w, min_periods=1).mean()\n        df[f'spread_ma_{w}'] = df['bid_ask_spread'].rolling(w, min_periods=1).mean()\n    \n    # Lag features (limited)\n    for lag in [1, 3, 5]:\n        df[f'volume_lag_{lag}'] = df['volume'].shift(lag)\n        df[f'ofi_lag_{lag}'] = df['order_flow_imbalance'].shift(lag)\n    \n    # Handle NaN values\n    numeric_cols = df.select_dtypes(include=[np.number]).columns\n    df[numeric_cols] = df[numeric_cols].fillna(0)\n    \n    # Clip extreme values\n    for col in numeric_cols:\n        if col not in ['label', 'timestamp']:\n            q99 = df[col].quantile(0.99)\n            q01 = df[col].quantile(0.01)\n            df[col] = df[col].clip(q01, q99)\n    \n    return df\n\n# Neural Network Architectures for Diversity\nclass ResidualBlock(nn.Module):\n    \"\"\"Residual block for deeper networks\"\"\"\n    def __init__(self, dim):\n        super().__init__()\n        self.fc1 = nn.Linear(dim, dim)\n        self.bn1 = nn.BatchNorm1d(dim)\n        self.fc2 = nn.Linear(dim, dim)\n        self.bn2 = nn.BatchNorm1d(dim)\n        self.dropout = nn.Dropout(0.1)\n        \n    def forward(self, x):\n        residual = x\n        x = F.relu(self.bn1(self.fc1(x)))\n        x = self.dropout(x)\n        x = self.bn2(self.fc2(x))\n        return F.relu(x + residual)\n\nclass ResNet(nn.Module):\n    \"\"\"Residual Network for tabular data\"\"\"\n    def __init__(self, input_dim, hidden_dim=128, num_blocks=4):\n        super().__init__()\n        self.input_layer = nn.Sequential(\n            nn.Linear(input_dim, hidden_dim),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n            nn.Dropout(0.2)\n        )\n        \n        self.residual_blocks = nn.Sequential(*[\n            ResidualBlock(hidden_dim) for _ in range(num_blocks)\n        ])\n        \n        self.output_layer = nn.Sequential(\n            nn.Linear(hidden_dim, 64),\n            nn.BatchNorm1d(64),\n            nn.ReLU(),\n            nn.Dropout(0.1),\n            nn.Linear(64, 1)\n        )\n        \n    def forward(self, x):\n        x = self.input_layer(x)\n        x = self.residual_blocks(x)\n        return self.output_layer(x).squeeze(-1)\n\nclass AttentionNN(nn.Module):\n    \"\"\"Neural network with self-attention mechanism\"\"\"\n    def __init__(self, input_dim, hidden_dim=128, num_heads=4):\n        super().__init__()\n        self.embedding = nn.Linear(input_dim, hidden_dim)\n        self.attention = nn.MultiheadAttention(hidden_dim, num_heads, batch_first=True)\n        self.norm1 = nn.LayerNorm(hidden_dim)\n        self.norm2 = nn.LayerNorm(hidden_dim)\n        \n        self.ffn = nn.Sequential(\n            nn.Linear(hidden_dim, hidden_dim * 4),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(hidden_dim * 4, hidden_dim)\n        )\n        \n        self.output = nn.Sequential(\n            nn.Linear(hidden_dim, 64),\n            nn.ReLU(),\n            nn.Dropout(0.1),\n            nn.Linear(64, 1)\n        )\n        \n    def forward(self, x):\n        # Add sequence dimension\n        x = self.embedding(x).unsqueeze(1)\n        \n        # Self-attention\n        attn_out, _ = self.attention(x, x, x)\n        x = self.norm1(x + attn_out)\n        \n        # FFN\n        ffn_out = self.ffn(x)\n        x = self.norm2(x + ffn_out)\n        \n        # Output\n        x = x.squeeze(1)\n        return self.output(x).squeeze(-1)\n\nclass GatedNN(nn.Module):\n    \"\"\"Neural network with gating mechanism\"\"\"\n    def __init__(self, input_dim, hidden_dims=[256, 128, 64]):\n        super().__init__()\n        \n        self.gates = nn.ModuleList()\n        self.transforms = nn.ModuleList()\n        \n        in_dim = input_dim\n        for hidden_dim in hidden_dims:\n            self.gates.append(nn.Sequential(\n                nn.Linear(in_dim, hidden_dim),\n                nn.Sigmoid()\n            ))\n            self.transforms.append(nn.Sequential(\n                nn.Linear(in_dim, hidden_dim),\n                nn.BatchNorm1d(hidden_dim),\n                nn.ReLU()\n            ))\n            in_dim = hidden_dim\n        \n        self.output = nn.Sequential(\n            nn.Linear(hidden_dims[-1], 32),\n            nn.ReLU(),\n            nn.Dropout(0.1),\n            nn.Linear(32, 1)\n        )\n        \n    def forward(self, x):\n        for gate, transform in zip(self.gates, self.transforms):\n            g = gate(x)\n            t = transform(x)\n            x = g * t\n        return self.output(x).squeeze(-1)\n\nclass CryptoDataset(Dataset):\n    def __init__(self, X, y=None):\n        self.X = torch.FloatTensor(X.values if hasattr(X, 'values') else X)\n        self.y = torch.FloatTensor(y.values if hasattr(y, 'values') else y) if y is not None else None\n        \n    def __len__(self):\n        return len(self.X)\n    \n    def __getitem__(self, idx):\n        if self.y is not None:\n            return self.X[idx], self.y[idx]\n        return self.X[idx]\n\ndef train_neural_network(model, train_loader, valid_loader, epochs, device, lr=0.001):\n    \"\"\"Train neural network with early stopping\"\"\"\n    model = model.to(device)\n    \n    optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=CFG.weight_decay)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5)\n    criterion = nn.MSELoss()\n    \n    best_valid_loss = float('inf')\n    patience_counter = 0\n    best_model_state = None\n    \n    for epoch in range(epochs):\n        # Training\n        model.train()\n        train_loss = 0\n        \n        for batch_x, batch_y in train_loader:\n            batch_x, batch_y = batch_x.to(device), batch_y.to(device)\n            \n            optimizer.zero_grad()\n            outputs = model(batch_x)\n            loss = criterion(outputs, batch_y)\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            optimizer.step()\n            \n            train_loss += loss.item()\n        \n        # Validation\n        model.eval()\n        valid_loss = 0\n        \n        with torch.no_grad():\n            for batch_x, batch_y in valid_loader:\n                batch_x, batch_y = batch_x.to(device), batch_y.to(device)\n                outputs = model(batch_x)\n                loss = criterion(outputs, batch_y)\n                valid_loss += loss.item()\n        \n        avg_valid_loss = valid_loss / len(valid_loader)\n        scheduler.step(avg_valid_loss)\n        \n        # Early stopping\n        if avg_valid_loss < best_valid_loss:\n            best_valid_loss = avg_valid_loss\n            best_model_state = copy.deepcopy(model.state_dict())\n            patience_counter = 0\n        else:\n            patience_counter += 1\n        \n        if patience_counter >= CFG.early_stopping_patience:\n            break\n    \n    if best_model_state is not None:\n        model.load_state_dict(best_model_state)\n    \n    return model\n\ndef get_diverse_models(input_dim, pca_dim=None):\n    \"\"\"Get a diverse set of models\"\"\"\n    models = {\n        # Gradient Boosting variants\n        'lgb_goss': LGBMRegressor(\n            n_estimators=800, learning_rate=0.02, num_leaves=31,\n            max_depth=6, min_child_samples=30, subsample=0.8,\n            colsample_bytree=0.8, reg_alpha=5, reg_lambda=5,\n            boosting_type='goss', random_state=42, n_jobs=-1, verbose=-1\n        ),\n        \n        'xgb_dart': XGBRegressor(\n            n_estimators=800, learning_rate=0.02, max_depth=6,\n            subsample=0.8, colsample_bytree=0.8,\n            booster='dart', rate_drop=0.1, skip_drop=0.5,\n            reg_alpha=5, reg_lambda=5, random_state=42,\n            verbosity=0\n        ),\n        \n        'cat_plain': CatBoostRegressor(\n            iterations=800, learning_rate=0.02, depth=6,\n            l2_leaf_reg=5, random_state=42, verbose=False,\n            boosting_type='Plain',\n            task_type='GPU' if CFG.use_gpu else 'CPU'\n        ),\n        \n        # Random Forest variants\n        'rf': RandomForestRegressor(\n            n_estimators=200, max_depth=15, min_samples_split=20,\n            min_samples_leaf=10, max_features='sqrt',\n            random_state=42, n_jobs=-1\n        ),\n        \n        'extra_trees': ExtraTreesRegressor(\n            n_estimators=200, max_depth=15, min_samples_split=20,\n            min_samples_leaf=10, max_features='sqrt',\n            random_state=42, n_jobs=-1\n        ),\n        \n        # Linear models with different regularization\n        'lasso': Lasso(alpha=0.001, random_state=42, max_iter=1000),\n        'elastic': ElasticNet(alpha=0.001, l1_ratio=0.5, random_state=42, max_iter=1000),\n        'huber': HuberRegressor(epsilon=1.35, alpha=0.001, max_iter=200),\n        'bayesian_ridge': BayesianRidge(alpha_1=1e-6, alpha_2=1e-6, lambda_1=1e-6, lambda_2=1e-6),\n        \n        # Non-parametric models\n        'knn': KNeighborsRegressor(n_neighbors=50, weights='distance', p=1, n_jobs=-1),\n        'svr_rbf': SVR(kernel='rbf', C=10.0, gamma='scale', epsilon=0.01),\n        'svr_poly': SVR(kernel='poly', C=10.0, degree=2, epsilon=0.01),\n        \n        # Neural networks\n        'resnet': ResNet(input_dim, hidden_dim=128, num_blocks=4),\n        'attention_nn': AttentionNN(input_dim, hidden_dim=128, num_heads=4),\n        'gated_nn': GatedNN(input_dim, hidden_dims=[256, 128, 64]),\n        \n        # Traditional neural network\n        'mlp': MLPRegressor(\n            hidden_layer_sizes=(200, 100, 50),\n            activation='relu', solver='adam',\n            alpha=0.001, learning_rate='adaptive',\n            learning_rate_init=0.001, max_iter=200,\n            early_stopping=True, n_iter_no_change=20,\n            random_state=42\n        ),\n        \n        # Simple models for stability\n        'ridge_simple': Ridge(alpha=1.0, random_state=42),\n        'gbm_simple': GradientBoostingRegressor(\n            n_estimators=100, learning_rate=0.1, max_depth=3,\n            subsample=0.8, random_state=42\n        ),\n    }\n    \n    # Add PCA-based models if specified\n    if pca_dim is not None:\n        models['lgb_pca'] = LGBMRegressor(\n            n_estimators=500, learning_rate=0.02, num_leaves=31,\n            max_depth=5, random_state=42, n_jobs=-1, verbose=-1\n        )\n        models['rf_pca'] = RandomForestRegressor(\n            n_estimators=100, max_depth=10, random_state=42, n_jobs=-1\n        )\n    \n    return models\n\nclass OptimizedEnsemble:\n    \"\"\"Optimized ensemble using differential evolution\"\"\"\n    def __init__(self, model_names):\n        self.model_names = model_names\n        self.weights = None\n        \n    def objective(self, weights, predictions, y_true):\n        weights = np.abs(weights) / np.abs(weights).sum()\n        ensemble_pred = sum(w * p for w, p in zip(weights, predictions))\n        return -pearsonr(y_true, ensemble_pred)[0]\n    \n    def fit(self, predictions, y_true):\n        n_models = len(predictions)\n        bounds = [(0, 1) for _ in range(n_models)]\n        \n        result = differential_evolution(\n            self.objective,\n            bounds,\n            args=(predictions, y_true),\n            maxiter=100,\n            popsize=15,\n            seed=42\n        )\n        \n        self.weights = result.x / result.x.sum()\n        return self\n    \n    def predict(self, predictions):\n        return sum(w * p for w, p in zip(self.weights, predictions))\n\ndef main_pipeline():\n    \"\"\"Main execution pipeline with maximum diversity\"\"\"\n    print(\"\\n\" + \"=\"*60)\n    print(\"STARTING PART 4: MAXIMUM DIVERSITY PIPELINE\")\n    print(\"=\"*60)\n    \n    # Load data\n    print(\"\\n[1/7] Loading Data...\")\n    train = pd.read_parquet(CFG.train_path)\n    test = pd.read_parquet(CFG.test_path)\n    submission = pd.read_csv(CFG.sample_sub_path)\n    \n    print(f\"Train shape: {train.shape}\")\n    print(f\"Test shape: {test.shape}\")\n    \n    # Select features\n    available_features = [col for col in CFG.all_selected_features if col in train.columns]\n    print(f\"\\nUsing {len(available_features)} features from specified list\")\n    \n    train = train[available_features + ['label']]\n    test = test[available_features]\n    \n    # Create market features\n    print(\"\\n[2/7] Creating Market Features...\")\n    train = create_market_features(train)\n    test = create_market_features(test)\n    \n    # Memory optimization\n    train = reduce_mem_usage(train)\n    test = reduce_mem_usage(test)\n    \n    # Get feature columns\n    feature_cols = [col for col in train.columns if col != 'label']\n    print(f\"Total features after engineering: {len(feature_cols)}\")\n    \n    # Prepare data\n    print(\"\\n[3/7] Preparing Data...\")\n    X = train[feature_cols]\n    y = train['label']\n    X_test = test[feature_cols]\n    \n    # Use recent data\n    train_size = int(CFG.use_recent_ratio * len(X))\n    X = X.iloc[-train_size:]\n    y = y.iloc[-train_size:]\n    print(f\"Using last {len(X):,} samples for training\")\n    \n    # Create multiple feature representations for diversity\n    print(\"\\n[4/7] Creating Feature Representations...\")\n    \n    # 1. Standard scaled features\n    scaler_standard = StandardScaler()\n    X_standard = scaler_standard.fit_transform(X)\n    X_test_standard = scaler_standard.transform(X_test)\n    \n    # 2. Robust scaled features\n    scaler_robust = RobustScaler()\n    X_robust = scaler_robust.fit_transform(X)\n    X_test_robust = scaler_robust.transform(X_test)\n    \n    # 3. Quantile transformed features\n    scaler_quantile = QuantileTransformer(n_quantiles=min(1000, len(X)), random_state=42)\n    X_quantile = scaler_quantile.fit_transform(X)\n    X_test_quantile = scaler_quantile.transform(X_test)\n    \n    # 4. PCA features\n    pca_dim = None\n    if CFG.use_pca:\n        pca = PCA(n_components=CFG.pca_components, random_state=42)\n        X_pca = pca.fit_transform(X_standard)\n        X_test_pca = pca.transform(X_test_standard)\n        pca_dim = CFG.pca_components\n        print(f\"PCA explained variance: {pca.explained_variance_ratio_.sum():.3f}\")\n    \n    # 5. Feature selected subset\n    if CFG.use_feature_selection:\n        selector = SelectKBest(f_regression, k=min(CFG.top_k_features, len(feature_cols)))\n        X_selected = selector.fit_transform(X_standard, y)\n        X_test_selected = selector.transform(X_test_standard)\n        selected_features = [feature_cols[i] for i in selector.get_support(indices=True)]\n        print(f\"Selected {len(selected_features)} features\")\n    \n    # Convert to DataFrames\n    X_standard = pd.DataFrame(X_standard, columns=feature_cols, index=X.index)\n    X_test_standard = pd.DataFrame(X_test_standard, columns=feature_cols)\n    \n    X_robust = pd.DataFrame(X_robust, columns=feature_cols, index=X.index)\n    X_test_robust = pd.DataFrame(X_test_robust, columns=feature_cols)\n    \n    X_quantile = pd.DataFrame(X_quantile, columns=feature_cols, index=X.index)\n    X_test_quantile = pd.DataFrame(X_test_quantile, columns=feature_cols)\n    \n    if CFG.use_pca:\n        pca_cols = [f'pca_{i}' for i in range(CFG.pca_components)]\n        X_pca = pd.DataFrame(X_pca, columns=pca_cols, index=X.index)\n        X_test_pca = pd.DataFrame(X_test_pca, columns=pca_cols)\n    \n    if CFG.use_feature_selection:\n        X_selected = pd.DataFrame(X_selected, columns=selected_features, index=X.index)\n        X_test_selected = pd.DataFrame(X_test_selected, columns=selected_features)\n    \n    # Clear memory\n    del train, test\n    gc.collect()\n    \n    # Get models\n    print(\"\\n[5/7] Getting Diverse Models...\")\n    models = get_diverse_models(len(feature_cols), pca_dim)\n    \n    # Storage\n    all_predictions = {}\n    all_scores = {}\n    oof_predictions = {}\n    \n    # Different CV strategies for diversity\n    kf_standard = KFold(n_splits=CFG.n_folds, shuffle=True, random_state=CFG.random_state)\n    kf_shuffled = KFold(n_splits=CFG.n_folds, shuffle=True, random_state=CFG.random_state + 1)\n    tscv = TimeSeriesSplit(n_splits=CFG.n_folds)\n    \n    print(\"\\n[6/7] Training Diverse Models...\")\n    \n    # Model-specific feature and CV assignments\n    model_configs = {\n        # Standard scaling models\n        'lgb_goss': (X_standard, X_test_standard, kf_standard),\n        'xgb_dart': (X_standard, X_test_standard, kf_shuffled),\n        'cat_plain': (X_standard, X_test_standard, kf_standard),\n        \n        # Robust scaling for tree models\n        'rf': (X_robust, X_test_robust, kf_standard),\n        'extra_trees': (X_robust, X_test_robust, kf_shuffled),\n        \n        # Quantile transform for linear models\n        'lasso': (X_quantile, X_test_quantile, kf_standard),\n        'elastic': (X_quantile, X_test_quantile, kf_standard),\n        'huber': (X_robust, X_test_robust, kf_standard),\n        'bayesian_ridge': (X_standard, X_test_standard, kf_standard),\n        \n        # Non-parametric with standard scaling\n        'knn': (X_standard, X_test_standard, kf_standard),\n        'svr_rbf': (X_standard, X_test_standard, kf_standard),\n        'svr_poly': (X_quantile, X_test_quantile, kf_standard),\n        \n        # Neural networks with standard scaling\n        'resnet': (X_standard, X_test_standard, kf_standard),\n        'attention_nn': (X_standard, X_test_standard, kf_standard),\n        'gated_nn': (X_standard, X_test_standard, kf_standard),\n        'mlp': (X_standard, X_test_standard, kf_standard),\n        \n        # Simple models\n        'ridge_simple': (X_standard, X_test_standard, tscv),\n        'gbm_simple': (X_robust, X_test_robust, tscv),\n    }\n    \n    # Add PCA models if available\n    if CFG.use_pca:\n        model_configs['lgb_pca'] = (X_pca, X_test_pca, kf_standard)\n        model_configs['rf_pca'] = (X_pca, X_test_pca, kf_standard)\n    \n    # Add feature selection models\n    if CFG.use_feature_selection:\n        # Create new model instances for feature-selected data\n        models['lgb_selected'] = LGBMRegressor(\n            n_estimators=500, learning_rate=0.02, num_leaves=31,\n            max_depth=5, random_state=42, n_jobs=-1, verbose=-1\n        )\n        model_configs['lgb_selected'] = (X_selected, X_test_selected, kf_standard)\n    \n    # Train each model\n    for model_name, model in models.items():\n        if model_name not in model_configs:\n            continue\n            \n        print(f\"\\nTraining {model_name}...\")\n        start_time = time.time()\n        \n        X_train_full, X_test_model, cv_split = model_configs[model_name]\n        \n        fold_predictions = []\n        fold_scores = []\n        oof_preds = np.zeros(len(X_train_full))\n        \n        for fold, (train_idx, valid_idx) in enumerate(cv_split.split(X_train_full)):\n            X_train = X_train_full.iloc[train_idx]\n            y_train = y.iloc[train_idx]\n            X_valid = X_train_full.iloc[valid_idx]\n            y_valid = y.iloc[valid_idx]\n            \n            try:\n                if model_name in ['resnet', 'attention_nn', 'gated_nn']:\n                    # PyTorch neural networks\n                    input_dim = X_train.shape[1]\n                    \n                    if model_name == 'resnet':\n                        model_nn = ResNet(input_dim, hidden_dim=128, num_blocks=4)\n                    elif model_name == 'attention_nn':\n                        model_nn = AttentionNN(input_dim, hidden_dim=128, num_heads=4)\n                    else:  # gated_nn\n                        model_nn = GatedNN(input_dim, hidden_dims=[256, 128, 64])\n                    \n                    train_dataset = CryptoDataset(X_train, y_train)\n                    valid_dataset = CryptoDataset(X_valid, y_valid)\n                    \n                    train_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True)\n                    valid_loader = DataLoader(valid_dataset, batch_size=CFG.batch_size * 2, shuffle=False)\n                    \n                    # Different learning rates for different architectures\n                    lr = 0.001 if model_name != 'attention_nn' else 0.0005\n                    trained_model = train_neural_network(model_nn, train_loader, valid_loader, CFG.epochs, CFG.device, lr)\n                    \n                    # Predict\n                    trained_model.eval()\n                    with torch.no_grad():\n                        test_dataset = CryptoDataset(X_test_model)\n                        test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size * 4, shuffle=False)\n                        \n                        test_preds = []\n                        for batch_x in test_loader:\n                            batch_x = batch_x.to(CFG.device)\n                            pred = trained_model(batch_x)\n                            test_preds.extend(pred.cpu().numpy())\n                        test_pred = np.array(test_preds)\n                        \n                        valid_tensor = torch.FloatTensor(X_valid.values).to(CFG.device)\n                        valid_pred = trained_model(valid_tensor).cpu().numpy()\n                    \n                    del trained_model, model_nn\n                    if CFG.use_gpu:\n                        torch.cuda.empty_cache()\n                    \n                elif model_name in ['lgb_goss', 'lgb_pca', 'lgb_selected']:\n                    import lightgbm as lgb\n                    model_clone = copy.deepcopy(model)\n                    model_clone.fit(\n                        X_train, y_train,\n                        eval_set=[(X_valid, y_valid)],\n                        callbacks=[lgb.early_stopping(30), lgb.log_evaluation(0)]\n                    )\n                    test_pred = model_clone.predict(X_test_model)\n                    valid_pred = model_clone.predict(X_valid)\n                    del model_clone\n                    \n                elif model_name in ['xgb_dart']:\n                    model_clone = copy.deepcopy(model)\n                    model_clone.fit(\n                        X_train, y_train,\n                        eval_set=[(X_valid, y_valid)],\n                        early_stopping_rounds=30,\n                        verbose=False\n                    )\n                    test_pred = model_clone.predict(X_test_model)\n                    valid_pred = model_clone.predict(X_valid)\n                    del model_clone\n                    \n                elif model_name in ['cat_plain']:\n                    model_clone = copy.deepcopy(model)\n                    model_clone.fit(\n                        X_train, y_train,\n                        eval_set=[(X_valid, y_valid)],\n                        early_stopping_rounds=30,\n                        verbose=False\n                    )\n                    test_pred = model_clone.predict(X_test_model)\n                    valid_pred = model_clone.predict(X_valid)\n                    del model_clone\n                    \n                else:\n                    # Other sklearn models\n                    model_clone = copy.deepcopy(model)\n                    model_clone.fit(X_train, y_train)\n                    test_pred = model_clone.predict(X_test_model)\n                    valid_pred = model_clone.predict(X_valid)\n                    del model_clone\n                \n                # Score\n                score = pearsonr(y_valid, valid_pred)[0]\n                fold_scores.append(score)\n                fold_predictions.append(test_pred)\n                oof_preds[valid_idx] = valid_pred\n                \n                print(f\"  Fold {fold+1}: {score:.4f}\")\n                \n            except Exception as e:\n                print(f\"  Fold {fold+1}: Error - {str(e)}\")\n                continue\n            \n            # Cleanup\n            gc.collect()\n        \n        # Store results\n        if fold_predictions:\n            avg_score = np.mean(fold_scores)\n            all_predictions[model_name] = np.mean(fold_predictions, axis=0)\n            all_scores[model_name] = avg_score\n            oof_predictions[model_name] = oof_preds\n            \n            print(f\"  Average: {avg_score:.4f} (Time: {time.time()-start_time:.1f}s)\")\n    \n    # Create ensemble\n    print(\"\\n[7/7] Creating Diverse Ensemble...\")\n    \n    if all_predictions:\n        # Calculate diversity metrics\n        print(\"\\nModel Diversity Analysis:\")\n        model_names = list(all_predictions.keys())\n        n_models = len(model_names)\n        \n        # Correlation matrix of predictions\n        corr_matrix = np.zeros((n_models, n_models))\n        for i, name1 in enumerate(model_names):\n            for j, name2 in enumerate(model_names):\n                corr_matrix[i, j] = pearsonr(oof_predictions[name1], oof_predictions[name2])[0]\n        \n        # Average pairwise correlation\n        mask = np.triu(np.ones_like(corr_matrix), k=1).astype(bool)\n        avg_corr = corr_matrix[mask].mean()\n        print(f\"Average pairwise correlation: {avg_corr:.4f}\")\n        \n        # Find most diverse models\n        diversity_scores = {}\n        for i, name in enumerate(model_names):\n            # Average correlation with other models\n            other_corrs = [corr_matrix[i, j] for j in range(n_models) if i != j]\n            diversity_scores[name] = 1 - np.mean(other_corrs)\n        \n        print(\"\\nMost diverse models:\")\n        for name, div_score in sorted(diversity_scores.items(), key=lambda x: x[1], reverse=True)[:5]:\n            print(f\"  {name}: diversity={div_score:.3f}, score={all_scores[name]:.4f}\")\n        \n        # Optimized ensemble\n        ensemble = OptimizedEnsemble(model_names)\n        ensemble.fit([oof_predictions[name] for name in model_names], y.values)\n        ensemble_pred = ensemble.predict([all_predictions[name] for name in model_names])\n        \n        # Calculate ensemble score\n        ensemble_oof = ensemble.predict([oof_predictions[name] for name in model_names])\n        ensemble_score = pearsonr(y.values, ensemble_oof)[0]\n        \n        print(\"\\nOptimized ensemble weights:\")\n        for name, weight in sorted(zip(model_names, ensemble.weights), key=lambda x: x[1], reverse=True):\n            if weight > 0.01:\n                print(f\"  {name}: {weight:.3f} (score: {all_scores[name]:.4f})\")\n        \n        print(f\"\\nEnsemble OOF score: {ensemble_score:.4f}\")\n        \n        # Save predictions\n        submission['prediction'] = ensemble_pred\n        submission.to_csv('submission_part4_diverse.csv', index=False)\n        \n        print(\"\\n\" + \"=\"*60)\n        print(\"PART 4 PIPELINE COMPLETED SUCCESSFULLY!\")\n        print(\"=\"*60)\n        print(f\"Total models trained: {len(all_predictions)}\")\n        print(f\"Best single model: {max(all_scores.items(), key=lambda x: x[1])}\")\n        print(\"Submission saved to: submission_part4_diverse.csv\")\n    else:\n        print(\"\\nNo models trained successfully!\")\n\n# Execute\nif __name__ == \"__main__\":\n    main_pipeline()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}