{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!/usr/bin/env python\n# coding: utf-8\n\n# Install required packages\n!pip install -q lightgbm catboost xgboost\n!pip install -q scikit-learn scipy numpy pandas\n!pip install -q torch\n\nimport numpy as np\nimport pandas as pd\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Core imports\nimport gc\nimport os\nimport time\nfrom typing import Dict, List, Tuple\nimport copy\nimport random\n\n# ML imports\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, MinMaxScaler\nfrom sklearn.cluster import KMeans\nfrom sklearn.isotonic import IsotonicRegression\n\n# Models\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.linear_model import Ridge, Lasso, ElasticNet, QuantileRegressor, TweedieRegressor\nfrom sklearn.svm import NuSVR\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, HistGradientBoostingRegressor\n\nfrom scipy.stats import pearsonr\nfrom scipy.optimize import differential_evolution\n\n# Deep Learning\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\n# Set seeds\ndef set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\nset_seed(42)\n\nprint(\"=\" * 80)\nprint(\"DRW Crypto Market Prediction - Part 5: Unique Hybrid Models\")\nprint(\"=\" * 80)\nprint(f\"GPU Available: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n    print(f\"GPU Device: {torch.cuda.get_device_name(0)}\")\n\n# Configuration\nclass CFG:\n    # Data paths\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    # Model settings\n    n_folds = 5\n    random_state = 42\n    use_gpu = torch.cuda.is_available()\n    device = torch.device('cuda' if use_gpu else 'cpu')\n    \n    # Data settings\n    use_recent_ratio = 0.3  # Use last 30% of data\n    \n    # Neural network settings\n    batch_size = 2048\n    epochs = 30\n    early_stopping_patience = 10\n    \n    # Feature lists\n    original_features = [\n        'X363', 'X405', 'X321', 'X175', 'X179', 'X137', 'X197', 'X22', 'X40', 'X181',\n        'X28', 'X169', 'X198', 'X173', 'X338', 'X288', 'X385', 'X344', 'X427', 'X587',\n        'X450', 'X97', 'X52', 'X444', 'X598', 'X379', 'X696', 'X297', 'X138', 'X572',\n        'X343', 'X586', 'X466', 'X438', 'X452', 'X459', 'X435', 'X386', 'X55', 'X341',\n        'X683', 'X428', 'X605', 'X445', 'X272', 'X180', 'X593', 'X680', 'X686', 'X692',\n        'X695', \"X603\", \"X674\", \"X421\", \"X333\", \"X415\", \"X345\", \"X174\", \"X302\", \"X178\",\n        \"X168\", \"X612\", 'X298', 'X45', 'X46', 'X39', 'X752', 'X759', 'X41', 'X42',\n        \"buy_qty\", \"sell_qty\", \"volume\", \"bid_qty\", \"ask_qty\"\n    ]\n    \n    new_features = [\n        'X758', 'X296', 'X611', 'X780', 'X451', 'X25', 'X591'\n    ]\n    \n    all_selected_features = original_features + new_features\n\n# Memory optimization\ndef reduce_mem_usage(df, verbose=True):\n    \"\"\"Optimize memory usage\"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose:\n        print(f'Memory usage: {start_mem:.2f} MB -> {end_mem:.2f} MB ({100*(start_mem-end_mem)/start_mem:.1f}% reduction)')\n    return df\n\ndef create_market_features(df):\n    \"\"\"Create essential market microstructure features\"\"\"\n    eps = 1e-8\n    \n    # Core microstructure features\n    df['bid_ask_spread'] = df['ask_qty'] - df['bid_qty']\n    df['bid_ask_ratio'] = df['bid_qty'] / (df['ask_qty'] + eps)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['volume'] + eps)\n    df['trade_volume'] = df['buy_qty'] + df['sell_qty']\n    df['total_liquidity'] = df['bid_qty'] + df['ask_qty']\n    \n    # Key microstructure metrics\n    df['kyle_lambda'] = df['order_flow_imbalance'] / (np.sqrt(df['volume']) + eps)\n    df['amihud_illiquidity'] = np.abs(df['order_flow_imbalance']) / (np.log1p(df['volume']) + eps)\n    df['pin_proxy'] = np.abs(df['order_flow_imbalance']) / (df['trade_volume'] + eps)\n    \n    # Simple transformations\n    df['volume_log'] = np.log1p(df['volume'])\n    df['volume_sqrt'] = np.sqrt(df['volume'])\n    \n    # Key ratios\n    df['bid_total_ratio'] = df['bid_qty'] / (df['total_liquidity'] + eps)\n    df['buy_volume_ratio'] = df['buy_qty'] / (df['volume'] + eps)\n    \n    # Rolling features (limited windows to save memory)\n    for w in [5, 10, 20]:\n        df[f'volume_ma_{w}'] = df['volume'].rolling(w, min_periods=1).mean()\n        df[f'ofi_ma_{w}'] = df['order_flow_imbalance'].rolling(w, min_periods=1).mean()\n        df[f'spread_ma_{w}'] = df['bid_ask_spread'].rolling(w, min_periods=1).mean()\n    \n    # Lag features (limited)\n    for lag in [1, 3, 5]:\n        df[f'volume_lag_{lag}'] = df['volume'].shift(lag)\n        df[f'ofi_lag_{lag}'] = df['order_flow_imbalance'].shift(lag)\n    \n    # Handle NaN values\n    numeric_cols = df.select_dtypes(include=[np.number]).columns\n    df[numeric_cols] = df[numeric_cols].fillna(0)\n    \n    # Clip extreme values\n    for col in numeric_cols:\n        if col not in ['label', 'timestamp']:\n            q99 = df[col].quantile(0.99)\n            q01 = df[col].quantile(0.01)\n            df[col] = df[col].clip(q01, q99)\n    \n    return df\n\n# ============= HYBRID MODEL ARCHITECTURES =============\n\nclass HybridBoostingModel:\n    \"\"\"Combines multiple boosting algorithms in a hybrid approach\"\"\"\n    def __init__(self, n_estimators=500, random_state=42):\n        self.models = {\n            'lgb': LGBMRegressor(\n                n_estimators=n_estimators // 3, learning_rate=0.02,\n                num_leaves=31, max_depth=5, random_state=random_state,\n                n_jobs=-1, verbose=-1\n            ),\n            'xgb': XGBRegressor(\n                n_estimators=n_estimators // 3, learning_rate=0.02,\n                max_depth=5, random_state=random_state, verbosity=0\n            ),\n            'cat': CatBoostRegressor(\n                iterations=n_estimators // 3, learning_rate=0.02,\n                depth=5, random_state=random_state, verbose=False,\n                task_type='GPU' if CFG.use_gpu else 'CPU'\n            )\n        }\n        self.weights = None\n        \n    def fit(self, X, y, X_valid=None, y_valid=None):\n        predictions = []\n        \n        for name, model in self.models.items():\n            if name == 'lgb' and X_valid is not None:\n                import lightgbm as lgb\n                model.fit(X, y, eval_set=[(X_valid, y_valid)], \n                         callbacks=[lgb.early_stopping(30), lgb.log_evaluation(0)])\n            elif name in ['xgb', 'cat'] and X_valid is not None:\n                model.fit(X, y, eval_set=[(X_valid, y_valid)], \n                         early_stopping_rounds=30, verbose=False)\n            else:\n                model.fit(X, y)\n            \n            if X_valid is not None:\n                pred = model.predict(X_valid)\n                predictions.append(pred)\n        \n        # Optimize weights on validation set\n        if X_valid is not None and y_valid is not None:\n            self.weights = self._optimize_weights(predictions, y_valid)\n        else:\n            self.weights = np.array([1/3, 1/3, 1/3])\n        \n        return self\n    \n    def _optimize_weights(self, predictions, y_true):\n        def objective(w):\n            w = w / w.sum()\n            pred = sum(w[i] * p for i, p in enumerate(predictions))\n            return -pearsonr(y_true, pred)[0]\n        \n        result = differential_evolution(objective, [(0, 1)] * 3, maxiter=50, seed=42)\n        return result.x / result.x.sum()\n    \n    def predict(self, X):\n        predictions = [model.predict(X) for model in self.models.values()]\n        return sum(w * p for w, p in zip(self.weights, predictions))\n\nclass StackedTreeModel:\n    \"\"\"Stacks different tree-based models with meta-learner\"\"\"\n    def __init__(self, random_state=42):\n        self.base_models = {\n            'rf': RandomForestRegressor(\n                n_estimators=100, max_depth=10, min_samples_split=20,\n                random_state=random_state, n_jobs=-1\n            ),\n            'et': ExtraTreesRegressor(\n                n_estimators=100, max_depth=10, min_samples_split=20,\n                random_state=random_state, n_jobs=-1\n            ),\n            'hgb': HistGradientBoostingRegressor(\n                max_iter=100, max_depth=10, random_state=random_state\n            )\n        }\n        self.meta_model = Ridge(alpha=1.0, random_state=random_state)\n        self.base_predictions = None\n        \n    def fit(self, X, y):\n        # Get out-of-fold predictions for meta model\n        n_splits = 3\n        kf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n        \n        meta_features = np.zeros((len(X), len(self.base_models)))\n        \n        for train_idx, val_idx in kf.split(X):\n            X_train_fold = X.iloc[train_idx] if hasattr(X, 'iloc') else X[train_idx]\n            y_train_fold = y.iloc[train_idx] if hasattr(y, 'iloc') else y[train_idx]\n            X_val_fold = X.iloc[val_idx] if hasattr(X, 'iloc') else X[val_idx]\n            \n            for i, (name, model) in enumerate(self.base_models.items()):\n                model_clone = copy.deepcopy(model)\n                model_clone.fit(X_train_fold, y_train_fold)\n                meta_features[val_idx, i] = model_clone.predict(X_val_fold)\n        \n        # Train base models on full data\n        for model in self.base_models.values():\n            model.fit(X, y)\n        \n        # Train meta model\n        self.meta_model.fit(meta_features, y)\n        \n        return self\n    \n    def predict(self, X):\n        meta_features = np.column_stack([\n            model.predict(X) for model in self.base_models.values()\n        ])\n        return self.meta_model.predict(meta_features)\n\nclass ClusterBasedModel:\n    \"\"\"Different models for different data clusters\"\"\"\n    def __init__(self, n_clusters=5, random_state=42):\n        self.n_clusters = n_clusters\n        self.kmeans = KMeans(n_clusters=n_clusters, random_state=random_state, n_init=10)\n        self.cluster_models = {}\n        self.scaler = StandardScaler()\n        \n    def fit(self, X, y):\n        # Scale features for clustering\n        X_scaled = self.scaler.fit_transform(X)\n        \n        # Fit clusters\n        clusters = self.kmeans.fit_predict(X_scaled)\n        \n        # Train different model for each cluster\n        for i in range(self.n_clusters):\n            mask = clusters == i\n            if mask.sum() < 10:  # Skip small clusters\n                continue\n                \n            X_cluster = X[mask] if isinstance(X, np.ndarray) else X.iloc[mask]\n            y_cluster = y[mask] if isinstance(y, np.ndarray) else y.iloc[mask]\n            \n            # Use different model types for different clusters\n            if i % 3 == 0:\n                model = LGBMRegressor(n_estimators=200, num_leaves=31, random_state=42, verbose=-1)\n            elif i % 3 == 1:\n                model = RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1)\n            else:\n                model = Ridge(alpha=1.0, random_state=42)\n            \n            model.fit(X_cluster, y_cluster)\n            self.cluster_models[i] = model\n        \n        # Fallback model for uncovered clusters\n        self.fallback_model = Ridge(alpha=1.0, random_state=42)\n        self.fallback_model.fit(X, y)\n        \n        return self\n    \n    def predict(self, X):\n        X_scaled = self.scaler.transform(X)\n        clusters = self.kmeans.predict(X_scaled)\n        \n        predictions = np.zeros(len(X))\n        \n        for i in range(self.n_clusters):\n            mask = clusters == i\n            if mask.sum() == 0:\n                continue\n                \n            if i in self.cluster_models:\n                X_cluster = X[mask] if isinstance(X, np.ndarray) else X.iloc[mask]\n                predictions[mask] = self.cluster_models[i].predict(X_cluster)\n            else:\n                X_cluster = X[mask] if isinstance(X, np.ndarray) else X.iloc[mask]\n                predictions[mask] = self.fallback_model.predict(X_cluster)\n        \n        return predictions\n\n# ============= NEURAL NETWORK HYBRIDS =============\n\nclass WaveNet(nn.Module):\n    \"\"\"1D WaveNet-style architecture for tabular data\"\"\"\n    def __init__(self, input_dim, hidden_dim=64, num_layers=4, kernel_size=3):\n        super().__init__()\n        \n        self.input_conv = nn.Conv1d(input_dim, hidden_dim, kernel_size=1)\n        \n        self.dilated_convs = nn.ModuleList()\n        self.residual_convs = nn.ModuleList()\n        self.skip_convs = nn.ModuleList()\n        \n        for i in range(num_layers):\n            dilation = 2 ** i\n            self.dilated_convs.append(\n                nn.Conv1d(hidden_dim, hidden_dim * 2, kernel_size, \n                         dilation=dilation, padding=dilation)\n            )\n            self.residual_convs.append(nn.Conv1d(hidden_dim, hidden_dim, 1))\n            self.skip_convs.append(nn.Conv1d(hidden_dim, hidden_dim, 1))\n        \n        self.output_conv1 = nn.Conv1d(hidden_dim, hidden_dim, 1)\n        self.output_conv2 = nn.Conv1d(hidden_dim, 1, 1)\n        \n    def forward(self, x):\n        # Reshape input to (batch, features, 1)\n        x = x.unsqueeze(-1)\n        \n        x = self.input_conv(x)\n        skip_connections = []\n        \n        for dilated, residual, skip in zip(self.dilated_convs, self.residual_convs, self.skip_convs):\n            # Dilated convolution\n            h = dilated(x)\n            \n            # Gated activation\n            h_tanh, h_sigmoid = h.chunk(2, dim=1)\n            h = torch.tanh(h_tanh) * torch.sigmoid(h_sigmoid)\n            \n            # Residual and skip connections\n            res = residual(h)\n            skip_connections.append(skip(h))\n            x = x + res\n        \n        # Sum skip connections\n        out = sum(skip_connections)\n        out = F.relu(out)\n        out = self.output_conv1(out)\n        out = F.relu(out)\n        out = self.output_conv2(out)\n        \n        return out.squeeze(-1).squeeze(-1)\n\nclass MixtureOfExperts(nn.Module):\n    \"\"\"Mixture of Experts model\"\"\"\n    def __init__(self, input_dim, num_experts=4, hidden_dim=128):\n        super().__init__()\n        \n        # Gating network\n        self.gate = nn.Sequential(\n            nn.Linear(input_dim, hidden_dim),\n            nn.ReLU(),\n            nn.Linear(hidden_dim, num_experts),\n            nn.Softmax(dim=1)\n        )\n        \n        # Expert networks\n        self.experts = nn.ModuleList([\n            nn.Sequential(\n                nn.Linear(input_dim, hidden_dim),\n                nn.BatchNorm1d(hidden_dim),\n                nn.ReLU(),\n                nn.Dropout(0.2),\n                nn.Linear(hidden_dim, hidden_dim // 2),\n                nn.BatchNorm1d(hidden_dim // 2),\n                nn.ReLU(),\n                nn.Linear(hidden_dim // 2, 1)\n            ) for _ in range(num_experts)\n        ])\n        \n    def forward(self, x):\n        # Get gating weights\n        gate_weights = self.gate(x)\n        \n        # Get expert outputs\n        expert_outputs = []\n        for expert in self.experts:\n            expert_outputs.append(expert(x))\n        \n        # Stack expert outputs\n        expert_outputs = torch.stack(expert_outputs, dim=1)  # (batch, num_experts, 1)\n        \n        # Weighted sum\n        output = (gate_weights.unsqueeze(-1) * expert_outputs).sum(dim=1)\n        \n        return output.squeeze(-1)\n\nclass SimpleTransformer(nn.Module):\n    \"\"\"Simplified transformer for tabular data\"\"\"\n    def __init__(self, input_dim, d_model=128, nhead=4, num_layers=2):\n        super().__init__()\n        \n        # Feature embedding\n        self.feature_embedding = nn.Linear(input_dim, d_model)\n        \n        # Positional encoding for features\n        self.pos_encoding = nn.Parameter(torch.randn(1, 1, d_model) * 0.02)\n        \n        # Transformer encoder\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=d_model, nhead=nhead, dim_feedforward=d_model*4,\n            dropout=0.1, activation='relu', batch_first=True\n        )\n        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)\n        \n        # Output head\n        self.output = nn.Sequential(\n            nn.LayerNorm(d_model),\n            nn.Linear(d_model, d_model // 2),\n            nn.ReLU(),\n            nn.Dropout(0.1),\n            nn.Linear(d_model // 2, 1)\n        )\n        \n    def forward(self, x):\n        # Embed features\n        x = self.feature_embedding(x).unsqueeze(1)  # (batch, 1, d_model)\n        \n        # Add positional encoding\n        x = x + self.pos_encoding\n        \n        # Transformer\n        x = self.transformer(x)\n        \n        # Output\n        x = x.squeeze(1)\n        return self.output(x).squeeze(-1)\n\nclass CryptoDataset(Dataset):\n    def __init__(self, X, y=None):\n        self.X = torch.FloatTensor(X.values if hasattr(X, 'values') else X)\n        self.y = torch.FloatTensor(y.values if hasattr(y, 'values') else y) if y is not None else None\n        \n    def __len__(self):\n        return len(self.X)\n    \n    def __getitem__(self, idx):\n        if self.y is not None:\n            return self.X[idx], self.y[idx]\n        return self.X[idx]\n\n# Custom loss for Pearson correlation optimization\nclass PearsonCorrelationLoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n    def forward(self, pred, target):\n        # Center predictions and targets\n        pred_centered = pred - pred.mean()\n        target_centered = target - target.mean()\n        \n        # Calculate correlation\n        num = (pred_centered * target_centered).sum()\n        denom = torch.sqrt((pred_centered ** 2).sum() * (target_centered ** 2).sum())\n        \n        # Return negative correlation (we want to maximize correlation)\n        return -num / (denom + 1e-8)\n\ndef train_neural_network(model, train_loader, valid_loader, epochs, device, lr=0.001):\n    \"\"\"Train neural network with early stopping\"\"\"\n    model = model.to(device)\n    \n    optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-5)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5)\n    \n    # Use combination of MSE and Pearson correlation loss\n    mse_loss = nn.MSELoss()\n    corr_loss = PearsonCorrelationLoss()\n    \n    best_valid_loss = float('inf')\n    patience_counter = 0\n    best_model_state = None\n    \n    for epoch in range(epochs):\n        # Training\n        model.train()\n        train_loss = 0\n        \n        for batch_x, batch_y in train_loader:\n            batch_x, batch_y = batch_x.to(device), batch_y.to(device)\n            \n            optimizer.zero_grad()\n            outputs = model(batch_x)\n            \n            # Combined loss: 80% MSE + 20% correlation\n            loss = 0.8 * mse_loss(outputs, batch_y) + 0.2 * corr_loss(outputs, batch_y)\n            \n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            optimizer.step()\n            \n            train_loss += loss.item()\n        \n        # Validation\n        model.eval()\n        valid_loss = 0\n        \n        with torch.no_grad():\n            for batch_x, batch_y in valid_loader:\n                batch_x, batch_y = batch_x.to(device), batch_y.to(device)\n                outputs = model(batch_x)\n                loss = mse_loss(outputs, batch_y)\n                valid_loss += loss.item()\n        \n        avg_valid_loss = valid_loss / len(valid_loader)\n        scheduler.step(avg_valid_loss)\n        \n        # Early stopping\n        if avg_valid_loss < best_valid_loss:\n            best_valid_loss = avg_valid_loss\n            best_model_state = copy.deepcopy(model.state_dict())\n            patience_counter = 0\n        else:\n            patience_counter += 1\n        \n        if patience_counter >= CFG.early_stopping_patience:\n            break\n    \n    if best_model_state is not None:\n        model.load_state_dict(best_model_state)\n    \n    return model\n\ndef get_hybrid_models(input_dim):\n    \"\"\"Get unique hybrid models\"\"\"\n    models = {\n        # Hybrid ensemble models\n        'hybrid_boost': HybridBoostingModel(n_estimators=600, random_state=42),\n        'stacked_trees': StackedTreeModel(random_state=42),\n        'cluster_model': ClusterBasedModel(n_clusters=5, random_state=42),\n        \n        # Advanced gradient boosting configurations\n        'lgb_rf': LGBMRegressor(\n            n_estimators=500, learning_rate=0.02, num_leaves=63,\n            max_depth=7, min_child_samples=15, subsample=0.9,\n            colsample_bytree=0.9, reg_alpha=1, reg_lambda=1,\n            boosting_type='rf', subsample_freq=1, bagging_freq=1, bagging_fraction=0.9,\n            random_state=42, n_jobs=-1, verbose=-1\n        ),\n        \n        'xgb_linear': XGBRegressor(\n            n_estimators=500, learning_rate=0.02,\n            booster='gblinear', reg_alpha=1, reg_lambda=1,\n            random_state=42, verbosity=0\n        ),\n        \n        # Quantile and robust regressors\n        'quantile_50': QuantileRegressor(quantile=0.5, alpha=1.0, solver='highs'),\n        'quantile_75': QuantileRegressor(quantile=0.75, alpha=1.0, solver='highs'),\n        'tweedie': TweedieRegressor(power=1.5, alpha=1.0, max_iter=1000),\n        \n        # SVM variant\n        'nu_svr': NuSVR(nu=0.5, C=10.0, kernel='rbf', gamma='scale'),\n        \n        # Neural network hybrids\n        'wavenet': WaveNet(input_dim, hidden_dim=64, num_layers=4),\n        'moe': MixtureOfExperts(input_dim, num_experts=4, hidden_dim=128),\n        'transformer': SimpleTransformer(input_dim, d_model=128, nhead=4, num_layers=2),\n        \n        # Feature interaction models\n        'interaction_lgb': LGBMRegressor(\n            n_estimators=400, learning_rate=0.02, num_leaves=127,\n            max_depth=8, min_child_samples=10,\n            feature_fraction=0.7, feature_fraction_bynode=0.7,\n            random_state=42, n_jobs=-1, verbose=-1\n        ),\n        \n        # Isotonic regression wrapper\n        'iso_rf': RandomForestRegressor(\n            n_estimators=150, max_depth=12, min_samples_split=15,\n            random_state=42, n_jobs=-1\n        ),\n    }\n    \n    return models\n\nclass IsotonicWrapper:\n    \"\"\"Wrapper to apply isotonic regression calibration\"\"\"\n    def __init__(self, base_model):\n        self.base_model = base_model\n        self.isotonic = IsotonicRegression(out_of_bounds='clip')\n        \n    def fit(self, X, y, X_valid=None, y_valid=None):\n        # Fit base model\n        self.base_model.fit(X, y)\n        \n        # Fit isotonic on validation predictions\n        if X_valid is not None:\n            pred_valid = self.base_model.predict(X_valid)\n            self.isotonic.fit(pred_valid, y_valid)\n        else:\n            pred_train = self.base_model.predict(X)\n            self.isotonic.fit(pred_train, y)\n        \n        return self\n    \n    def predict(self, X):\n        base_pred = self.base_model.predict(X)\n        return self.isotonic.predict(base_pred)\n\nclass OptimizedEnsemble:\n    \"\"\"Optimized ensemble\"\"\"\n    def __init__(self, model_names):\n        self.model_names = model_names\n        self.weights = None\n        \n    def objective(self, weights, predictions, y_true):\n        weights = np.abs(weights) / np.abs(weights).sum()\n        ensemble_pred = sum(w * p for w, p in zip(weights, predictions))\n        return -pearsonr(y_true, ensemble_pred)[0]\n    \n    def fit(self, predictions, y_true):\n        n_models = len(predictions)\n        bounds = [(0, 1) for _ in range(n_models)]\n        \n        result = differential_evolution(\n            self.objective,\n            bounds,\n            args=(predictions, y_true),\n            maxiter=100,\n            popsize=15,\n            seed=42\n        )\n        \n        self.weights = result.x / result.x.sum()\n        return self\n    \n    def predict(self, predictions):\n        return sum(w * p for w, p in zip(self.weights, predictions))\n\ndef main_pipeline():\n    \"\"\"Main execution pipeline\"\"\"\n    print(\"\\n\" + \"=\"*60)\n    print(\"STARTING PART 5: UNIQUE HYBRID MODELS PIPELINE\")\n    print(\"=\"*60)\n    \n    # Load data\n    print(\"\\n[1/6] Loading Data...\")\n    train = pd.read_parquet(CFG.train_path)\n    test = pd.read_parquet(CFG.test_path)\n    submission = pd.read_csv(CFG.sample_sub_path)\n    \n    print(f\"Train shape: {train.shape}\")\n    print(f\"Test shape: {test.shape}\")\n    \n    # Select features\n    available_features = [col for col in CFG.all_selected_features if col in train.columns]\n    print(f\"\\nUsing {len(available_features)} features from specified list\")\n    \n    train = train[available_features + ['label']]\n    test = test[available_features]\n    \n    # Create market features\n    print(\"\\n[2/6] Creating Market Features...\")\n    train = create_market_features(train)\n    test = create_market_features(test)\n    \n    # Memory optimization\n    train = reduce_mem_usage(train)\n    test = reduce_mem_usage(test)\n    \n    # Get feature columns\n    feature_cols = [col for col in train.columns if col != 'label']\n    print(f\"Total features after engineering: {len(feature_cols)}\")\n    \n    # Prepare data\n    print(\"\\n[3/6] Preparing Data...\")\n    X = train[feature_cols]\n    y = train['label']\n    X_test = test[feature_cols]\n    \n    # Use recent data\n    train_size = int(CFG.use_recent_ratio * len(X))\n    X = X.iloc[-train_size:]\n    y = y.iloc[-train_size:]\n    print(f\"Using last {len(X):,} samples for training\")\n    \n    # Create different scalings\n    print(\"\\n[4/6] Creating Feature Transformations...\")\n    \n    # Standard scaling\n    scaler_standard = StandardScaler()\n    X_standard = scaler_standard.fit_transform(X)\n    X_test_standard = scaler_standard.transform(X_test)\n    \n    # MinMax scaling\n    scaler_minmax = MinMaxScaler()\n    X_minmax = scaler_minmax.fit_transform(X)\n    X_test_minmax = scaler_minmax.transform(X_test)\n    \n    # Convert to DataFrames\n    X_standard = pd.DataFrame(X_standard, columns=feature_cols, index=X.index)\n    X_test_standard = pd.DataFrame(X_test_standard, columns=feature_cols)\n    \n    X_minmax = pd.DataFrame(X_minmax, columns=feature_cols, index=X.index)\n    X_test_minmax = pd.DataFrame(X_test_minmax, columns=feature_cols)\n    \n    # Clear memory\n    del train, test\n    gc.collect()\n    \n    # Get models\n    print(\"\\n[5/6] Training Hybrid Models...\")\n    models = get_hybrid_models(len(feature_cols))\n    \n    # Add isotonic wrapper to RF\n    models['iso_rf'] = IsotonicWrapper(models['iso_rf'])\n    \n    # Storage\n    all_predictions = {}\n    all_scores = {}\n    oof_predictions = {}\n    \n    # Cross-validation\n    kf = KFold(n_splits=CFG.n_folds, shuffle=True, random_state=CFG.random_state)\n    \n    # Model-specific configurations\n    model_configs = {\n        # Hybrid models use standard scaling\n        'hybrid_boost': X_standard,\n        'stacked_trees': X_standard,\n        'cluster_model': X_standard,\n        \n        # Boosting variants\n        'lgb_rf': X_standard,\n        'xgb_linear': X_standard,\n        'interaction_lgb': X_standard,\n        \n        # Robust models use different scaling\n        'quantile_50': X_minmax,\n        'quantile_75': X_minmax,\n        'tweedie': X_minmax,\n        'nu_svr': X_standard,\n        \n        # Neural networks\n        'wavenet': X_standard,\n        'moe': X_standard,\n        'transformer': X_standard,\n        \n        # Isotonic\n        'iso_rf': X_standard,\n    }\n    \n    # Test configurations\n    test_configs = {\n        'hybrid_boost': X_test_standard,\n        'stacked_trees': X_test_standard,\n        'cluster_model': X_test_standard,\n        'lgb_rf': X_test_standard,\n        'xgb_linear': X_test_standard,\n        'interaction_lgb': X_test_standard,\n        'quantile_50': X_test_minmax,\n        'quantile_75': X_test_minmax,\n        'tweedie': X_test_minmax,\n        'nu_svr': X_test_standard,\n        'wavenet': X_test_standard,\n        'moe': X_test_standard,\n        'transformer': X_test_standard,\n        'iso_rf': X_test_standard,\n    }\n    \n    # Train each model\n    for model_name, model in models.items():\n        if model_name not in model_configs:\n            continue\n            \n        print(f\"\\nTraining {model_name}...\")\n        start_time = time.time()\n        \n        X_train_full = model_configs[model_name]\n        X_test_model = test_configs[model_name]\n        \n        fold_predictions = []\n        fold_scores = []\n        oof_preds = np.zeros(len(X_train_full))\n        \n        for fold, (train_idx, valid_idx) in enumerate(kf.split(X_train_full)):\n            X_train = X_train_full.iloc[train_idx]\n            y_train = y.iloc[train_idx]\n            X_valid = X_train_full.iloc[valid_idx]\n            y_valid = y.iloc[valid_idx]\n            \n            try:\n                if model_name in ['wavenet', 'moe', 'transformer']:\n                    # Neural networks\n                    input_dim = X_train.shape[1]\n                    \n                    if model_name == 'wavenet':\n                        model_nn = WaveNet(input_dim, hidden_dim=64, num_layers=4)\n                    elif model_name == 'moe':\n                        model_nn = MixtureOfExperts(input_dim, num_experts=4, hidden_dim=128)\n                    else:  # transformer\n                        model_nn = SimpleTransformer(input_dim, d_model=128, nhead=4, num_layers=2)\n                    \n                    train_dataset = CryptoDataset(X_train, y_train)\n                    valid_dataset = CryptoDataset(X_valid, y_valid)\n                    \n                    train_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True)\n                    valid_loader = DataLoader(valid_dataset, batch_size=CFG.batch_size * 2, shuffle=False)\n                    \n                    trained_model = train_neural_network(model_nn, train_loader, valid_loader, \n                                                       CFG.epochs, CFG.device)\n                    \n                    # Predict\n                    trained_model.eval()\n                    with torch.no_grad():\n                        test_dataset = CryptoDataset(X_test_model)\n                        test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size * 4, shuffle=False)\n                        \n                        test_preds = []\n                        for batch_x in test_loader:\n                            batch_x = batch_x.to(CFG.device)\n                            pred = trained_model(batch_x)\n                            test_preds.extend(pred.cpu().numpy())\n                        test_pred = np.array(test_preds)\n                        \n                        valid_tensor = torch.FloatTensor(X_valid.values).to(CFG.device)\n                        valid_pred = trained_model(valid_tensor).cpu().numpy()\n                    \n                    del trained_model, model_nn\n                    if CFG.use_gpu:\n                        torch.cuda.empty_cache()\n                \n                elif model_name in ['hybrid_boost', 'iso_rf']:\n                    # Models that benefit from validation set\n                    model_clone = copy.deepcopy(model)\n                    model_clone.fit(X_train, y_train, X_valid, y_valid)\n                    test_pred = model_clone.predict(X_test_model)\n                    valid_pred = model_clone.predict(X_valid)\n                    del model_clone\n                    \n                else:\n                    # Other sklearn models\n                    model_clone = copy.deepcopy(model)\n                    model_clone.fit(X_train, y_train)\n                    test_pred = model_clone.predict(X_test_model)\n                    valid_pred = model_clone.predict(X_valid)\n                    del model_clone\n                \n                # Score\n                score = pearsonr(y_valid, valid_pred)[0]\n                fold_scores.append(score)\n                fold_predictions.append(test_pred)\n                oof_preds[valid_idx] = valid_pred\n                \n                print(f\"  Fold {fold+1}: {score:.4f}\")\n                \n            except Exception as e:\n                print(f\"  Fold {fold+1}: Error - {str(e)}\")\n                continue\n            \n            # Cleanup\n            gc.collect()\n        \n        # Store results\n        if fold_predictions:\n            avg_score = np.mean(fold_scores)\n            all_predictions[model_name] = np.mean(fold_predictions, axis=0)\n            all_scores[model_name] = avg_score\n            oof_predictions[model_name] = oof_preds\n            \n            print(f\"  Average: {avg_score:.4f} (Time: {time.time()-start_time:.1f}s)\")\n    \n    # Create ensemble\n    print(\"\\n[6/6] Creating Hybrid Ensemble...\")\n    \n    if all_predictions:\n        # Model performance summary\n        print(\"\\nModel Performance Summary:\")\n        for name, score in sorted(all_scores.items(), key=lambda x: x[1], reverse=True):\n            print(f\"  {name}: {score:.4f}\")\n        \n        # Optimized ensemble\n        ensemble = OptimizedEnsemble(list(all_predictions.keys()))\n        ensemble.fit([oof_predictions[name] for name in all_predictions.keys()], y.values)\n        ensemble_pred = ensemble.predict([all_predictions[name] for name in all_predictions.keys()])\n        \n        # Calculate ensemble score\n        ensemble_oof = ensemble.predict([oof_predictions[name] for name in all_predictions.keys()])\n        ensemble_score = pearsonr(y.values, ensemble_oof)[0]\n        \n        print(\"\\nOptimized ensemble weights:\")\n        for name, weight in sorted(zip(all_predictions.keys(), ensemble.weights), \n                                  key=lambda x: x[1], reverse=True):\n            if weight > 0.01:\n                print(f\"  {name}: {weight:.3f} (score: {all_scores[name]:.4f})\")\n        \n        print(f\"\\nEnsemble OOF score: {ensemble_score:.4f}\")\n        \n        # Save predictions\n        submission['prediction'] = ensemble_pred\n        submission.to_csv('submission_part5_hybrid.csv', index=False)\n        \n        print(\"\\n\" + \"=\"*60)\n        print(\"PART 5 PIPELINE COMPLETED SUCCESSFULLY!\")\n        print(\"=\"*60)\n        print(f\"Total models trained: {len(all_predictions)}\")\n        print(f\"Best single model: {max(all_scores.items(), key=lambda x: x[1])}\")\n        print(\"Submission saved to: submission_part5_hybrid.csv\")\n    else:\n        print(\"\\nNo models trained successfully!\")\n\n# Execute\nif __name__ == \"__main__\":\n    main_pipeline()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}