{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport pandas as pd\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import TimeSeriesSplit\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm import tqdm\nimport warnings\nfrom scipy.stats import pearsonr\nimport random\nimport os\nimport pickle\nfrom pathlib import Path\nwarnings.filterwarnings('ignore')\n\ndef set_random_seeds(seed):\n    \"\"\"Set all random seeds for reproducibility\"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    print(f\"🌱 Set all random seeds to {seed}\")\n\nclass Swish(nn.Module):\n    \"\"\"Swish activation function - prevents dead neurons and provides smooth gradients\"\"\"\n    def forward(self, x):\n        return x * torch.sigmoid(x)\n\nclass GaussianNoise(nn.Module):\n    \"\"\"Gaussian noise layer for data augmentation and overfitting prevention\"\"\"\n    def __init__(self, std=0.05):\n        super(GaussianNoise, self).__init__()\n        self.std = std\n        \n    def forward(self, x):\n        if self.training:\n            noise = torch.randn_like(x) * self.std\n            return x + noise\n        return x\n\nclass AutoEncoder(nn.Module):\n    \"\"\"AutoEncoder for feature learning with proper expanding encoder and compressing decoder\"\"\"\n    def __init__(self, input_size, encoding_size=128, dropout=0.7):\n        super(AutoEncoder, self).__init__()\n        \n        # Calculate intermediate dimensions for smooth expansion/compression\n        # For input_size=25, we want: 25 -> 64 -> 128 -> 128 (encoding)\n        hidden1_size = max(input_size * 2, 64)  # First expansion\n        hidden2_size = max(hidden1_size, encoding_size)  # Second expansion to at least encoding_size\n        \n        # Encoder pathway - EXPAND dimensions for richer feature representation\n        self.encoder = nn.Sequential(\n            # First expansion: input_size -> hidden1_size (25 -> 64)\n            nn.Linear(input_size, hidden1_size),\n            nn.BatchNorm1d(hidden1_size),\n            Swish(),\n            nn.Dropout(dropout),\n            \n            # Second expansion: hidden1_size -> hidden2_size (64 -> 128)\n            nn.Linear(hidden1_size, hidden2_size),\n            nn.BatchNorm1d(hidden2_size),\n            Swish(),\n            nn.Dropout(dropout),\n            \n            # Final encoding layer: hidden2_size -> encoding_size (128 -> 128)\n            nn.Linear(hidden2_size, encoding_size),\n            nn.BatchNorm1d(encoding_size),\n            Swish()\n        )\n        \n        # Decoder pathway - COMPRESS back to original dimensions\n        self.decoder = nn.Sequential(\n            # First decompression: encoding_size -> hidden2_size (128 -> 128)\n            nn.Linear(encoding_size, hidden2_size),\n            nn.BatchNorm1d(hidden2_size),\n            Swish(),\n            nn.Dropout(dropout),\n            \n            # Second decompression: hidden2_size -> hidden1_size (128 -> 64)\n            nn.Linear(hidden2_size, hidden1_size),\n            nn.BatchNorm1d(hidden1_size),\n            Swish(),\n            nn.Dropout(dropout),\n            \n            # Final reconstruction: hidden1_size -> input_size (64 -> 25)\n            nn.Linear(hidden1_size, input_size)\n        )\n        \n        # Store dimensions for reference\n        self.input_size = input_size\n        self.encoding_size = encoding_size\n        self.hidden1_size = hidden1_size\n        self.hidden2_size = hidden2_size\n        \n        print(f\"AutoEncoder Architecture: {input_size} -> {hidden1_size} -> {hidden2_size} -> {encoding_size} -> {hidden2_size} -> {hidden1_size} -> {input_size}\")\n        \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return encoded, decoded\n\nclass CryptoMLPWithAutoEncoder(nn.Module):\n    \"\"\"Enhanced MLP with AutoEncoder feature learning and Gaussian noise augmentation\"\"\"\n    def __init__(self, input_size, encoding_size=128, dropout=0.7, hidden_size=256, noise_std=0.05):\n        super(CryptoMLPWithAutoEncoder, self).__init__()\n        \n        # Gaussian noise layer for data augmentation\n        self.noise_layer = GaussianNoise(std=noise_std)\n        \n        # AutoEncoder for feature learning (now with proper expanding/compressing architecture)\n        self.autoencoder = AutoEncoder(input_size, encoding_size, dropout=0.7)\n        \n        # Combined input size: original features + encoded features\n        combined_input_size = input_size + encoding_size\n        self.input_bn = nn.BatchNorm1d(combined_input_size)\n        \n        # Block 0: 3 dense layers with hidden_size units\n        self.block0_layer1 = nn.Linear(combined_input_size, hidden_size)\n        self.block0_bn1 = nn.BatchNorm1d(hidden_size)\n        self.block0_layer2 = nn.Linear(hidden_size, hidden_size)\n        self.block0_bn2 = nn.BatchNorm1d(hidden_size)\n        self.block0_layer3 = nn.Linear(hidden_size, hidden_size)\n        self.block0_bn3 = nn.BatchNorm1d(hidden_size)\n        \n        # Blocks 1-8: 2 dense layers each with hidden_size units\n        self.blocks = nn.ModuleList()\n        self.blocks_bn1 = nn.ModuleList()\n        self.blocks_bn2 = nn.ModuleList()\n        \n        for i in range(8):\n            block = nn.ModuleDict({\n                'layer1': nn.Linear(hidden_size, hidden_size),\n                'layer2': nn.Linear(hidden_size, hidden_size)\n            })\n            self.blocks.append(block)\n            self.blocks_bn1.append(nn.BatchNorm1d(hidden_size))\n            self.blocks_bn2.append(nn.BatchNorm1d(hidden_size))\n        \n        # Dropout and activation\n        self.dropout = nn.Dropout(dropout)\n        self.swish = Swish()\n        \n        # Output layer\n        self.output = nn.Linear(hidden_size, 1)\n        \n        # Store sizes for reference\n        self.hidden_size = hidden_size\n        self.encoding_size = encoding_size\n        self.input_size = input_size\n        \n        print(f\"Main Network: {combined_input_size} features -> {hidden_size} hidden -> 1 output\")\n        \n    def forward(self, x, return_ae_loss=False):\n        # Apply Gaussian noise for data augmentation (only during training)\n        x_noisy = self.noise_layer(x)\n        \n        # Get autoencoder features\n        encoded, decoded = self.autoencoder(x_noisy)\n        \n        # Concatenate original features with encoded features\n        x_combined = torch.cat([x, encoded], dim=1)\n        \n        # Input block with BatchNorm\n        x_combined = self.input_bn(x_combined)\n        \n        # Block 0: 3 dense layers\n        x0 = self.block0_layer1(x_combined)\n        x0 = self.block0_bn1(x0)\n        x0 = self.swish(x0)\n        x0 = self.dropout(x0)\n        \n        x0 = self.block0_layer2(x0)\n        x0 = self.block0_bn2(x0)\n        x0 = self.swish(x0)\n        x0 = self.dropout(x0)\n        \n        x0 = self.block0_layer3(x0)\n        x0 = self.block0_bn3(x0)\n        x0 = self.swish(x0)\n        \n        # Blocks 1-8 with skip connections from Block 0\n        x_current = x0\n        for i, (block, bn1, bn2) in enumerate(zip(self.blocks, self.blocks_bn1, self.blocks_bn2)):\n            # First layer of block i+1\n            x_block = block['layer1'](x_current)\n            x_block = bn1(x_block)\n            x_block = self.swish(x_block)\n            x_block = self.dropout(x_block)\n            \n            # Second layer of block i+1\n            x_block = block['layer2'](x_block)\n            x_block = bn2(x_block)\n            x_block = self.swish(x_block)\n            \n            # Skip connection from Block 0\n            x_current = x_block + x0\n        \n        # Output layer\n        out = self.output(x_current)\n        \n        if return_ae_loss:\n            return out, decoded, x_noisy\n        return out\n\nclass CryptoDataset(Dataset):\n    def __init__(self, features, labels=None):\n        self.features = torch.FloatTensor(features)\n        self.labels = torch.FloatTensor(labels) if labels is not None else None\n        \n    def __len__(self):\n        return len(self.features)\n        \n    def __getitem__(self, idx):\n        if self.labels is not None:\n            return self.features[idx], self.labels[idx]\n        return self.features[idx]\n\ndef calculate_correlation(predictions, targets):\n    \"\"\"Calculate Pearson correlation coefficient\"\"\"\n    predictions = predictions.flatten()\n    targets = targets.flatten()\n    \n    # Remove any NaN or inf values\n    mask = np.isfinite(predictions) & np.isfinite(targets)\n    if mask.sum() < 2:\n        return 0.0\n    \n    predictions = predictions[mask]\n    targets = targets[mask]\n    \n    if np.std(predictions) == 0 or np.std(targets) == 0:\n        return 0.0\n    \n    try:\n        corr, _ = pearsonr(predictions, targets)\n        return corr if not np.isnan(corr) else 0.0\n    except:\n        return 0.0\n\nclass PearsonCorrelationLoss(nn.Module):\n    \"\"\"Pearson correlation coefficient loss function\"\"\"\n    def __init__(self, eps=1e-8):\n        super(PearsonCorrelationLoss, self).__init__()\n        self.eps = eps\n        \n    def forward(self, y_pred, y_true):\n        y_pred = y_pred.view(-1)\n        y_true = y_true.view(-1)\n        \n        y_pred_centered = y_pred - torch.mean(y_pred)\n        y_true_centered = y_true - torch.mean(y_true)\n        \n        numerator = torch.sum(y_pred_centered * y_true_centered)\n        \n        pred_std = torch.sqrt(torch.sum(y_pred_centered ** 2) + self.eps)\n        true_std = torch.sqrt(torch.sum(y_true_centered ** 2) + self.eps)\n        denominator = pred_std * true_std\n        \n        correlation = numerator / denominator\n        return -correlation\n\nclass CombinedLossWithAE(nn.Module):\n    \"\"\"Combined MSE, Correlation Loss, and AutoEncoder Reconstruction Loss\"\"\"\n    def __init__(self, mse_weight=0.25, corr_weight=0.6, ae_weight=0.15, eps=1e-8):\n        super(CombinedLossWithAE, self).__init__()\n        self.mse_weight = mse_weight\n        self.corr_weight = corr_weight\n        self.ae_weight = ae_weight\n        self.eps = eps\n        self.mse_loss = nn.MSELoss()\n        \n    def forward(self, y_pred, y_true, decoded=None, original=None):\n        # Primary prediction loss (MSE)\n        mse = self.mse_loss(y_pred, y_true)\n        \n        # Correlation loss\n        y_pred_flat = y_pred.view(-1)\n        y_true_flat = y_true.view(-1)\n        \n        y_pred_centered = y_pred_flat - torch.mean(y_pred_flat)\n        y_true_centered = y_true_flat - torch.mean(y_true_flat)\n        \n        numerator = torch.sum(y_pred_centered * y_true_centered)\n        pred_std = torch.sqrt(torch.sum(y_pred_centered ** 2) + self.eps)\n        true_std = torch.sqrt(torch.sum(y_true_centered ** 2) + self.eps)\n        denominator = pred_std * true_std\n        \n        correlation = numerator / denominator\n        \n        # AutoEncoder reconstruction loss\n        ae_loss = 0.0\n        if decoded is not None and original is not None:\n            ae_loss = self.mse_loss(decoded, original)\n        \n        # Combined loss\n        total_loss = (self.mse_weight * mse - \n                     self.corr_weight * correlation + \n                     self.ae_weight * ae_loss)\n        \n        return total_loss, mse.item(), correlation.item(), ae_loss.item() if isinstance(ae_loss, torch.Tensor) else ae_loss\n\ndef load_and_preprocess_data():\n    \"\"\"Load and preprocess the crypto data\"\"\"\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    \n    print(\"Loading data...\")\n    train_df = pd.read_parquet(train_path)\n    test_df = pd.read_parquet(test_path)\n    \n    print(f\"Train shape: {train_df.shape}\")\n    print(f\"Test shape: {test_df.shape}\")\n    \n    # Remove redundant columns and keep only the selected features\n    cols_to_keep = [\n        \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n        \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\", 'label'\n    ]\n    \n    # Ensure all columns exist in both datasets\n    available_cols_train = [col for col in cols_to_keep if col in train_df.columns]\n    available_cols_test = [col for col in cols_to_keep if col in test_df.columns and col != 'label']\n    \n    print(f\"Available features: {len(available_cols_train) - 1}\")  # -1 for label\n    \n    train_df = train_df[available_cols_train]\n    test_df = test_df[available_cols_test + ['label']]  # test has label column but it's all zeros\n    \n    # Prepare features and labels\n    feature_cols = [col for col in train_df.columns if col not in ['timestamp', 'label']]\n    \n    print(f\"Final feature count: {len(feature_cols)}\")\n    print(f\"Features: {feature_cols}\")\n    \n    # Sort by timestamp for time series split\n    if 'timestamp' in train_df.columns:\n        train_df = train_df.sort_values('timestamp')\n    train_df = train_df.reset_index(drop=True)\n    \n    X_full = train_df[feature_cols].values\n    y_full = train_df['label'].values\n    \n    # Handle missing values and infinities\n    X_full = np.nan_to_num(X_full, nan=0.0, posinf=0.0, neginf=0.0)\n    \n    # Prepare test data\n    X_test = test_df[feature_cols].values\n    X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n    \n    print(f\"Train features shape: {X_full.shape}\")\n    print(f\"Test features shape: {X_test.shape}\")\n    \n    return X_full, X_test, y_full, train_df, X_full.shape[1]\n\ndef train_single_fold(fold_idx, train_idx, val_idx, X_full, y_full, X_test, num_features, \n                     model_dir, num_epochs=80, lr=0.0001, encoding_size=128, seed=42):\n    \"\"\"Train a single enhanced model for one time series fold\"\"\"\n    \n    print(f\"\\n{'='*80}\")\n    print(f\"🏋️‍♂️ Training Enhanced Fold {fold_idx + 1} with Corrected AutoEncoder\")\n    print(f\"{'='*80}\")\n    print(f\"Train samples: {len(train_idx)}, Val samples: {len(val_idx)}\")\n    print(f\"Input features: {num_features}, Encoding size: {encoding_size}\")\n    \n    # Set seed for this fold\n    set_random_seeds(seed + fold_idx)  # Different seed per fold for diversity\n    \n    # Get train and validation data for this fold\n    X_train = X_full[train_idx]\n    X_val = X_full[val_idx]\n    y_train = y_full[train_idx]\n    y_val = y_full[val_idx]\n    \n    # Create datasets and dataloaders\n    train_dataset = CryptoDataset(X_train, y_train)\n    val_dataset = CryptoDataset(X_val, y_val)\n    \n    batch_size = 4096\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n    \n    # Initialize enhanced model with corrected AutoEncoder\n    model = CryptoMLPWithAutoEncoder(\n        input_size=num_features, \n        encoding_size=encoding_size,\n        dropout=0.7,\n        hidden_size=256,\n        noise_std=0.05\n    )\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model = model.to(device)\n    \n    # Initialize weights\n    def init_weights(m):\n        if isinstance(m, nn.Linear):\n            torch.nn.init.xavier_uniform_(m.weight)\n            m.bias.data.fill_(0.01)\n    \n    model.apply(init_weights)\n    \n    # Use combined loss with AutoEncoder\n    criterion = CombinedLossWithAE(mse_weight=0.25, corr_weight=0.6, ae_weight=0.15)\n    \n    optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=5e-3)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n        optimizer, mode='max', patience=5, factor=0.5, min_lr=1e-6\n    )\n    \n    best_val_corr = -float('inf')\n    best_model_state = None\n    best_epoch = 0\n    early_stop_counter = 0\n    patience = 10\n    \n    print(f\"Training on {device} | Parameters: {sum(p.numel() for p in model.parameters()):,}\")\n    \n    # Training loop\n    for epoch in range(num_epochs):\n        # Training phase\n        model.train()\n        train_losses = []\n        train_mse_losses = []\n        train_corr_losses = []\n        train_ae_losses = []\n        \n        for batch_features, batch_labels in train_loader:\n            batch_features = batch_features.to(device)\n            batch_labels = batch_labels.to(device)\n            \n            optimizer.zero_grad()\n            \n            # Forward pass with AutoEncoder loss\n            outputs, decoded, noisy_input = model(batch_features, return_ae_loss=True)\n            \n            # Calculate combined loss\n            loss, mse_loss, corr_loss, ae_loss = criterion(\n                outputs, batch_labels, decoded, noisy_input\n            )\n            \n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            \n            loss.backward()\n            optimizer.step()\n            \n            train_losses.append(loss.item())\n            train_mse_losses.append(mse_loss)\n            train_corr_losses.append(corr_loss)\n            train_ae_losses.append(ae_loss)\n        \n        # Validation phase\n        model.eval()\n        val_losses = []\n        val_predictions = []\n        val_targets = []\n        val_mse_losses = []\n        val_corr_losses = []\n        val_ae_losses = []\n        \n        with torch.no_grad():\n            for batch_features, batch_labels in val_loader:\n                batch_features = batch_features.to(device)\n                batch_labels = batch_labels.to(device)\n                \n                outputs, decoded, noisy_input = model(batch_features, return_ae_loss=True)\n                loss, mse_loss, corr_loss, ae_loss = criterion(\n                    outputs, batch_labels, decoded, noisy_input\n                )\n                \n                val_losses.append(loss.item())\n                val_mse_losses.append(mse_loss)\n                val_corr_losses.append(corr_loss)\n                val_ae_losses.append(ae_loss)\n                val_predictions.extend(outputs.cpu().numpy().flatten())\n                val_targets.extend(batch_labels.cpu().numpy().flatten())\n        \n        # Calculate metrics\n        avg_train_loss = np.mean(train_losses)\n        avg_val_loss = np.mean(val_losses)\n        avg_train_mse = np.mean(train_mse_losses)\n        avg_val_mse = np.mean(val_mse_losses)\n        avg_train_ae = np.mean(train_ae_losses)\n        avg_val_ae = np.mean(val_ae_losses)\n        \n        val_pred_array = np.array(val_predictions)\n        val_target_array = np.array(val_targets)\n        val_corr = calculate_correlation(val_pred_array, val_target_array)\n        \n        scheduler.step(val_corr)\n        \n        # Save best model\n        improvement = \"\"\n        if val_corr > best_val_corr:\n            best_val_corr = val_corr\n            best_model_state = model.state_dict().copy()\n            best_epoch = epoch + 1\n            early_stop_counter = 0\n            improvement = \" ⭐\"\n        else:\n            early_stop_counter += 1\n        \n        # Print progress every 10 epochs with detailed loss breakdown\n        if (epoch + 1) % 10 == 0:\n            current_lr = optimizer.param_groups[0]['lr']\n            print(f\"Epoch {epoch+1:3d}/{num_epochs} | \"\n                  f\"Loss: {avg_val_loss:.6f} | \"\n                  f\"MSE: {avg_val_mse:.6f} | \"\n                  f\"AE: {avg_val_ae:.6f} | \"\n                  f\"Corr: {val_corr:.6f} | \"\n                  f\"Best: {best_val_corr:.6f} | \"\n                  f\"LR: {current_lr:.2e} | \"\n                  f\"ES: {early_stop_counter}/{patience}{improvement}\")\n        \n        # Early stopping\n        if early_stop_counter >= patience:\n            print(f\"🛑 Early stopping at epoch {epoch+1}\")\n            break\n    \n    # Load best model\n    model.load_state_dict(best_model_state)\n    \n    # Save model\n    model_path = model_dir / f\"enhanced_model_fold_{fold_idx}.pt\"\n    torch.save({\n        'model_state_dict': best_model_state,\n        'best_val_corr': best_val_corr,\n        'best_epoch': best_epoch,\n        'fold_idx': fold_idx,\n        'num_features': num_features,\n        'encoding_size': encoding_size\n    }, model_path)\n    \n    # Make predictions on test set\n    test_dataset = CryptoDataset(X_test)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n    \n    model.eval()\n    test_predictions = []\n    \n    with torch.no_grad():\n        for batch_features in test_loader:\n            batch_features = batch_features.to(device)\n            outputs = model(batch_features, return_ae_loss=False)\n            test_predictions.append(outputs.cpu().numpy())\n    \n    test_pred_array = np.vstack(test_predictions)[:, 0]\n    \n    print(f\"✅ Enhanced Fold {fold_idx + 1} completed | Best Correlation: {best_val_corr:.6f} (Epoch {best_epoch})\")\n    \n    return {\n        'fold_idx': fold_idx,\n        'model': model,\n        'best_val_corr': best_val_corr,\n        'best_epoch': best_epoch,\n        'test_predictions': test_pred_array,\n        'model_path': model_path\n    }\n\ndef enhanced_timeseries_ensemble_training(n_splits=5, max_train_size=100_000_000, gap=1, \n                                        num_epochs=80, lr=0.0001, encoding_size=128, seed=42):\n    \"\"\"Train enhanced ensemble with corrected AutoEncoder using TimeSeriesSplit\"\"\"\n    \n    print(f\"🚀 DRW Crypto Enhanced TimeSeriesSplit Ensemble Training Framework\")\n    print(f\"🔧 Enhancements: Corrected AutoEncoder + Gaussian Noise + Swish Activation\")\n    print(f\"N Splits: {n_splits}\")\n    print(f\"Max Train Size: {max_train_size:,}\")\n    print(f\"Gap: {gap}\")\n    print(f\"Epochs per Model: {num_epochs}\")\n    print(f\"Encoding Size: {encoding_size}\")\n    print(f\"Learning Rate: {lr}\")\n    print(\"=\" * 80)\n    \n    # Create model directory\n    model_dir = Path(\"/kaggle/working/enhanced_ensemble_models\")\n    model_dir.mkdir(exist_ok=True)\n    \n    # Load data once\n    print(\"🔄 Loading data...\")\n    X_full, X_test, y_full, train_df, num_features = load_and_preprocess_data()\n    \n    # Create TimeSeriesSplit\n    tss = TimeSeriesSplit(n_splits=n_splits, max_train_size=max_train_size, gap=gap)\n    all_splits = list(tss.split(train_df.index))\n    \n    print(f\"📊 TimeSeriesSplit created {len(all_splits)} folds\")\n    \n    # Store all results\n    all_results = []\n    all_test_predictions = []\n    \n    # Train each fold\n    for fold_idx, (train_idx, val_idx) in enumerate(all_splits):\n        print(f\"\\n🚀 Training Enhanced Fold {fold_idx + 1}/{len(all_splits)}\")\n        \n        result = train_single_fold(\n            fold_idx=fold_idx,\n            train_idx=train_idx,\n            val_idx=val_idx,\n            X_full=X_full,\n            y_full=y_full,\n            X_test=X_test,\n            num_features=num_features,\n            model_dir=model_dir,\n            num_epochs=num_epochs,\n            lr=lr,\n            encoding_size=encoding_size,\n            seed=seed\n        )\n        \n        all_results.append(result)\n        all_test_predictions.append(result['test_predictions'])\n        \n        # Print intermediate summary\n        correlations = [r['best_val_corr'] for r in all_results]\n        print(f\"📊 Progress: {fold_idx + 1}/{len(all_splits)} | \"\n              f\"Current: {result['best_val_corr']:.6f} | \"\n              f\"Avg so far: {np.mean(correlations):.6f} | \"\n              f\"Best so far: {np.max(correlations):.6f}\")\n    \n    # Ensemble predictions\n    print(f\"\\n🔮 Creating Enhanced Ensemble Predictions...\")\n    \n    # Simple average ensemble\n    ensemble_predictions = np.mean(all_test_predictions, axis=0)\n    \n    # Weighted ensemble based on validation performance\n    weights = np.array([r['best_val_corr'] for r in all_results])\n    weights = np.maximum(weights, 0)  # Ensure non-negative weights\n    weights = weights / np.sum(weights)  # Normalize\n    \n    weighted_ensemble_predictions = np.average(all_test_predictions, axis=0, weights=weights)\n    \n    # Create submissions\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    sample_submission = pd.read_csv(sample_sub_path)\n    \n    # Simple ensemble submission\n    simple_submission = sample_submission.copy()\n    simple_submission.iloc[:, 1] = ensemble_predictions\n    simple_submission.to_csv('/kaggle/working/enhanced_ensemble_simple_submission.csv', index=False)\n    \n    # Weighted ensemble submission\n    weighted_submission = sample_submission.copy()\n    weighted_submission.iloc[:, 1] = weighted_ensemble_predictions\n    weighted_submission.to_csv('/kaggle/working/enhanced_ensemble_weighted_submission.csv', index=False)\n    \n    # Save ensemble results\n    ensemble_results = {\n        'all_results': all_results,\n        'ensemble_predictions': ensemble_predictions,\n        'weighted_ensemble_predictions': weighted_ensemble_predictions,\n        'weights': weights,\n        'all_splits': all_splits,\n        'encoding_size': encoding_size\n    }\n    \n    with open(model_dir / 'enhanced_ensemble_results.pkl', 'wb') as f:\n        pickle.dump(ensemble_results, f)\n    \n    # Print final summary\n    print(f\"\\n🎉 Enhanced TimeSeriesSplit Ensemble Training Completed!\")\n    print(\"=\" * 80)\n    print(\"📊 Individual Fold Performance:\")\n    \n    for i, result in enumerate(all_results):\n        print(f\"  Fold {i+1:2d}: \"\n              f\"Corr = {result['best_val_corr']:8.6f} | \"\n              f\"Epoch = {result['best_epoch']:3d} | \"\n              f\"Weight = {weights[i]:6.4f}\")\n    \n    correlations = [r['best_val_corr'] for r in all_results]\n    print(f\"\\n📈 Enhanced Ensemble Statistics:\")\n    print(f\"  Average Correlation: {np.mean(correlations):.6f}\")\n    print(f\"  Best Single Fold:   {np.max(correlations):.6f}\")\n    print(f\"  Worst Single Fold:  {np.min(correlations):.6f}\")\n    print(f\"  Standard Deviation:  {np.std(correlations):.6f}\")\n    \n    print(f\"\\n🔧 Architecture Enhancements Applied:\")\n    print(f\"  ✅ Corrected AutoEncoder: Expanding Encoder -> Compressing Decoder\")\n    print(f\"  ✅ AutoEncoder latent space: {encoding_size}D\")\n    print(f\"  ✅ Gaussian Noise data augmentation (σ=0.05)\")\n    print(f\"  ✅ Swish activation for smooth gradients\")\n    print(f\"  ✅ Combined loss: MSE(0.25) + Correlation(0.6) + AE(0.15)\")\n    \n    print(f\"\\n💾 Enhanced Files Saved:\")\n    print(f\"  Simple Ensemble:   enhanced_ensemble_simple_submission.csv\")\n    print(f\"  Weighted Ensemble: enhanced_ensemble_weighted_submission.csv\")\n    print(f\"  Model Directory:   {model_dir}\")\n    print(f\"  Results Pickle:    enhanced_ensemble_results.pkl\")\n    \n    return ensemble_results\n\ndef main():\n    \"\"\"Main enhanced ensemble training pipeline\"\"\"\n    \n    # Run Enhanced TimeSeriesSplit ensemble training\n    results = enhanced_timeseries_ensemble_training(\n        n_splits=4,\n        max_train_size=100_000_000,\n        gap=5000,\n        num_epochs=80,  # Adjust based on your time constraints\n        lr=0.0001,\n        encoding_size=64,  # AutoEncoder latent dimension\n        seed=42\n    )\n    \n    return results\n\nif __name__ == \"__main__\":\n    enhanced_ensemble_results = main()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-31T13:54:13.773181Z","iopub.execute_input":"2025-05-31T13:54:13.773496Z","iopub.status.idle":"2025-05-31T13:54:13.847828Z","shell.execute_reply.started":"2025-05-31T13:54:13.773475Z","shell.execute_reply":"2025-05-31T13:54:13.847100Z"}},"outputs":[],"execution_count":null}]}