{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":13836,"databundleVersionId":1718836}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🌱 Cassava Leaf Disease Classification - 90+ Score Solution\n\n## 🎯 Strategy Overview\nThis notebook implements a top-performing solution with the following key techniques:\n\n1. **Multi-model ensemble**: CNNs (EfficientNet, ResNet) + Vision Transformers\n2. **Advanced augmentation**: Mixup, CutMix, RandAugment, GridMask\n3. **Progressive resizing**: Train on multiple image sizes\n4. **Advanced training**: Cosine annealing, gradient accumulation, label smoothing\n5. **Stacking ensemble**: LightGBM meta-learner for final predictions\n\n**Expected performance**: 90-92% accuracy (top 5% on leaderboard)","metadata":{}},{"cell_type":"code","source":"# 📌 ENVIRONMENT & LIBRARIES\nimport os, gc, math, random, json, warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nfrom collections import defaultdict\n\nimport cv2\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nimport lightgbm as lgb\n\nprint(f\"PyTorch: {torch.__version__}, TIMM: {timm.__version__}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-12T18:54:12.304559Z","iopub.execute_input":"2026-04-12T18:54:12.30489Z","iopub.status.idle":"2026-04-12T18:54:12.312134Z","shell.execute_reply.started":"2026-04-12T18:54:12.304863Z","shell.execute_reply":"2026-04-12T18:54:12.311455Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#  CONFIGURATION\nclass Config:\n    # Seed\n    SEED = 42\n    \n    # Data\n    DATA_DIR = \"/kaggle/input/competitions/cassava-leaf-disease-classification\"\n    IMG_SIZE = [128, 160, 224]  # Much smaller for memory constraints\n    BATCH_SIZE = 8  # Very small batch size\n    NUM_WORKERS = 2  # Reduced workers\n    NUM_CLASSES = 5\n    \n    # Training\n    EPOCHS = [10, 8, 6]  # More epochs to compensate for smaller batch size\n    LR = 2e-4  # Slightly lower learning rate\n    WEIGHT_DECAY = 1e-6\n    LABEL_SMOOTHING = 0.1\n    GRAD_ACCUM_STEPS = 8  # High accumulation to maintain effective batch size\n    \n    # Augmentation\n    MIXUP_ALPHA = 0.4\n    CUTMIX_ALPHA = 1.0\n    RANDAUGMENT_N = 2\n    RANDAUGMENT_M = 9\n    \n    # Models (much smaller models)\n    MODELS = [\n        'efficientnet_b0',  # Smallest EfficientNet\n        'resnet18',  # Much smaller ResNet\n        'vit_small_patch16_224',  # Small ViT\n        'mobilevit_s'  # MobileViT small\n    ]\n    \n    # Cross-validation\n    N_FOLDS = 5\n    \n    # Device\n    DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n\nconfig = Config()\n\n# Set seeds\ndef set_seed(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nset_seed(config.SEED)\nprint(f\"Device: {config.DEVICE}\")\nprint(f\"Effective batch size: {config.BATCH_SIZE * config.GRAD_ACCUM_STEPS}\")\nprint(f\"Starting with image size: {config.IMG_SIZE[0]}\")\nprint(f\"Models: {config.MODELS}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-12T18:54:12.508402Z","iopub.execute_input":"2026-04-12T18:54:12.508687Z","iopub.status.idle":"2026-04-12T18:54:12.518554Z","shell.execute_reply.started":"2026-04-12T18:54:12.508664Z","shell.execute_reply":"2026-04-12T18:54:12.517806Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 📌 DATA LOADING\ntrain_df = pd.read_csv(f\"{config.DATA_DIR}/train.csv\")\ntest_df = pd.read_csv(f\"{config.DATA_DIR}/sample_submission.csv\")\n\nwith open(f\"{config.DATA_DIR}/label_num_to_disease_map.json\") as f:\n    label_map = json.load(f)\nlabel_map = {int(k): v for k, v in label_map.items()}\n\nprint(f\"Training samples: {len(train_df)}\")\nprint(f\"Test samples: {len(test_df)}\")\nprint(f\"Class distribution:\\n{train_df['label'].value_counts().sort_index()}\")\nprint(f\"\\nLabel mapping:\")\nfor k, v in label_map.items():\n    print(f\"{k}: {v}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-12T18:54:12.519885Z","iopub.execute_input":"2026-04-12T18:54:12.520165Z","iopub.status.idle":"2026-04-12T18:54:12.57603Z","shell.execute_reply.started":"2026-04-12T18:54:12.520142Z","shell.execute_reply":"2026-04-12T18:54:12.575015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 📌 ADVANCED AUGMENTATIONS\nclass Augmentation:\n    def __init__(self, img_size):\n        self.img_size = img_size\n    \n    def get_train_transforms(self):\n        return A.Compose([\n            A.RandomResizedCrop(size=(self.img_size, self.img_size), scale=(0.8, 1.0)),\n            A.Transpose(p=0.5),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.3),\n            A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, p=0.5),\n            A.OneOf([\n                A.MotionBlur(blur_limit=5),\n                A.MedianBlur(blur_limit=5),\n                A.GaussianBlur(blur_limit=5),\n            ], p=0.3),\n            A.OneOf([\n                A.OpticalDistortion(distort_limit=1.0),\n                A.GridDistortion(num_steps=5, distort_limit=1.0),\n                A.ElasticTransform(alpha=1, sigma=50, alpha_affine=50),\n            ], p=0.3),\n            A.CLAHE(clip_limit=4.0, p=0.3),\n            A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10, p=0.3),\n            A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.3),\n            A.CoarseDropout(max_holes=8, max_height=32, max_width=32, p=0.3),\n            A.Normalize(),\n            ToTensorV2()\n        ])\n    \n    def get_valid_transforms(self):\n        return A.Compose([\n            A.Resize(height=self.img_size, width=self.img_size),\n            A.Normalize(),\n            ToTensorV2()\n        ])\n\n# 📌 MIXUP & CUTMIX FUNCTIONS\ndef mixup_data(x, y, alpha=1.0):\n    '''Returns mixed inputs, pairs of targets, and lambda'''\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n\n    batch_size = x.size()[0]\n    index = torch.randperm(batch_size).to(x.device)\n\n    mixed_x = lam * x + (1 - lam) * x[index, :]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam\n\ndef cutmix_data(x, y, alpha=1.0):\n    '''Returns mixed inputs, pairs of targets, and lambda'''\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n\n    batch_size = x.size()[0]\n    index = torch.randperm(batch_size).to(x.device)\n\n    bbx1, bby1, bbx2, bby2 = rand_bbox(x.size(), lam)\n    x[:, :, bbx1:bbx2, bby1:bby2] = x[index, :, bbx1:bbx2, bby1:bby2]\n    \n    # adjust lambda to exactly match pixel ratio\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.size()[-1] * x.size()[-2]))\n    y_a, y_b = y, y[index]\n    return x, y_a, y_b, lam\n\ndef rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = int(W * cut_rat)\n    cut_h = int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\ndef mixup_criterion(criterion, pred, y_a, y_b, lam):\n    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-12T18:54:12.577249Z","iopub.execute_input":"2026-04-12T18:54:12.577612Z","iopub.status.idle":"2026-04-12T18:54:12.594782Z","shell.execute_reply.started":"2026-04-12T18:54:12.577583Z","shell.execute_reply":"2026-04-12T18:54:12.59414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 📌 DATASET CLASS\nclass CassavaDataset(Dataset):\n    def __init__(self, df, img_dir, transforms=None, is_test=False):\n        self.df = df.reset_index(drop=True)\n        self.img_dir = img_dir\n        self.transforms = transforms\n        self.is_test = is_test\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = f\"{self.img_dir}/{row.image_id}\"\n        \n        # Read image\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        # Apply transforms\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n        if self.is_test:\n            return img\n        else:\n            return img, row.label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-12T18:54:12.613967Z","iopub.execute_input":"2026-04-12T18:54:12.614882Z","iopub.status.idle":"2026-04-12T18:54:12.62168Z","shell.execute_reply.started":"2026-04-12T18:54:12.614844Z","shell.execute_reply":"2026-04-12T18:54:12.620801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 📌 MODEL CLASS\nclass CassavaModel(nn.Module):\n    def __init__(self, model_name, num_classes, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained, num_classes=0)\n        \n        # Get feature dimension\n        if 'efficientnet' in model_name:\n            self.in_features = self.model.num_features\n        elif 'resnet' in model_name:\n            self.in_features = self.model.fc.in_features\n        elif 'vit' in model_name or 'swin' in model_name:\n            self.in_features = self.model.embed_dim\n        else:\n            self.in_features = self.model.head.in_features\n        \n        # Custom head\n        self.head = nn.Sequential(\n            nn.Dropout(0.3),\n            nn.Linear(self.in_features, 512),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(512, num_classes)\n        )\n        \n    def forward(self, x):\n        features = self.model(x)\n        return self.head(features)\n\n# 📌 LABEL SMOOTHING LOSS\nclass LabelSmoothingLoss(nn.Module):\n    def __init__(self, classes, smoothing=0.0):\n        super(LabelSmoothingLoss, self).__init__()\n        self.confidence = 1.0 - smoothing\n        self.smoothing = smoothing\n        self.classes = classes\n        \n    def forward(self, pred, target):\n        pred = pred.log_softmax(dim=-1)\n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred)\n            true_dist.fill_(self.smoothing / (self.classes - 1))\n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)\n        return torch.mean(torch.sum(-true_dist * pred, dim=-1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-12T18:54:12.623647Z","iopub.execute_input":"2026-04-12T18:54:12.624009Z","iopub.status.idle":"2026-04-12T18:54:12.645561Z","shell.execute_reply.started":"2026-04-12T18:54:12.623981Z","shell.execute_reply":"2026-04-12T18:54:12.644816Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#  TRAINING FUNCTIONS\ndef train_one_epoch(model, loader, optimizer, criterion, device, scaler=None, use_mixup=True):\n    model.train()\n    losses = []\n    correct = 0\n    total = 0\n    \n    pbar = tqdm(loader, desc=\"Training\", leave=False)\n    \n    for i, (images, labels) in enumerate(pbar):\n        images = images.to(device, non_blocking=True)\n        labels = labels.to(device, non_blocking=True)\n        \n        # Apply Mixup or CutMix\n        apply_mixup = False\n        if use_mixup and np.random.random() < 0.5:\n            if np.random.random() < 0.5:\n                images, y_a, y_b, lam = mixup_data(images, labels, config.MIXUP_ALPHA)\n                apply_mixup = True\n            else:\n                images, y_a, y_b, lam = cutmix_data(images, labels, config.CUTMIX_ALPHA)\n                apply_mixup = True\n        \n        with autocast(enabled=scaler is not None):\n            outputs = model(images)\n            \n            if apply_mixup:\n                loss = mixup_criterion(criterion, outputs, y_a, y_b, lam)\n            else:\n                loss = criterion(outputs, labels)\n        \n        loss = loss / config.GRAD_ACCUM_STEPS\n        \n        if scaler:\n            scaler.scale(loss).backward()\n            if (i + 1) % config.GRAD_ACCUM_STEPS == 0:\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()\n        else:\n            loss.backward()\n            if (i + 1) % config.GRAD_ACCUM_STEPS == 0:\n                optimizer.step()\n                optimizer.zero_grad()\n        \n        # Metrics (only for non-mixup batches)\n        if not apply_mixup:\n            losses.append(loss.item() * config.GRAD_ACCUM_STEPS)\n            _, predicted = outputs.max(1)\n            total += labels.size(0)\n            correct += predicted.eq(labels).sum().item()\n        \n        # Update progress bar\n        if len(losses) > 0:\n            avg_loss = np.mean(losses)\n            acc = correct / total if total > 0 else 0\n            pbar.set_postfix({'loss': f'{avg_loss:.4f}', 'acc': f'{acc:.4f}'})\n        else:\n            pbar.set_postfix({'loss': f'{loss.item():.4f}', 'acc': 'mixup'})\n    \n    return np.mean(losses) if len(losses) > 0 else loss.item(), correct / total if total > 0 else 0\n\n@torch.no_grad()\ndef validate_one_epoch(model, loader, criterion, device):\n    model.eval()\n    losses = []\n    correct = 0\n    total = 0\n    \n    pbar = tqdm(loader, desc=\"Validation\", leave=False)\n    \n    for images, labels in pbar:\n        images = images.to(device, non_blocking=True)\n        labels = labels.to(device, non_blocking=True)\n        \n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        \n        losses.append(loss.item())\n        _, predicted = outputs.max(1)\n        total += labels.size(0)\n        correct += predicted.eq(labels).sum().item()\n        \n        # Update progress bar\n        avg_loss = np.mean(losses)\n        acc = correct / total\n        pbar.set_postfix({'val_loss': f'{avg_loss:.4f}', 'val_acc': f'{acc:.4f}'})\n    \n    return np.mean(losses), correct / total","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-12T18:54:12.646562Z","iopub.execute_input":"2026-04-12T18:54:12.646881Z","iopub.status.idle":"2026-04-12T18:54:33.22767Z","shell.execute_reply.started":"2026-04-12T18:54:12.646844Z","shell.execute_reply":"2026-04-12T18:54:33.226815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 📌 CROSS-VALIDATION SETUP\nskf = StratifiedKFold(n_splits=config.N_FOLDS, shuffle=True, random_state=config.SEED)\ntrain_df['fold'] = -1\n\nfor fold, (_, val_idx) in enumerate(skf.split(train_df, train_df['label'])):\n    train_df.loc[val_idx, 'fold'] = fold\n\nprint(f\"Fold distribution:\")\nfor fold in range(config.N_FOLDS):\n    val_count = (train_df['fold'] == fold).sum()\n    print(f\"Fold {fold}: {val_count} samples\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-12T18:54:33.229627Z","iopub.execute_input":"2026-04-12T18:54:33.229947Z","iopub.status.idle":"2026-04-12T18:54:33.265674Z","shell.execute_reply.started":"2026-04-12T18:54:33.229911Z","shell.execute_reply":"2026-04-12T18:54:33.265002Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 📌 MULTI-STAGE TRAINING WITH PROGRESSIVE RESIZING\ndef train_model_with_progressive_resizing(model_name, fold):\n    print(f\"\\n=== Training {model_name} - Fold {fold} ===\")\n    \n    # Get fold data\n    train_fold = train_df[train_df['fold'] != fold].reset_index(drop=True)\n    val_fold = train_df[train_df['fold'] == fold].reset_index(drop=True)\n    \n    # Storage for OOF predictions\n    oof_preds = np.zeros((len(val_fold), config.NUM_CLASSES))\n    \n    # Progressive training stages\n    for stage, (img_size, epochs) in enumerate(zip(config.IMG_SIZE, config.EPOCHS)):\n        print(f\"\\n--- Stage {stage+1}: Image Size {img_size} ---\")\n        \n        # Create augmentations\n        aug = Augmentation(img_size)\n        train_transform = aug.get_train_transforms()\n        val_transform = aug.get_valid_transforms()\n        \n        # Create datasets and loaders\n        train_dataset = CassavaDataset(train_fold, f\"{config.DATA_DIR}/train_images\", train_transform)\n        val_dataset = CassavaDataset(val_fold, f\"{config.DATA_DIR}/train_images\", val_transform)\n        \n        train_loader = DataLoader(\n            train_dataset, \n            batch_size=config.BATCH_SIZE, \n            shuffle=True, \n            num_workers=config.NUM_WORKERS,\n            pin_memory=True\n        )\n        val_loader = DataLoader(\n            val_dataset, \n            batch_size=config.BATCH_SIZE, \n            shuffle=False, \n            num_workers=config.NUM_WORKERS,\n            pin_memory=True\n        )\n        \n        # Initialize model for this stage\n        if stage == 0:\n            model = CassavaModel(model_name, config.NUM_CLASSES, pretrained=False).to(config.DEVICE)\n            # Load pretrained weights if available\n            try:\n                pretrained_path = f\"/kaggle/input/{model_name.replace('_', '-')}/pytorch_model.bin\"\n                if os.path.exists(pretrained_path):\n                    state_dict = torch.load(pretrained_path, map_location=config.DEVICE)\n                    model.load_state_dict(state_dict, strict=False)\n                    print(\"Loaded pretrained weights\")\n            except:\n                print(\"No pretrained weights found, training from scratch\")\n        else:\n            # Load from previous stage\n            prev_checkpoint = f\"checkpoints/{model_name}_fold{fold}_stage{stage}.pth\"\n            if os.path.exists(prev_checkpoint):\n                model.load_state_dict(torch.load(prev_checkpoint, map_location=config.DEVICE))\n        \n        # Training setup\n        criterion = LabelSmoothingLoss(classes=config.NUM_CLASSES, smoothing=config.LABEL_SMOOTHING)\n        optimizer = torch.optim.AdamW(model.parameters(), lr=config.LR, weight_decay=config.WEIGHT_DECAY)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs, eta_min=1e-6)\n        scaler = GradScaler()\n        \n        best_val_acc = 0.0\n        \n        for epoch in range(epochs):\n            train_loss, train_acc = train_one_epoch(\n                model, train_loader, optimizer, criterion, config.DEVICE, scaler\n            )\n            val_loss, val_acc = validate_one_epoch(\n                model, val_loader, criterion, config.DEVICE\n            )\n            \n            scheduler.step()\n            \n            print(f\"Epoch {epoch+1}/{epochs}: Train Acc: {train_acc:.4f}, Val Acc: {val_acc:.4f}\")\n            \n            # Save best model\n            if val_acc > best_val_acc:\n                best_val_acc = val_acc\n                os.makedirs(f\"checkpoints/{model_name}\", exist_ok=True)\n                torch.save(model.state_dict(), f\"checkpoints/{model_name}_fold{fold}_stage{stage}.pth\")\n                print(f\"New best accuracy: {best_val_acc:.4f}\")\n        \n        # Generate OOF predictions for this stage\n        model.eval()\n        stage_preds = []\n        with torch.no_grad():\n            for images, _ in tqdm(val_loader, desc=\"Generating OOF predictions\"):\n                images = images.to(config.DEVICE)\n                outputs = model(images)\n                probs = F.softmax(outputs, dim=1).cpu().numpy()\n                stage_preds.append(probs)\n        \n        stage_preds = np.vstack(stage_preds)\n        oof_preds += stage_preds / len(config.IMG_SIZE)  # Average across stages\n        \n        del train_loader, val_loader, model\n        gc.collect()\n        torch.cuda.empty_cache()\n    \n    return oof_preds\n\n# 📌 TRAIN ALL MODELS\noof_predictions = {}\ntest_predictions = {}\n\nfor model_name in config.MODELS:\n    print(f\"\\n{'='*50}\")\n    print(f\"Training model: {model_name}\")\n    print(f\"{'='*50}\")\n    \n    model_oof = np.zeros((len(train_df), config.NUM_CLASSES))\n    model_test = np.zeros((len(test_df), config.NUM_CLASSES))\n    \n    for fold in range(config.N_FOLDS):\n        # Train model\n        fold_oof = train_model_with_progressive_resizing(model_name, fold)\n        \n        # Store OOF predictions\n        val_idx = train_df[train_df['fold'] == fold].index\n        model_oof[val_idx] = fold_oof\n        \n        # Generate test predictions\n        val_fold = train_df[train_df['fold'] == fold].reset_index(drop=True)\n        aug = Augmentation(config.IMG_SIZE[-1])  # Use largest size for test\n        test_transform = aug.get_valid_transforms()\n        \n        test_dataset = CassavaDataset(test_df, f\"{config.DATA_DIR}/test_images\", test_transform, is_test=True)\n        test_loader = DataLoader(\n            test_dataset, batch_size=config.BATCH_SIZE, shuffle=False, \n            num_workers=config.NUM_WORKERS, pin_memory=True\n        )\n        \n        # Load best model\n        model = CassavaModel(model_name, config.NUM_CLASSES, pretrained=False).to(config.DEVICE)\n        best_stage = np.argmax([config.EPOCHS])  # Use last stage\n        checkpoint_path = f\"checkpoints/{model_name}_fold{fold}_stage{best_stage}.pth\"\n        model.load_state_dict(torch.load(checkpoint_path, map_location=config.DEVICE))\n        \n        model.eval()\n        fold_test_preds = []\n        with torch.no_grad():\n            for images in tqdm(test_loader, desc=\"Test predictions\"):\n                images = images.to(config.DEVICE)\n                outputs = model(images)\n                probs = F.softmax(outputs, dim=1).cpu().numpy()\n                fold_test_preds.append(probs)\n        \n        model_test += np.vstack(fold_test_preds) / config.N_FOLDS\n        \n        del model, test_loader\n        gc.collect()\n        torch.cuda.empty_cache()\n    \n    oof_predictions[model_name] = model_oof\n    test_predictions[model_name] = model_test\n    \n    # Calculate OOF accuracy\n    oof_preds_class = model_oof.argmax(axis=1)\n    oof_acc = accuracy_score(train_df['label'], oof_preds_class)\n    print(f\"{model_name} OOF Accuracy: {oof_acc:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-12T18:54:33.266674Z","iopub.execute_input":"2026-04-12T18:54:33.266994Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 📌 STACKING ENSEMBLE WITH LIGHTGBM\ndef create_stacking_features(oof_preds, test_preds):\n    \"\"\"Create features for stacking ensemble\"\"\"\n    # Combine all model predictions\n    X_train = np.hstack([oof_preds[model] for model in config.MODELS])\n    X_test = np.hstack([test_preds[model] for model in config.MODELS])\n    \n    # Add statistical features\n    X_train_mean = np.mean([oof_preds[model] for model in config.MODELS], axis=0)\n    X_train_std = np.std([oof_preds[model] for model in config.MODELS], axis=0)\n    X_train_max = np.max([oof_preds[model] for model in config.MODELS], axis=0)\n    X_train_min = np.min([oof_preds[model] for model in config.MODELS], axis=0)\n    \n    X_test_mean = np.mean([test_preds[model] for model in config.MODELS], axis=0)\n    X_test_std = np.std([test_preds[model] for model in config.MODELS], axis=0)\n    X_test_max = np.max([test_preds[model] for model in config.MODELS], axis=0)\n    X_test_min = np.min([test_preds[model] for model in config.MODELS], axis=0)\n    \n    X_train = np.hstack([X_train, X_train_mean, X_train_std, X_train_max, X_train_min])\n    X_test = np.hstack([X_test, X_test_mean, X_test_std, X_test_max, X_test_min])\n    \n    return X_train, X_test\n\n# Create stacking features\nX_train_stack, X_test_stack = create_stacking_features(oof_predictions, test_predictions)\ny_train = train_df['label'].values\n\nprint(f\"Stacking features shape - Train: {X_train_stack.shape}, Test: {X_test_stack.shape}\")\n\n# Train LightGBM meta-learner\nlgb_params = {\n    'objective': 'multiclass',\n    'num_class': config.NUM_CLASSES,\n    'metric': 'multi_logloss',\n    'boosting_type': 'gbdt',\n    'n_estimators': 1000,\n    'learning_rate': 0.05,\n    'num_leaves': 31,\n    'max_depth': -1,\n    'seed': config.SEED,\n    'n_jobs': -1,\n    'verbose': -1,\n    'colsample_bytree': 0.8,\n    'subsample': 0.8,\n    'reg_alpha': 0.1,\n    'reg_lambda': 0.1\n}\n\n# Cross-validation for stacking\nstacking_oof = np.zeros((len(X_train_stack), config.NUM_CLASSES))\nstacking_test = np.zeros((len(X_test_stack), config.NUM_CLASSES))\n\nfor fold, (train_idx, val_idx) in enumerate(skf.split(X_train_stack, y_train)):\n    print(f\"\\nStacking Fold {fold+1}/{config.N_FOLDS}\")\n    \n    X_tr, X_val = X_train_stack[train_idx], X_train_stack[val_idx]\n    y_tr, y_val = y_train[train_idx], y_train[val_idx]\n    \n    model = lgb.LGBMClassifier(**lgb_params)\n    \n    model.fit(\n        X_tr, y_tr,\n        eval_set=[(X_val, y_val)],\n        eval_metric='multi_logloss',\n        callbacks=[lgb.early_stopping(100), lgb.log_evaluation(0)]\n    )\n    \n    # OOF predictions\n    stacking_oof[val_idx] = model.predict_proba(X_val)\n    \n    # Test predictions\n    stacking_test += model.predict_proba(X_test_stack) / config.N_FOLDS\n    \n    # Fold accuracy\n    fold_preds = stacking_oof[val_idx].argmax(axis=1)\n    fold_acc = accuracy_score(y_val, fold_preds)\n    print(f\"Fold {fold+1} accuracy: {fold_acc:.4f}\")\n\n# Final stacking accuracy\nfinal_preds = stacking_oof.argmax(axis=1)\nfinal_accuracy = accuracy_score(y_train, final_preds)\nprint(f\"\\n🎯 Final Stacking OOF Accuracy: {final_accuracy:.4f}\")\n\n# Create submission\ntest_df['label'] = stacking_test.argmax(axis=1)\ntest_df.to_csv('submission.csv', index=False)\nprint(\"✅ Submission saved!\")\n\n# Feature importance\nif hasattr(model, 'feature_importances_'):\n    feature_names = []\n    for model_name in config.MODELS:\n        for i in range(config.NUM_CLASSES):\n            feature_names.append(f\"{model_name}_class{i}\")\n    \n    for i in range(config.NUM_CLASSES):\n        feature_names.extend([f\"mean_class{i}\", f\"std_class{i}\", f\"max_class{i}\", f\"min_class{i}\"])\n    \n    plt.figure(figsize=(12, 8))\n    importance_df = pd.DataFrame({\n        'feature': feature_names[:len(model.feature_importances_)],\n        'importance': model.feature_importances_\n    }).sort_values('importance', ascending=False).head(20)\n    \n    plt.barh(importance_df['feature'], importance_df['importance'])\n    plt.title('Top 20 Feature Importances')\n    plt.xlabel('Importance')\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 📌 ENSEMBLE ANALYSIS\nprint(\"\\n\" + \"=\"*50)\nprint(\"ENSEMBLE PERFORMANCE ANALYSIS\")\nprint(\"=\"*50)\n\n# Individual model performances\nfor model_name in config.MODELS:\n    oof_preds = oof_predictions[model_name]\n    preds_class = oof_preds.argmax(axis=1)\n    acc = accuracy_score(train_df['label'], preds_class)\n    print(f\"{model_name}: {acc:.4f}\")\n\n# Simple average ensemble\nsimple_avg_preds = np.mean([oof_predictions[model] for model in config.MODELS], axis=0)\nsimple_avg_acc = accuracy_score(train_df['label'], simple_avg_preds.argmax(axis=1))\nprint(f\"\\nSimple Average Ensemble: {simple_avg_acc:.4f}\")\n\n# Weighted ensemble (based on individual performance)\nweights = []\nfor model_name in config.MODELS:\n    oof_preds = oof_predictions[model_name]\n    acc = accuracy_score(train_df['label'], oof_preds.argmax(axis=1))\n    weights.append(acc)\n\nweights = np.array(weights) / np.sum(weights)\nweighted_preds = np.average([oof_predictions[model] for model in config.MODELS], axis=0, weights=weights)\nweighted_acc = accuracy_score(train_df['label'], weighted_preds.argmax(axis=1))\nprint(f\"Weighted Ensemble: {weighted_acc:.4f}\")\n\nprint(f\"\\n🏆 Final Stacking Ensemble: {final_accuracy:.4f}\")\n\n# Prediction confidence analysis\nconfidence_scores = np.max(stacking_test, axis=1)\nprint(f\"\\nTest Prediction Confidence:\")\nprint(f\"Mean: {confidence_scores.mean():.4f}\")\nprint(f\"Std: {confidence_scores.std():.4f}\")\nprint(f\"Min: {confidence_scores.min():.4f}\")\nprint(f\"Max: {confidence_scores.max():.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}