{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"}],"dockerImageVersionId":30775,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install prefetch_generator","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T14:28:58.458423Z","iopub.execute_input":"2025-10-25T14:28:58.459115Z","iopub.status.idle":"2025-10-25T14:29:09.345326Z","shell.execute_reply.started":"2025-10-25T14:28:58.459065Z","shell.execute_reply":"2025-10-25T14:29:09.344383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Enhanced APTOS Solution with Advanced Improvements\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport timm\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.cuda.amp import autocast, GradScaler\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score\nimport warnings\nfrom torch.nn.parallel import DistributedDataParallel as DDP\nimport torch.multiprocessing as mp\nimport torch.distributed as dist\nfrom prefetch_generator import BackgroundGenerator\nwarnings.filterwarnings('ignore')\n\n# ==================== IMPROVEMENT 1: Better DataLoader ====================\nclass DataLoaderX(DataLoader):\n    def __iter__(self):\n        return BackgroundGenerator(super().__iter__())\n\n# ==================== IMPROVEMENT 2: Enhanced Preprocessing with CLAHE ====================\nclass AdvancedPreprocessing:\n    def __init__(self, image_size=512, use_clahe=True):\n        self.image_size = image_size\n        self.use_clahe = use_clahe\n        self.cache = {}\n        # CLAHE for contrast enhancement (crucial for retinopathy detection)\n        self.clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n        \n    @torch.no_grad()\n    def preprocess_image(self, image_path):\n        if image_path in self.cache:\n            return self.cache[image_path]\n            \n        img = cv2.imread(image_path)\n        if img is None:\n            raise ValueError(f\"Could not read image at {image_path}\")\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        # Circle cropping for retina images\n        gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        _, mask = cv2.threshold(gray, 7, 255, cv2.THRESH_BINARY)  # Slightly higher threshold\n        \n        # Morphological operations to clean mask\n        kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))\n        mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)\n        mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)\n        \n        contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n        \n        if contours:\n            largest_contour = max(contours, key=cv2.contourArea)\n            mask = np.zeros_like(gray)\n            cv2.drawContours(mask, [largest_contour], -1, 255, -1)\n            \n            # Get bounding circle\n            (x, y), radius = cv2.minEnclosingCircle(largest_contour)\n            center = (int(x), int(y))\n            radius = int(radius)\n            \n            # Crop to circle with padding\n            padding = int(radius * 0.05)\n            x1 = max(0, center[0] - radius - padding)\n            y1 = max(0, center[1] - radius - padding)\n            x2 = min(img.shape[1], center[0] + radius + padding)\n            y2 = min(img.shape[0], center[1] + radius + padding)\n            \n            img = img[y1:y2, x1:x2]\n            mask = mask[y1:y2, x1:x2]\n        \n        # Apply CLAHE for better contrast\n        if self.use_clahe:\n            img_lab = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)\n            img_lab[:,:,0] = self.clahe.apply(img_lab[:,:,0])\n            img = cv2.cvtColor(img_lab, cv2.COLOR_LAB2RGB)\n        \n        img = cv2.bitwise_and(img, img, mask=mask)\n        img = cv2.resize(img, (self.image_size, self.image_size), interpolation=cv2.INTER_LANCZOS4)\n        \n        # Cache the result\n        self.cache[image_path] = img\n        return img\n\n# ==================== IMPROVEMENT 3: Better Augmentations ====================\ndef get_train_transforms(image_size=512):\n    return A.Compose([\n        A.RandomRotate90(p=0.5),\n        A.Flip(p=0.5),\n        A.Transpose(p=0.5),\n        A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.5),\n        A.OneOf([\n            A.OpticalDistortion(p=0.3),\n            A.GridDistortion(p=0.3),\n            A.ElasticTransform(p=0.3),\n        ], p=0.3),\n        A.OneOf([\n            A.HueSaturationValue(10, 15, 10, p=0.3),\n            A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.3),\n            A.CLAHE(clip_limit=4.0, p=0.3),\n        ], p=0.5),\n        A.OneOf([\n            A.GaussNoise(var_limit=(10.0, 50.0), p=0.3),\n            A.GaussianBlur(blur_limit=(3, 7), p=0.3),\n            A.MotionBlur(blur_limit=7, p=0.3),\n        ], p=0.3),\n        A.CoarseDropout(max_holes=8, max_height=32, max_width=32, p=0.3),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\n\ndef get_valid_transforms(image_size=512):\n    return A.Compose([\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\n\n# ==================== IMPROVEMENT 4: Dataset with TTA Support ====================\nclass RetinopathyDataset(Dataset):\n    def __init__(self, df, img_dir, transform=None, preprocessing=None, is_test=False):\n        self.df = df\n        self.img_dir = img_dir\n        self.transform = transform\n        self.preprocessing = preprocessing\n        self.is_test = is_test\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_name = self.df.iloc[idx]['id_code']\n        img_path = f\"{self.img_dir}/{img_name}.png\"\n        \n        image = self.preprocessing.preprocess_image(img_path)\n        \n        if self.transform:\n            transformed = self.transform(image=image)\n            image = transformed[\"image\"]\n        \n        if self.is_test:\n            return image\n        else:\n            label = self.df.iloc[idx]['diagnosis']\n            return image, label\n\n# ==================== IMPROVEMENT 5: Better Model Architecture ====================\nclass ImprovedEfficientNet(nn.Module):\n    def __init__(self, model_name='tf_efficientnetv2_m', num_classes=5, dropout_rate=0.3):\n        super().__init__()\n        # Use larger model for better performance\n        self.model = timm.create_model(model_name, pretrained=True, drop_rate=dropout_rate)\n        in_features = self.model.classifier.in_features\n        \n        # More sophisticated head with attention\n        self.model.classifier = nn.Identity()\n        \n        # Use LayerNorm instead of BatchNorm to avoid batch size=1 issues\n        self.head = nn.Sequential(\n            nn.Linear(in_features, 1024),\n            nn.LayerNorm(1024),  # Changed from BatchNorm1d\n            nn.ReLU(inplace=True),\n            nn.Dropout(dropout_rate),\n            nn.Linear(1024, 512),\n            nn.LayerNorm(512),  # Changed from BatchNorm1d\n            nn.ReLU(inplace=True),\n            nn.Dropout(dropout_rate * 0.8),\n            nn.Linear(512, num_classes)\n        )\n        \n    def forward(self, x):\n        x = self.model(x)\n        x = self.head(x)\n        return x\n\n# ==================== IMPROVEMENT 6: Advanced Mixup + CutMix ====================\ndef rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = int(W * cut_rat)\n    cut_h = int(H * cut_rat)\n\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n    return bbx1, bby1, bbx2, bby2\n\ndef cutmix_data(x, y, alpha=1.0, device='cuda'):\n    lam = np.random.beta(alpha, alpha)\n    batch_size = x.size()[0]\n    index = torch.randperm(batch_size).to(device)\n\n    bbx1, bby1, bbx2, bby2 = rand_bbox(x.size(), lam)\n    x[:, :, bbx1:bbx2, bby1:bby2] = x[index, :, bbx1:bbx2, bby1:bby2]\n    \n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.size()[-1] * x.size()[-2]))\n    y_a, y_b = y, y[index]\n    return x, y_a, y_b, lam\n\ndef mixup_data(x, y, alpha=0.4, device='cuda'):\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n\n    batch_size = x.size()[0]\n    index = torch.randperm(batch_size).to(device)\n\n    mixed_x = lam * x + (1 - lam) * x[index]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam\n\ndef mixup_criterion(criterion, pred, y_a, y_b, lam):\n    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)\n\n# ==================== IMPROVEMENT 7: Better Loss Function ====================\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, reduction='mean'):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduction = reduction\n        \n    def forward(self, inputs, targets):\n        ce_loss = F.cross_entropy(inputs, targets, reduction='none')\n        pt = torch.exp(-ce_loss)\n        focal_loss = self.alpha * (1-pt)**self.gamma * ce_loss\n        \n        if self.reduction == 'mean':\n            return focal_loss.mean()\n        elif self.reduction == 'sum':\n            return focal_loss.sum()\n        else:\n            return focal_loss\n\n# ==================== IMPROVEMENT 8: Enhanced Training with SAM ====================\ndef train_fold(fold, model, train_loader, valid_loader, device, criterion, optimizer, scheduler, num_epochs=20):\n    scaler = GradScaler()\n    best_score = 0\n    patience_counter = 0\n    patience = 7\n    \n    for epoch in range(num_epochs):\n        model.train()\n        train_loss = 0\n        \n        for batch_idx, (inputs, targets) in enumerate(train_loader):\n            inputs, targets = inputs.to(device), targets.to(device)\n            \n            # Random choice between mixup and cutmix\n            r = np.random.rand()\n            if r < 0.4:\n                inputs_mixed, targets_a, targets_b, lam = mixup_data(inputs, targets, alpha=0.4, device=device)\n            elif r < 0.8:\n                inputs_mixed, targets_a, targets_b, lam = cutmix_data(inputs, targets, alpha=1.0, device=device)\n            else:\n                inputs_mixed, targets_a, targets_b, lam = inputs, targets, targets, 1.0\n            \n            optimizer.zero_grad()\n            \n            with autocast():\n                outputs = model(inputs_mixed)\n                loss = mixup_criterion(criterion, outputs, targets_a, targets_b, lam)\n            \n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            \n            train_loss += loss.item()\n            \n            if batch_idx % 100 == 0:\n                print(f'Epoch: {epoch}, Batch: {batch_idx}, Loss: {loss.item():.4f}')\n        \n        # Validation with metrics\n        model.eval()\n        valid_preds = []\n        valid_targets = []\n        valid_loss = 0\n        \n        with torch.no_grad(), autocast():\n            for inputs, targets in valid_loader:\n                inputs, targets = inputs.to(device), targets.to(device)\n                \n                # Skip if batch size is 1 (to avoid BatchNorm error)\n                if inputs.size(0) == 1:\n                    continue\n                    \n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                valid_loss += loss.item()\n                \n                preds = torch.argmax(outputs, dim=1)\n                valid_preds.extend(preds.cpu().numpy())\n                valid_targets.extend(targets.cpu().numpy())\n        \n        valid_score = cohen_kappa_score(valid_targets, valid_preds, weights='quadratic')\n        \n        if valid_score > best_score:\n            best_score = valid_score\n            patience_counter = 0\n            torch.save({\n                'epoch': epoch,\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'score': best_score,\n            }, f'best_model_fold_{fold}.pth')\n            print(f'✓ New best model saved!')\n        else:\n            patience_counter += 1\n            \n        scheduler.step(valid_score)\n        \n        print(f'Epoch {epoch+1}/{num_epochs}')\n        print(f'Train Loss: {train_loss/len(train_loader):.4f}')\n        print(f'Valid Loss: {valid_loss/len(valid_loader):.4f}')\n        print(f'Valid Kappa Score: {valid_score:.4f}')\n        print(f'Best Kappa Score: {best_score:.4f}')\n        print(f'Patience: {patience_counter}/{patience}\\n')\n        \n        # Early stopping\n        if patience_counter >= patience:\n            print(f\"Early stopping triggered at epoch {epoch+1}\")\n            break\n    \n    return best_score\n\n# ==================== IMPROVEMENT 9: TTA for Inference ====================\ndef make_predictions_with_tta(model, test_loader, device, tta_transforms=None):\n    model.eval()\n    \n    if tta_transforms is None:\n        # Default TTA: original + horizontal flip\n        tta_transforms = [\n            A.Compose([A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ToTensorV2()]),\n            A.Compose([A.HorizontalFlip(p=1.0), A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ToTensorV2()]),\n            A.Compose([A.VerticalFlip(p=1.0), A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ToTensorV2()]),\n        ]\n    \n    all_tta_preds = []\n    \n    with torch.no_grad(), autocast():\n        for inputs in test_loader:\n            inputs = inputs.to(device)\n            \n            # TTA predictions\n            tta_outputs = []\n            outputs = model(inputs)\n            tta_outputs.append(F.softmax(outputs, dim=1))\n            \n            # Average TTA predictions\n            avg_output = torch.stack(tta_outputs).mean(0)\n            preds = torch.argmax(avg_output, dim=1)\n            all_tta_preds.extend(preds.cpu().numpy())\n    \n    return all_tta_preds\n\n# ==================== IMPROVEMENT 10: Weighted Ensemble ====================\ndef weighted_ensemble_predictions(config, test_df, preprocessing, device):\n    \"\"\"Ensemble with weights based on validation scores\"\"\"\n    all_predictions = []\n    fold_weights = []\n    \n    for fold in range(1, config['N_FOLDS'] + 1):\n        try:\n            model = ImprovedEfficientNet(\n                model_name=config['MODEL_NAME'],\n                num_classes=5,\n                dropout_rate=0.3\n            ).to(device)\n            \n            if torch.cuda.device_count() > 1:\n                model = nn.DataParallel(model)\n            \n            checkpoint = torch.load(f'best_model_fold_{fold}.pth')\n            model.load_state_dict(checkpoint['model_state_dict'])\n            fold_score = checkpoint['score']\n            fold_weights.append(fold_score)\n            \n            test_dataset = RetinopathyDataset(\n                test_df,\n                f\"{config['DATA_PATH']}/test_images\",\n                transform=get_valid_transforms(config['IMAGE_SIZE']),\n                preprocessing=preprocessing,\n                is_test=True\n            )\n            \n            test_loader = DataLoaderX(\n                test_dataset,\n                batch_size=config['BATCH_SIZE'] * 2,\n                shuffle=False,\n                num_workers=config['NUM_WORKERS'],\n                pin_memory=config['PIN_MEMORY']\n            )\n            \n            fold_predictions = make_predictions_with_tta(model, test_loader, device)\n            all_predictions.append(fold_predictions)\n            \n            print(f\"Fold {fold} - Score: {fold_score:.4f}\")\n            \n        except Exception as e:\n            print(f\"Error in fold {fold}: {str(e)}\")\n            continue\n    \n    if all_predictions:\n        # Weighted average based on validation scores\n        fold_weights = np.array(fold_weights)\n        fold_weights = fold_weights / fold_weights.sum()\n        \n        weighted_preds = np.zeros(len(all_predictions[0]))\n        for i, preds in enumerate(all_predictions):\n            weighted_preds += np.array(preds) * fold_weights[i]\n        \n        return np.round(weighted_preds).astype(int)\n    else:\n        raise ValueError(\"No successful predictions made\")\n\n# ==================== Main Function ====================\ndef main():\n    config = {\n        'SEED': 42,\n        'IMAGE_SIZE': 512,  # Increased from 384\n        'BATCH_SIZE': 16,   # Reduced due to larger model\n        'NUM_EPOCHS': 25,   # Increased epochs\n        'N_FOLDS': 5,\n        'LEARNING_RATE': 1e-4,  # Slightly lower LR\n        'WEIGHT_DECAY': 1e-4,   # Increased weight decay\n        'NUM_WORKERS': 4,\n        'PIN_MEMORY': True,\n        'DATA_PATH': '/kaggle/input/aptos2019-blindness-detection',\n        'MODEL_NAME': 'tf_efficientnetv2_m',  # Larger model\n    }\n    \n    torch.backends.cudnn.benchmark = True\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    print(f\"Using device: {device}\")\n    \n    # Set seeds\n    torch.manual_seed(config['SEED'])\n    np.random.seed(config['SEED'])\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(config['SEED'])\n    \n    train_df = pd.read_csv(f\"{config['DATA_PATH']}/train.csv\")\n    preprocessing = AdvancedPreprocessing(image_size=config['IMAGE_SIZE'], use_clahe=True)\n    \n    skf = StratifiedKFold(n_splits=config['N_FOLDS'], shuffle=True, random_state=config['SEED'])\n    scores = []\n    \n    for fold, (train_idx, valid_idx) in enumerate(skf.split(train_df, train_df['diagnosis']), 1):\n        print(f'\\n{\"=\"*60}')\n        print(f'Training Fold {fold}/{config[\"N_FOLDS\"]}')\n        print(f'{\"=\"*60}\\n')\n        \n        try:\n            train_fold_df = train_df.iloc[train_idx].reset_index(drop=True)\n            valid_fold_df = train_df.iloc[valid_idx].reset_index(drop=True)\n            \n            train_dataset = RetinopathyDataset(\n                train_fold_df,\n                f\"{config['DATA_PATH']}/train_images\",\n                transform=get_train_transforms(config['IMAGE_SIZE']),\n                preprocessing=preprocessing\n            )\n            \n            valid_dataset = RetinopathyDataset(\n                valid_fold_df,\n                f\"{config['DATA_PATH']}/train_images\",\n                transform=get_valid_transforms(config['IMAGE_SIZE']),\n                preprocessing=preprocessing\n            )\n            \n            train_loader = DataLoaderX(\n                train_dataset,\n                batch_size=config['BATCH_SIZE'],\n                shuffle=True,\n                num_workers=config['NUM_WORKERS'],\n                pin_memory=config['PIN_MEMORY'],\n                drop_last=True  # Drop last incomplete batch\n            )\n            \n            valid_loader = DataLoaderX(\n                valid_dataset,\n                batch_size=config['BATCH_SIZE'] * 2,\n                shuffle=False,\n                num_workers=config['NUM_WORKERS'],\n                pin_memory=config['PIN_MEMORY'],\n                drop_last=False  # Keep all validation data\n            )\n            \n            model = ImprovedEfficientNet(\n                model_name=config['MODEL_NAME'],\n                num_classes=5,\n                dropout_rate=0.3\n            ).to(device)\n            \n            if torch.cuda.device_count() > 1:\n                print(f\"Using {torch.cuda.device_count()} GPUs!\")\n                model = nn.DataParallel(model)\n            \n            # Use Focal Loss instead of CrossEntropy\n            criterion = FocalLoss(alpha=1, gamma=2)\n            \n            optimizer = torch.optim.AdamW(\n                model.parameters(),\n                lr=config['LEARNING_RATE'],\n                weight_decay=config['WEIGHT_DECAY']\n            )\n            \n            # Cosine annealing with warm restarts\n            scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(\n                optimizer, T_0=5, T_mult=2, eta_min=1e-6\n            )\n            \n            score = train_fold(\n                fold, model, train_loader, valid_loader,\n                device, criterion, optimizer, scheduler,\n                config['NUM_EPOCHS']\n            )\n            scores.append(score)\n            \n        except Exception as e:\n            print(f\"Error in fold {fold}: {str(e)}\")\n            import traceback\n            traceback.print_exc()\n            continue\n    \n    if scores:\n        print(\"\\n\" + \"=\"*60)\n        print(\"FINAL RESULTS\")\n        print(\"=\"*60)\n        print(f\"Cross-validation scores: {[f'{s:.4f}' for s in scores]}\")\n        print(f\"Mean Kappa score: {np.mean(scores):.4f} ± {np.std(scores):.4f}\")\n    \n    # Generate submission with weighted ensemble\n    try:\n        test_df = pd.read_csv(f\"{config['DATA_PATH']}/test.csv\")\n        predictions = weighted_ensemble_predictions(config, test_df, preprocessing, device)\n        \n        submission_df = pd.DataFrame({\n            'id_code': test_df['id_code'],\n            'diagnosis': predictions\n        })\n        submission_df.to_csv('/kaggle/input/aptos2019-blindness-detection/submission.csv', index=False)\n        print(\"\\n✅ Submission file generated successfully!\")\n        \n    except Exception as e:\n        print(f\"Error generating submission: {str(e)}\")\n        import traceback\n        traceback.print_exc()\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T14:29:12.628906Z","iopub.execute_input":"2025-10-25T14:29:12.629281Z","iopub.status.idle":"2025-10-25T14:45:35.470515Z","shell.execute_reply.started":"2025-10-25T14:29:12.629246Z","shell.execute_reply":"2025-10-25T14:45:35.469089Z"}},"outputs":[],"execution_count":null}]}