{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":532013,"sourceType":"datasetVersion","datasetId":253160}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-30T20:37:19.552452Z","iopub.execute_input":"2025-11-30T20:37:19.552766Z","iopub.status.idle":"2025-11-30T20:37:23.027140Z","shell.execute_reply.started":"2025-11-30T20:37:19.552741Z","shell.execute_reply":"2025-11-30T20:37:23.026367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport time\nimport random\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\nfrom torchvision import transforms\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score, accuracy_score, cohen_kappa_score\nfrom tqdm.auto import tqdm\nimport timm\nimport gc\nfrom PIL import Image\n\n# --- CONFIGURATION ---\nCONFIG = {\n    'seed': 42,\n    'img_size': 256,       # Must be 256 for Swin V2 Tiny\n    'batch_size': 16,      # Fits in 16GB VRAM\n    'accum_steps': 2,      # Effective batch size = 32\n    'num_classes': 5,\n    'device': torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n}\n\n# --- PATHS ---\n# 2015 Data (Pre-training)\nDATA_2015_PATH = '../input/resized-2015-2019-blindness-detection-images/resized train 15'\nLABEL_2015_PATH = '../input/resized-2015-2019-blindness-detection-images/labels/trainLabels15.csv'\n\n# 2019 Data (Fine-tuning)\nDATA_2019_PATH = '../input/aptos2019-blindness-detection/train_images'\nLABEL_2019_PATH = '../input/aptos2019-blindness-detection/train.csv'\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_everything(CONFIG['seed'])\nprint(f\"Device: {CONFIG['device']}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T20:40:13.690353Z","iopub.execute_input":"2025-11-30T20:40:13.690960Z","iopub.status.idle":"2025-11-30T20:40:13.700172Z","shell.execute_reply.started":"2025-11-30T20:40:13.690922Z","shell.execute_reply":"2025-11-30T20:40:13.699458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class h_sigmoid(nn.Module):\n    def __init__(self, inplace=True):\n        super(h_sigmoid, self).__init__()\n        self.relu = nn.ReLU6(inplace=inplace)\n    def forward(self, x):\n        return self.relu(x + 3) / 6\n\nclass h_swish(nn.Module):\n    def __init__(self, inplace=True):\n        super(h_swish, self).__init__()\n        self.sigmoid = h_sigmoid(inplace=inplace)\n    def forward(self, x):\n        return x * self.sigmoid(x)\n\nclass CoordinateAttention(nn.Module):\n    \"\"\"Paper Eq 10: Coordinate Attention mechanism\"\"\"\n    def __init__(self, inp, reduction=32):\n        super(CoordinateAttention, self).__init__()\n        self.pool_h = nn.AdaptiveAvgPool2d((None, 1))\n        self.pool_w = nn.AdaptiveAvgPool2d((1, None))\n\n        mip = max(8, inp // reduction)\n\n        self.conv1 = nn.Conv2d(inp, mip, kernel_size=1, stride=1, padding=0)\n        self.bn1 = nn.BatchNorm2d(mip)\n        self.act = h_swish()\n        \n        self.conv_h = nn.Conv2d(mip, inp, kernel_size=1, stride=1, padding=0)\n        self.conv_w = nn.Conv2d(mip, inp, kernel_size=1, stride=1, padding=0)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        identity = x\n        n, c, h, w = x.size()\n        x_h = self.pool_h(x)\n        x_w = self.pool_w(x).permute(0, 1, 3, 2)\n\n        y = torch.cat([x_h, x_w], dim=2)\n        y = self.conv1(y)\n        y = self.bn1(y)\n        y = self.act(y) \n        \n        x_h, x_w = torch.split(y, [h, w], dim=2)\n        x_w = x_w.permute(0, 1, 3, 2)\n\n        a_h = self.sigmoid(self.conv_h(x_h))\n        a_w = self.sigmoid(self.conv_w(x_w))\n\n        return identity * a_h * a_w","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T20:40:17.932108Z","iopub.execute_input":"2025-11-30T20:40:17.932699Z","iopub.status.idle":"2025-11-30T20:40:17.941677Z","shell.execute_reply.started":"2025-11-30T20:40:17.932673Z","shell.execute_reply":"2025-11-30T20:40:17.940944Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class HybridModel(nn.Module):\n    def __init__(self, num_classes=5):\n        super(HybridModel, self).__init__()\n        \n        # --- Branch 1: EfficientNet-B3 ---\n        self.effnet = timm.create_model('efficientnet_b3', pretrained=True)\n        self.eff_n_features = self.effnet.classifier.in_features\n        self.effnet.classifier = nn.Identity() \n        self.effnet.global_pool = nn.Identity() \n        \n        # MHSA for EfficientNet\n        self.eff_mhsa = nn.MultiheadAttention(embed_dim=self.eff_n_features, num_heads=8, batch_first=True)\n\n        # --- Branch 2: Swin Transformer V2 ---\n        self.swin = timm.create_model('swinv2_tiny_window8_256', pretrained=True)\n        self.swin_n_features = self.swin.head.in_features\n        self.swin.head = nn.Identity() \n        \n        # Coordinate Attention for Swin\n        self.swin_coord_att = CoordinateAttention(inp=self.swin_n_features)\n        \n        # --- Fusion & Classifier ---\n        fusion_dim = self.eff_n_features + self.swin_n_features\n        self.classifier = nn.Sequential(\n            nn.Dropout(0.2), \n            nn.Linear(fusion_dim, 512),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(512, num_classes)\n        )\n\n    def forward(self, x):\n        # --- EfficientNet Forward ---\n        eff_feat = self.effnet.forward_features(x) # (B, C, H, W)\n        \n        # Reshape for MHSA: (B, C, H, W) -> (B, H*W, C)\n        b, c, h, w = eff_feat.shape\n        eff_tokens = eff_feat.flatten(2).transpose(1, 2)\n        \n        # Apply MHSA\n        eff_att, _ = self.eff_mhsa(eff_tokens, eff_tokens, eff_tokens)\n        eff_out = torch.mean(eff_att, dim=1) \n        \n        # --- Swin Forward ---\n        swin_feat = self.swin.forward_features(x) # (B, H, W, C)\n        \n        # Fix Dimensions for Coordinate Attention: (B, H, W, C) -> (B, C, H, W)\n        if swin_feat.dim() == 4:\n            swin_feat = swin_feat.permute(0, 3, 1, 2)\n        \n        # Apply Coordinate Attention\n        swin_att = self.swin_coord_att(swin_feat)\n        \n        # Global Pooling\n        swin_out = torch.mean(swin_att.flatten(2), dim=2)\n\n        # --- Fusion ---\n        combined = torch.cat((eff_out, swin_out), dim=1)\n        output = self.classifier(combined)\n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T20:40:21.440227Z","iopub.execute_input":"2025-11-30T20:40:21.440860Z","iopub.status.idle":"2025-11-30T20:40:21.448310Z","shell.execute_reply.started":"2025-11-30T20:40:21.440834Z","shell.execute_reply":"2025-11-30T20:40:21.447678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class DRDataset(Dataset):\n    def __init__(self, df, base_path, extension=\".png\", transform=None):\n        self.df = df\n        self.base_path = base_path\n        self.transform = transform\n        self.extension = extension\n        \n        # Sanity Check\n        if len(df) > 0:\n            sample_id = df.iloc[0, 0] if 'image' in df.columns else df.iloc[0, 0]\n            # Verify if at least one file exists\n            found = False\n            for ext in [extension, '.jpg', '.jpeg', '.png']:\n                if os.path.exists(os.path.join(base_path, f\"{sample_id}{ext}\")):\n                    found = True\n                    break\n            if not found:\n                print(f\"⚠️ WARNING: Could not find sample image: {sample_id} in {base_path}\")\n                print(f\"⚠️ Expected path: {os.path.join(base_path, sample_id + extension)}\")\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        # Handle column name differences\n        if 'image' in self.df.columns:\n            id_code = self.df.loc[idx, 'image']\n            label = self.df.loc[idx, 'level']\n        else:\n            id_code = self.df.loc[idx, 'id_code']\n            label = self.df.loc[idx, 'diagnosis']\n            \n        # Try loading\n        image = None\n        # Try preferred extension first\n        try:\n            p = os.path.join(self.base_path, f\"{id_code}{self.extension}\")\n            if os.path.exists(p):\n                image = cv2.imread(p)\n        except: pass\n        \n        # Fallback to other extensions\n        if image is None:\n            for ext in ['.jpg', '.jpeg', '.png']:\n                try:\n                    p = os.path.join(self.base_path, f\"{id_code}{ext}\")\n                    if os.path.exists(p):\n                        image = cv2.imread(p)\n                        if image is not None: break\n                except: pass\n\n        # Safety Net: Black Image\n        if image is None:\n            image = np.zeros((CONFIG['img_size'], CONFIG['img_size'], 3), dtype=np.uint8)\n        else:\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        image = cv2.resize(image, (CONFIG['img_size'], CONFIG['img_size']))\n        img = Image.fromarray(image)\n        \n        if self.transform:\n            img = self.transform(img)\n            \n        return img, torch.tensor(label, dtype=torch.long)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T20:40:33.418329Z","iopub.execute_input":"2025-11-30T20:40:33.418879Z","iopub.status.idle":"2025-11-30T20:40:33.428081Z","shell.execute_reply.started":"2025-11-30T20:40:33.418855Z","shell.execute_reply":"2025-11-30T20:40:33.427351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_full_training():\n    \n    # --- Transforms ---\n    train_transforms = transforms.Compose([\n        transforms.RandomRotation(180),\n        transforms.RandomHorizontalFlip(),\n        transforms.RandomVerticalFlip(),\n        transforms.ColorJitter(brightness=0.15, contrast=0.15),\n        transforms.RandomResizedCrop(CONFIG['img_size'], scale=(0.85, 1.0)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n    ])\n    \n    val_transforms = transforms.Compose([\n        transforms.Resize((CONFIG['img_size'], CONFIG['img_size'])),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n    ])\n\n    # Initialize Model & Tools\n    model = HybridModel(num_classes=5).to(CONFIG['device'])\n    scaler = torch.amp.GradScaler('cuda')\n    criterion = nn.CrossEntropyLoss()\n\n    # ====================================================\n    # STAGE 1: BALANCED PRE-TRAINING (2015 DATASET)\n    # ====================================================\n    print(\"\\n\" + \"=\"*60)\n    print(\"STAGE 1: Pre-training on 2015 Dataset (15 Epochs)\")\n    print(\"=\"*60)\n    \n    df_2015 = pd.read_csv(LABEL_2015_PATH)\n    # Filter valid labels\n    df_2015 = df_2015[df_2015['level'].isin([0,1,2,3,4])].reset_index(drop=True)\n    \n    # Create Weighted Sampler for 2015\n    class_counts = df_2015['level'].value_counts().sort_index().values\n    weights = 1. / class_counts\n    samples_weights = torch.DoubleTensor([weights[t] for t in df_2015['level']])\n    # Sample 15,000 images per epoch\n    sampler_2015 = WeightedRandomSampler(samples_weights, num_samples=15000, replacement=True)\n    \n    ds_2015 = DRDataset(df_2015, DATA_2015_PATH, extension=\".jpg\", transform=train_transforms)\n    dl_2015 = DataLoader(ds_2015, batch_size=CONFIG['batch_size'], sampler=sampler_2015, \n                         num_workers=2, pin_memory=True, persistent_workers=True)\n    \n    optimizer = optim.AdamW(model.parameters(), lr=3e-4, weight_decay=1e-4)\n    \n    for epoch in range(15):\n        model.train()\n        running_loss = 0.0\n        train_preds, train_targets = [], []\n        \n        pbar = tqdm(dl_2015, desc=f\"Pre-train {epoch+1}/15\")\n        \n        for batch_idx, (images, labels) in enumerate(pbar):\n            images, labels = images.to(CONFIG['device']), labels.to(CONFIG['device'])\n            \n            with torch.amp.autocast('cuda'):\n                outputs = model(images)\n                loss = criterion(outputs, labels)\n                loss = loss / CONFIG['accum_steps']\n            \n            scaler.scale(loss).backward()\n            \n            if ((batch_idx + 1) % CONFIG['accum_steps'] == 0):\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()\n            \n            # Metrics tracking\n            current_loss = loss.item() * CONFIG['accum_steps']\n            running_loss += current_loss\n            _, predicted = torch.max(outputs.data, 1)\n            train_preds.extend(predicted.detach().cpu().numpy())\n            train_targets.extend(labels.detach().cpu().numpy())\n            \n            pbar.set_postfix({'loss': f\"{current_loss:.4f}\"})\n        \n        # Calculate Training Metrics\n        train_loss = running_loss / len(dl_2015)\n        train_acc = accuracy_score(train_targets, train_preds)\n        train_f1 = f1_score(train_targets, train_preds, average='weighted')\n        train_kappa = cohen_kappa_score(train_targets, train_preds, weights='quadratic')\n        \n        print(f\"Epoch {epoch+1} Results:\")\n        print(f\"  [Train] Loss: {train_loss:.4f} | Acc: {train_acc:.4f} | F1: {train_f1:.4f} | Kappa: {train_kappa:.4f}\")\n            \n    print(\"Saving Pre-trained weights...\")\n    torch.save(model.state_dict(), 'pretrained_2015.pth')\n\n    # ====================================================\n    # STAGE 2: FINE-TUNING (APTOS 2019)\n    # ====================================================\n    print(\"\\n\" + \"=\"*60)\n    print(\"STAGE 2: Fine-tuning on APTOS 2019 (30 Epochs)\")\n    print(\"=\"*60)\n    \n    df_2019 = pd.read_csv(LABEL_2019_PATH)\n    train_sub, val_sub = train_test_split(df_2019, test_size=0.2, stratify=df_2019['diagnosis'], random_state=CONFIG['seed'])\n    train_sub = train_sub.reset_index(drop=True)\n    val_sub = val_sub.reset_index(drop=True)\n    \n    # Weighted Sampler for 2019\n    class_counts_19 = train_sub['diagnosis'].value_counts().sort_index().values\n    weights_19 = 1. / class_counts_19\n    samples_weights_19 = torch.DoubleTensor([weights_19[t] for t in train_sub['diagnosis']])\n    sampler_19 = WeightedRandomSampler(samples_weights_19, len(samples_weights_19))\n    \n    ds_train = DRDataset(train_sub, DATA_2019_PATH, extension=\".png\", transform=train_transforms)\n    ds_val = DRDataset(val_sub, DATA_2019_PATH, extension=\".png\", transform=val_transforms)\n    \n    dl_train = DataLoader(ds_train, batch_size=CONFIG['batch_size'], sampler=sampler_19, \n                          num_workers=2, pin_memory=True, persistent_workers=True)\n    dl_val = DataLoader(ds_val, batch_size=CONFIG['batch_size'], shuffle=False, \n                        num_workers=2, pin_memory=True, persistent_workers=True)\n    \n    # Lower LR for fine-tuning\n    optimizer = optim.AdamW(model.parameters(), lr=1e-4, weight_decay=1e-4)\n    # Scheduler: Reduces LR if F1 score stagnates\n    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=3)\n    \n    best_f1 = 0.0\n    \n    for epoch in range(30):\n        model.train()\n        running_loss = 0.0\n        train_preds, train_targets = [], []\n        \n        pbar = tqdm(dl_train, desc=f\"Fine-tune {epoch+1}/30\")\n        optimizer.zero_grad()\n        \n        for batch_idx, (images, labels) in enumerate(pbar):\n            images, labels = images.to(CONFIG['device']), labels.to(CONFIG['device'])\n            \n            with torch.amp.autocast('cuda'):\n                outputs = model(images)\n                loss = criterion(outputs, labels)\n                loss = loss / CONFIG['accum_steps']\n                \n            scaler.scale(loss).backward()\n            \n            if ((batch_idx + 1) % CONFIG['accum_steps'] == 0):\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()\n            \n            # Track training metrics\n            current_loss = loss.item() * CONFIG['accum_steps']\n            running_loss += current_loss\n            _, predicted = torch.max(outputs.data, 1)\n            train_preds.extend(predicted.detach().cpu().numpy())\n            train_targets.extend(labels.detach().cpu().numpy())\n            \n            pbar.set_postfix({'loss': f\"{current_loss:.4f}\"})\n            \n        # Calculate Training Stats\n        train_loss = running_loss / len(dl_train)\n        train_acc = accuracy_score(train_targets, train_preds)\n        train_f1 = f1_score(train_targets, train_preds, average='weighted')\n        train_kappa = cohen_kappa_score(train_targets, train_preds, weights='quadratic')\n\n        # Validation with Test Time Augmentation (TTA)\n        model.eval()\n        val_preds, val_targets = [], []\n        with torch.no_grad():\n            for images, labels in dl_val:\n                images = images.to(CONFIG['device'])\n                with torch.amp.autocast('cuda'):\n                    # TTA: Predict twice (normal + flip) and average\n                    out1 = model(images)\n                    out2 = model(torch.flip(images, dims=[3]))\n                    outputs = (out1 + out2) / 2.0\n                    \n                _, predicted = torch.max(outputs.data, 1)\n                val_preds.extend(predicted.cpu().numpy())\n                val_targets.extend(labels.cpu().numpy())\n                \n        val_acc = accuracy_score(val_targets, val_preds)\n        val_f1 = f1_score(val_targets, val_preds, average='weighted')\n        val_kappa = cohen_kappa_score(val_targets, val_preds, weights='quadratic')\n        \n        print(f\"Epoch {epoch+1} Summary:\")\n        print(f\"  [Train] Loss: {train_loss:.4f} | Acc: {train_acc:.4f} | F1: {train_f1:.4f} | Kappa: {train_kappa:.4f}\")\n        print(f\"  [Valid] ---     | Acc: {val_acc:.4f} | F1: {val_f1:.4f} | Kappa: {val_kappa:.4f}\")\n        \n        scheduler.step(val_f1)\n        \n        if val_f1 > best_f1:\n            best_f1 = val_f1\n            torch.save(model.state_dict(), 'final_hybrid_model.pth')\n            print(\"  [+] Best Model Saved\")\n\n# Run the complete pipeline\nrun_full_training()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T20:42:47.428699Z","iopub.execute_input":"2025-11-30T20:42:47.428997Z"}},"outputs":[],"execution_count":null}]}