{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":2819730,"sourceType":"datasetVersion","datasetId":1723812}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"'''\n!pip install torch torchvision timm albumentations==1.2.1 opencv-python scikit-learn pandas\n!pip install grad-cam\n'''","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-05T01:06:53.930811Z","iopub.execute_input":"2025-10-05T01:06:53.931504Z","iopub.status.idle":"2025-10-05T01:06:53.939142Z","shell.execute_reply.started":"2025-10-05T01:06:53.931472Z","shell.execute_reply":"2025-10-05T01:06:53.938626Z"},"_kg_hide-input":false},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!/usr/bin/env python3\n\"\"\"\nTrain script for APTOS 2019 using ConvNeXt V2 & Swin V2 backbones,\nGeM pooling, SmoothL1 regression loss, warmup+cosine LR schedule, and QWK-based model checkpointing.\n\nUsage:\n    python train_aptos_convnext_swin.py --data_csv train.csv --img_dir train_images/\n\"\"\"\n\nimport os\nimport math\nimport random\nimport argparse\nfrom pathlib import Path\nfrom collections import defaultdict\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.amp import autocast, GradScaler\n\nimport timm\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# -------------------------------\n# Config / hyperparameters\n# -------------------------------\nDATA_CSV = \"/kaggle/input/aptos2019-blindness-detection/train.csv\"  # CSV with ['id_code', 'diagnosis']\nIMG_DIR = \"/kaggle/input/aptos2019-blindness-detection/train_images\"\nMODEL_NAME = \"convnextv2_base.fcmae\"\nALT_MODEL = \"swinv2_base_window12to24_192to384\"  # Optional ensemble model\nINPUT_SIZE = 512\nBATCH_SIZE = 8\nEPOCHS = 20\nFOLD = 0\nN_SPLITS = 5\nSEED = 42\nOUTDIR = Path(\"outputs\")\nUSE_AMP = True  # Use mixed precision\nLEARNING_RATE = 3e-4\nWEIGHT_DECAY = 1e-2\nACCUM_STEPS = 1  # Gradient accumulation\n\n# === ENVIRONMENT SETUP ===\nOUTDIR.mkdir(parents=True, exist_ok=True)\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# -------------------------------\n# Reproducibility\n# -------------------------------\ndef seed_everything(seed=SEED):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\nseed_everything(SEED)\n\n# -------------------------------\n# Dataset\n# -------------------------------\nclass AptosDataset(Dataset):\n    def __init__(self, df, img_dir, transforms=None):\n        self.df = df.reset_index(drop=True)\n        self.img_dir = Path(img_dir)\n        self.transforms = transforms\n\n    def __len__(self):\n        return len(self.df)\n\n    def load_image(self, fname):\n        p = self.img_dir / fname\n        img = Image.open(p).convert(\"RGB\")\n        return np.array(img)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img = self.load_image(f\"{row['id_code']}.png\")\n        if self.transforms:\n            data = self.transforms(image=img)\n            img = data[\"image\"]\n        target = torch.tensor(row[\"diagnosis\"], dtype=torch.float32)  # regression target (0..4)\n        return img, target","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T01:07:02.615023Z","iopub.execute_input":"2025-10-05T01:07:02.615272Z","iopub.status.idle":"2025-10-05T01:07:19.241493Z","shell.execute_reply.started":"2025-10-05T01:07:02.615253Z","shell.execute_reply":"2025-10-05T01:07:19.240717Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_transforms(input_size=INPUT_SIZE):\n    train_transforms = A.Compose([\n        A.RandomResizedCrop(size=(input_size, input_size), scale=(0.8, 1.0), ratio=(0.9, 1.1)),\n        A.HorizontalFlip(),\n        A.VerticalFlip(p=0.1),\n        A.Affine(\n            scale=(0.8, 1.2),\n            translate_percent=(0.1, 0.1),\n            rotate=(-180, 180),\n            shear=(-10, 10),\n            p=0.5\n        ),\n        A.RandomBrightnessContrast(0.2, 0.2),\n        A.HueSaturationValue(10, 20, 20),\n        A.GaussNoise(gauss_noise_var_limit=(10.0, 50.0), mean=0, p=0.2),\n        A.MotionBlur(p=0.2),\n        A.CoarseDropout(\n            max_holes=8,\n            max_height=int(input_size * 0.08),\n            max_width=int(input_size * 0.08),\n            min_holes=1,\n            mask_fill_value=0,\n            p=0.3\n        ),\n        A.Normalize(mean=(0.485, 0.456, 0.406),\n                    std=(0.229, 0.224, 0.225)),\n        ToTensorV2()\n    ])\n    valid_transforms = A.Compose([\n        A.Resize(height=input_size, width=input_size),\n        A.Normalize(mean=(0.485, 0.456, 0.406),\n                    std=(0.229, 0.224, 0.225)),\n        ToTensorV2()\n    ])\n    return train_transforms, valid_transforms\n\n# -------------------------------\n# GeM Pooling\n# -------------------------------\nclass GeM(nn.Module):\n    def __init__(self, p=3.0, eps=1e-6):\n        super().__init__()\n        self.p = nn.Parameter(torch.ones(1) * p)\n        self.eps = eps\n\n    def forward(self, x):\n        # x: (B, C, H, W)\n        return F.avg_pool2d(x.clamp(min=self.eps).pow(self.p), (x.size(-2), x.size(-1))).pow(1.0 / self.p)\n\n    def __repr__(self):\n        return f\"GeM(p={self.p.data.tolist()[0]:.4f}, eps={self.eps})\"\n\n# -------------------------------\n# Model wrappers\n# -------------------------------\nclass RegressionModel(nn.Module):\n    def __init__(self, backbone_name, pretrained=True, out_features=1):\n        super().__init__()\n        # create backbone with num_classes=0 to get features\n        self.backbone = timm.create_model(backbone_name, pretrained=pretrained, num_classes=0, global_pool=\"\")\n        feature_dim = self.backbone.num_features\n        self.pool = GeM()\n        self.fc = nn.Linear(feature_dim, out_features)\n\n    def forward(self, x):\n        # timm backbone expects NCHW; returns features (B, C, H, W)\n        feat = self.backbone.forward_features(x)  # some timm models have forward_features\n        pooled = self.pool(feat).flatten(1)\n        out = self.fc(pooled).squeeze(1)\n        return out\n\n# -------------------------------\n# Loss: Smooth L1 (Huber-like)\n# -------------------------------\ndef get_loss():\n    # We keep Smooth L1 (L1 with beta). Could combine with other losses as needed.\n    return nn.SmoothL1Loss()\n\n# -------------------------------\n# LR Scheduler: Warmup + CosineAnnealing\n# -------------------------------\ndef get_scheduler(optimizer, num_warmup_epochs, max_epochs, last_epoch=-1):\n    # We'll implement linear warmup via LambdaLR followed by CosineAnnealingLR\n    def lr_lambda(current_epoch):\n        if current_epoch < num_warmup_epochs:\n            return float(current_epoch) / float(max(1, num_warmup_epochs))\n        else:\n            # cosine decay for remaining epochs\n            progress = float(current_epoch - num_warmup_epochs) / float(max(1, max_epochs - num_warmup_epochs))\n            return 0.5 * (1.0 + math.cos(math.pi * progress))\n    return torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lr_lambda, last_epoch=last_epoch)\n\n# -------------------------------\n# Metrics\n# -------------------------------\ndef qwk(true, pred):\n    # Convert predictions to nearest class via thresholds (default)\n    # Here we expect true and pred to be 1D numpy arrays\n    # Default thresholds (can be optimized later)\n    thr = [0.5, 1.5, 2.5, 3.5]\n    pred_bin = np.digitize(pred, thr)\n    return cohen_kappa_score(true, pred_bin, weights='quadratic')\n\ndef optimize_thresholds(y_true, y_pred, search_space=None):\n    # simple grid search on first threshold (like write-up)\n    if search_space is None:\n        search_space = np.linspace(0.3, 0.9, 13)\n    best_thr = [0.5, 1.5, 2.5, 3.5]\n    best_k = -1\n    for t0 in search_space:\n        thr = [t0, 1.5, 2.5, 3.5]\n        pred_bin = np.digitize(y_pred, thr)\n        k = cohen_kappa_score(y_true, pred_bin, weights='quadratic')\n        if k > best_k:\n            best_k = k\n            best_thr = thr\n    return best_thr, best_k\n\n# -------------------------------\n# Training & validation loops\n# -------------------------------\ndef train_one_epoch(model, optimizer, loader, device, scaler, loss_fn, epoch, accum_steps=1):\n    model.train()\n    running_loss = 0.0\n    optimizer.zero_grad()\n    for step, (imgs, targets) in enumerate(loader):\n        imgs = imgs.to(device, non_blocking=True)\n        targets = targets.to(device, non_blocking=True)\n\n        with autocast('cuda', enabled=USE_AMP):\n            outputs = model(imgs)\n            loss = loss_fn(outputs, targets) / accum_steps\n\n        if USE_AMP:\n            scaler.scale(loss).backward()\n            if (step + 1) % accum_steps == 0:\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()\n        else:\n            loss.backward()\n            if (step + 1) % accum_steps == 0:\n                optimizer.step()\n                optimizer.zero_grad()\n\n        running_loss += loss.item() * accum_steps\n\n    return running_loss / len(loader)\n\n@torch.no_grad()\ndef validate_one_epoch(model, loader, device):\n    model.eval()\n    preds = []\n    gts = []\n    for imgs, targets in loader:\n        imgs = imgs.to(device)\n        outputs = model(imgs)\n        preds.append(outputs.detach().cpu().numpy())\n        gts.append(targets.numpy())\n    preds = np.concatenate(preds)\n    gts = np.concatenate(gts)\n    # compute QWK using default thresholds\n    kappa = qwk(gts, preds)\n    return kappa, gts, preds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T01:07:22.422436Z","iopub.execute_input":"2025-10-05T01:07:22.423249Z","iopub.status.idle":"2025-10-05T01:07:22.441372Z","shell.execute_reply.started":"2025-10-05T01:07:22.423220Z","shell.execute_reply":"2025-10-05T01:07:22.440585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_training(df, img_dir, fold=FOLD, n_splits=N_SPLITS, seed=SEED):\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n    splits = list(skf.split(df, df[\"diagnosis\"]))\n    train_idx, valid_idx = splits[fold]\n    train_df = df.iloc[train_idx].reset_index(drop=True)\n    valid_df = df.iloc[valid_idx].reset_index(drop=True)\n\n    train_tf, valid_tf = get_transforms(INPUT_SIZE)\n    train_ds = AptosDataset(train_df, img_dir, transforms=train_tf)\n    valid_ds = AptosDataset(valid_df, img_dir, transforms=valid_tf)\n\n    train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, pin_memory=True)\n    valid_loader = DataLoader(valid_ds, batch_size=BATCH_SIZE*2, shuffle=False, num_workers=4, pin_memory=True)\n\n    # two models for ensemble option (train sequentially)\n    backbones = [MODEL_NAME, ALT_MODEL]\n    best_info = {}\n\n    for backbone_name in backbones:\n        model = RegressionModel(backbone_name, pretrained=True, out_features=1)\n        model = model.to(DEVICE)\n\n        # small trick: freeze backbone for first epoch if desired (optional)\n        # for name, p in model.backbone.named_parameters():\n        #     p.requires_grad = False\n\n        optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n        scheduler = get_scheduler(optimizer, num_warmup_epochs=2, max_epochs=EPOCHS)\n        loss_fn = get_loss()\n        scaler_local = GradScaler('cuda') if USE_AMP else None\n\n        best_kappa = -999\n        best_epoch = -1\n        best_model_path = OUTDIR / f\"best_{Path(backbone_name).name}_fold{fold}.pt\"\n        \n        print(f\"\\n=== Training backbone: {backbone_name} ===\")\n        for epoch in range(EPOCHS):\n            train_loss = train_one_epoch(model, optimizer, train_loader, DEVICE, scaler_local, loss_fn, epoch, accum_steps=ACCUM_STEPS)\n            scheduler.step(epoch)  # calls the lambda\n            val_kappa, y_val, y_pred = validate_one_epoch(model, valid_loader, DEVICE)\n            print(f\"[{backbone_name}] Epoch {epoch+1}/{EPOCHS} | TrainLoss {train_loss:.4f} | ValQWK {val_kappa:.5f} | LR {optimizer.param_groups[0]['lr']:.2e}\")\n\n            # save best by QWK\n            if val_kappa > best_kappa:\n                best_kappa = val_kappa\n                best_epoch = epoch\n                torch.save({\n                    \"model_state\": model.state_dict(),\n                    \"optimizer_state\": optimizer.state_dict(),\n                    \"scaler_state\": scaler_local.state_dict() if scaler_local else None,\n                    \"val_kappa\": val_kappa,\n                    \"epoch\": epoch,\n                    \"backbone\": backbone_name\n                }, best_model_path)\n                # also save the raw predictions for threshold tuning\n                np.save(OUTDIR / f\"val_pred_{Path(backbone_name).name}_fold{fold}.npy\", y_pred)\n                np.save(OUTDIR / f\"val_true_{Path(backbone_name).name}_fold{fold}.npy\", y_val)\n\n        print(f\"Best val QWK for {backbone_name}: {best_kappa:.5f} at epoch {best_epoch}\")\n        best_info[backbone_name] = {\"best_kappa\": best_kappa, \"best_epoch\": best_epoch, \"model_path\": str(best_model_path)}\n\n    # After training both backbones, we can ensemble their saved predictions on the validation set\n    # and find optimized thresholds.\n    preds_all = []\n    truths = None\n    for backbone_name in backbones:\n        pred_path = OUTDIR / f\"val_pred_{Path(backbone_name).name}_fold{fold}.npy\"\n        true_path = OUTDIR / f\"val_true_{Path(backbone_name).name}_fold{fold}.npy\"\n        if pred_path.exists():\n            p = np.load(pred_path)\n            preds_all.append(p)\n            if truths is None:\n                truths = np.load(true_path)\n    if len(preds_all) > 0:\n        ensemble_pred = np.mean(np.vstack(preds_all), axis=0)\n        best_thr, best_k = optimize_thresholds(truths, ensemble_pred)\n        print(f\"Ensemble validation QWK after threshold tuning: {best_k:.5f} with thresholds {best_thr}\")\n\n    return best_info","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T01:07:27.928799Z","iopub.execute_input":"2025-10-05T01:07:27.929384Z","iopub.status.idle":"2025-10-05T01:07:27.939760Z","shell.execute_reply.started":"2025-10-05T01:07:27.929358Z","shell.execute_reply":"2025-10-05T01:07:27.938941Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"CUDA available:\", torch.cuda.is_available(), \" Device:\", DEVICE)\ndf = pd.read_csv(DATA_CSV)\n# Ensure file ids are strings (APTOS filenames)\ndf['id_code'] = df['id_code'].astype(str)\n# If diagnosis not numeric or has stray values, coerce\ndf['diagnosis'] = pd.to_numeric(df['diagnosis'], errors='coerce').fillna(0).astype(int)\nprint(\"Loaded CSV:\", df.shape)\n\n# Run training for the requested fold\ninfo = run_training(df, IMG_DIR, fold=FOLD, n_splits=N_SPLITS, seed=SEED)\nprint(\"Training finished. Models info:\", info)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-05T01:07:35.853762Z","iopub.execute_input":"2025-10-05T01:07:35.854309Z","execution_failed":"2025-10-05T05:49:45.156Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import cohen_kappa_score, f1_score, roc_auc_score, confusion_matrix, classification_report\n\nval_qwk = cohen_kappa_score(y_true, y_pred, weights='quadratic')\nf1 = f1_score(y_true, y_pred, average='macro')\nprint(confusion_matrix(y_true, y_pred))\nprint(classification_report(y_true, y_pred))","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-04T17:41:27.286Z"}},"outputs":[],"execution_count":null}]}