{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\nDiabetic Retinopathy Screening — ResNet50 Classifier\nSIH PS: Explainable AI for Diabetic Retinopathy Screening in Rural India\n\nRun this directly as a Kaggle Notebook (GPU T4 x2 or P100 enabled).\nAttach these datasets to your notebook before running:\n  1. APTOS 2019 Blindness Detection  -> \"aptos2019-blindness-detection\"\n  2. Any Messidor dataset mirror     -> e.g. \"mariaherrerot/messidor2preprocess\"\n     or \"andrewmvd/diabetic-retinopathy-messidor\" (any mirror with an image folder\n     + a CSV containing an id/filename column and a grade column works — the\n     loader below auto-detects column names).\n\nIf your Messidor dataset's folder/column names differ, just edit MESSIDOR_DIR\nand the two column-name lists below (IMG_COL_CANDIDATES / LABEL_COL_CANDIDATES).\n\"\"\"\n\nimport os, glob, random, time, copy\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score, confusion_matrix\nimport torchvision\nfrom torchvision import transforms\n\n# -----------------------------------------------------------------------------\n# 0. CONFIG — edit these paths to match the datasets you attach on Kaggle\n# -----------------------------------------------------------------------------\nAPTOS_DIR      = \"/kaggle/input/competitions/aptos2019-blindness-detection\"\nMESSIDOR_DIR   = \"/kaggle/input/datasets/parikshakaur/messidor\"   # <-- change to your slug\n\nIMG_SIZE       = 224\nBATCH_SIZE     = 32\nNUM_EPOCHS     = 20\nLR             = 3e-4\nNUM_CLASSES    = 5          # 0=No DR,1=Mild,2=Moderate,3=Severe,4=Proliferative\nSEED           = 42\nVAL_SPLIT      = 0.15\nOUT_DIR        = \"/kaggle/working\"\n\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Device:\", device)\n\n# -----------------------------------------------------------------------------\n# 1. BUILD A UNIFIED (filepath, label) TABLE FROM BOTH DATASETS\n# -----------------------------------------------------------------------------\nrecords = []  # list of (filepath, label, source)\n\n# --- APTOS ---\naptos_csv = os.path.join(APTOS_DIR, \"train.csv\")\nif os.path.exists(aptos_csv):\n    df = pd.read_csv(aptos_csv)\n    img_dir_candidates = [\n        os.path.join(APTOS_DIR, \"train_images\"),\n        os.path.join(APTOS_DIR, \"train_images\", \"train_images\"),\n    ]\n    img_dir = next((d for d in img_dir_candidates if os.path.isdir(d)), img_dir_candidates[0])\n    for _, row in df.iterrows():\n        fp = os.path.join(img_dir, f\"{row['id_code']}.png\")\n        if os.path.exists(fp):\n            records.append((fp, int(row[\"diagnosis\"]), \"aptos\"))\n    print(f\"APTOS: {len(df)} rows in csv, {sum(r[2]=='aptos' for r in records)} images found\")\nelse:\n    print(\"WARNING: APTOS train.csv not found at\", aptos_csv)\n\n# --- Messidor (auto-detect csv + column names) ---\nIMG_COL_CANDIDATES   = [\"id_code\", \"image\", \"image_id\", \"Image name\", \"filename\", \"img_path\"]\nLABEL_COL_CANDIDATES = [\"diagnosis\", \"grade\", \"Retinopathy grade\", \"adjudicated_dr_grade\", \"label\"]\n\nmessidor_csvs = glob.glob(os.path.join(MESSIDOR_DIR, \"**\", \"*.csv\"), recursive=True)\nif messidor_csvs:\n    mdf = pd.read_csv(messidor_csvs[0])\n    img_col   = next((c for c in IMG_COL_CANDIDATES if c in mdf.columns), None)\n    label_col = next((c for c in LABEL_COL_CANDIDATES if c in mdf.columns), None)\n    if img_col is None or label_col is None:\n        print(\"WARNING: could not auto-detect Messidor columns. Found columns:\", list(mdf.columns))\n        print(\"Edit IMG_COL_CANDIDATES / LABEL_COL_CANDIDATES to match, then rerun this cell.\")\n    else:\n        all_imgs = glob.glob(os.path.join(MESSIDOR_DIR, \"**\", \"*.*\"), recursive=True)\n        img_lookup = {os.path.splitext(os.path.basename(p))[0]: p for p in all_imgs\n                      if p.lower().endswith((\".png\", \".jpg\", \".jpeg\", \".tif\", \".tiff\"))}\n        found = 0\n        for _, row in mdf.iterrows():\n            key = os.path.splitext(str(row[img_col]))[0]\n            fp = img_lookup.get(key)\n            if fp is not None and pd.notna(row[label_col]):\n                # Messidor grades are 0-3 -> map directly onto APTOS's 0-3 (no class-4 examples from here)\n                label = int(row[label_col])\n                if 0 <= label <= 4:\n                    records.append((fp, label, \"messidor\"))\n                    found += 1\n        print(f\"Messidor: {len(mdf)} rows in csv, {found} images matched\")\nelse:\n    print(\"WARNING: no CSV found under\", MESSIDOR_DIR, \"- Messidor skipped\")\n\nassert len(records) > 0, \"No images loaded — check your dataset paths in the CONFIG section.\"\ndata_df = pd.DataFrame(records, columns=[\"filepath\", \"label\", \"source\"])\nprint(\"\\nTotal combined images:\", len(data_df))\nprint(data_df[\"label\"].value_counts().sort_index())\nprint(data_df[\"source\"].value_counts())\n\n# -----------------------------------------------------------------------------\n# 2. TRAIN / VAL SPLIT (stratified on label)\n# -----------------------------------------------------------------------------\ntrain_df, val_df = train_test_split(\n    data_df, test_size=VAL_SPLIT, stratify=data_df[\"label\"], random_state=SEED\n)\ntrain_df = train_df.reset_index(drop=True)\nval_df   = val_df.reset_index(drop=True)\nprint(f\"\\nTrain: {len(train_df)}  Val: {len(val_df)}\")\n\n# -----------------------------------------------------------------------------\n# 3. PREPROCESSING — Ben Graham style crop + resize, common for DR fundus images\n# -----------------------------------------------------------------------------\ndef crop_and_resize(img, size=IMG_SIZE):\n    \"\"\"Crop black borders around the fundus circle, then resize.\"\"\"\n    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n    mask = gray > 7\n    if mask.any():\n        coords = np.argwhere(mask)\n        y0, x0 = coords.min(axis=0)\n        y1, x1 = coords.max(axis=0) + 1\n        img = img[y0:y1, x0:x1]\n    img = cv2.resize(img, (size, size), interpolation=cv2.INTER_AREA)\n    return img\n\nIMAGENET_MEAN = [0.485, 0.456, 0.406]\nIMAGENET_STD  = [0.229, 0.224, 0.225]\n\ntrain_tf = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.RandomRotation(20),\n    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.1),\n    transforms.ToTensor(),\n    transforms.Normalize(IMAGENET_MEAN, IMAGENET_STD),\n])\nval_tf = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.ToTensor(),\n    transforms.Normalize(IMAGENET_MEAN, IMAGENET_STD),\n])\n\nclass DRDataset(Dataset):\n    def __init__(self, df, transform):\n        self.df = df.reset_index(drop=True)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img = cv2.imread(row[\"filepath\"])\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = crop_and_resize(img)\n        img = self.transform(img)\n        return img, torch.tensor(row[\"label\"], dtype=torch.long)\n\ntrain_ds = DRDataset(train_df, train_tf)\nval_ds   = DRDataset(val_df, val_tf)\n\n# Weighted sampler to counter class imbalance (DR datasets are heavily skewed to class 0)\nclass_counts = train_df[\"label\"].value_counts().sort_index()\nclass_weights_sample = 1.0 / class_counts\nsample_weights = train_df[\"label\"].map(class_weights_sample).values\nsampler = WeightedRandomSampler(sample_weights, num_samples=len(sample_weights), replacement=True)\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, sampler=sampler, num_workers=2, pin_memory=True)\nval_loader   = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=2, pin_memory=True)\n\n# -----------------------------------------------------------------------------\n# 4. MODEL — ResNet50, ImageNet pretrained, fine-tuned head + last block\n# -----------------------------------------------------------------------------\nmodel = torchvision.models.resnet50(weights=torchvision.models.ResNet50_Weights.IMAGENET1K_V2)\n\n# Freeze early layers, fine-tune layer4 + fc (fast + avoids overfitting on limited DR data)\nfor name, param in model.named_parameters():\n    if not (name.startswith(\"layer4\") or name.startswith(\"fc\")):\n        param.requires_grad = False\n\nmodel.fc = nn.Sequential(\n    nn.Dropout(0.3),\n    nn.Linear(model.fc.in_features, NUM_CLASSES)\n)\nmodel = model.to(device)\n\n# Class-weighted loss (inverse frequency) for the remaining imbalance\nclass_weights = torch.tensor(\n    (1.0 / class_counts.reindex(range(NUM_CLASSES), fill_value=1)).values, dtype=torch.float32\n).to(device)\ncriterion = nn.CrossEntropyLoss(weight=class_weights)\n\noptimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=LR, weight_decay=1e-4)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=NUM_EPOCHS)\nscaler = torch.cuda.amp.GradScaler()\n\n# -----------------------------------------------------------------------------\n# 5. TRAIN LOOP — tracks Quadratic Weighted Kappa (the standard DR grading metric)\n# -----------------------------------------------------------------------------\ndef run_epoch(loader, train_mode):\n    model.train() if train_mode else model.eval()\n    total_loss, all_preds, all_labels = 0.0, [], []\n    for imgs, labels in loader:\n        imgs, labels = imgs.to(device, non_blocking=True), labels.to(device, non_blocking=True)\n        with torch.set_grad_enabled(train_mode):\n            with torch.cuda.amp.autocast():\n                outputs = model(imgs)\n                loss = criterion(outputs, labels)\n            if train_mode:\n                optimizer.zero_grad()\n                scaler.scale(loss).backward()\n                scaler.step(optimizer)\n                scaler.update()\n        total_loss += loss.item() * imgs.size(0)\n        all_preds.extend(outputs.argmax(1).detach().cpu().numpy())\n        all_labels.extend(labels.detach().cpu().numpy())\n    avg_loss = total_loss / len(loader.dataset)\n    acc = accuracy_score(all_labels, all_preds)\n    qwk = cohen_kappa_score(all_labels, all_preds, weights=\"quadratic\")\n    return avg_loss, acc, qwk\n\nbest_qwk = -1.0\nbest_state = None\nhistory = []\n\nfor epoch in range(1, NUM_EPOCHS + 1):\n    t0 = time.time()\n    tr_loss, tr_acc, tr_qwk = run_epoch(train_loader, train_mode=True)\n    val_loss, val_acc, val_qwk = run_epoch(val_loader, train_mode=False)\n    scheduler.step()\n\n    print(f\"Epoch {epoch:02d}/{NUM_EPOCHS} | \"\n          f\"train loss {tr_loss:.4f} acc {tr_acc:.4f} qwk {tr_qwk:.4f} | \"\n          f\"val loss {val_loss:.4f} acc {val_acc:.4f} qwk {val_qwk:.4f} | \"\n          f\"{time.time()-t0:.1f}s\")\n    history.append([epoch, tr_loss, tr_acc, tr_qwk, val_loss, val_acc, val_qwk])\n\n    if val_qwk > best_qwk:\n        best_qwk = val_qwk\n        best_state = copy.deepcopy(model.state_dict())\n        torch.save(best_state, os.path.join(OUT_DIR, \"best_resnet50_dr.pth\"))\n        print(f\"  -> new best model saved (val QWK={best_qwk:.4f})\")\n\npd.DataFrame(history, columns=[\"epoch\", \"train_loss\", \"train_acc\", \"train_qwk\",\n                                \"val_loss\", \"val_acc\", \"val_qwk\"]).to_csv(\n    os.path.join(OUT_DIR, \"training_history.csv\"), index=False)\n\n# -----------------------------------------------------------------------------\n# 6. FINAL EVAL — confusion matrix on val set with best checkpoint\n# -----------------------------------------------------------------------------\nmodel.load_state_dict(best_state)\nmodel.eval()\nall_preds, all_labels = [], []\nwith torch.no_grad():\n    for imgs, labels in val_loader:\n        imgs = imgs.to(device)\n        outputs = model(imgs)\n        all_preds.extend(outputs.argmax(1).cpu().numpy())\n        all_labels.extend(labels.numpy())\n\nprint(\"\\nBest val QWK:\", best_qwk)\nprint(\"Confusion matrix (rows=true, cols=pred):\")\nprint(confusion_matrix(all_labels, all_preds))\n\n# -----------------------------------------------------------------------------\n# 7. GRAD-CAM — for the \"Explainable AI\" half of your PS\n#    Produces a heatmap over the fundus image showing what the model attended to.\n# -----------------------------------------------------------------------------\nimport matplotlib.pyplot as plt\n\nclass GradCAM:\n    def __init__(self, model, target_layer):\n        self.model = model\n        self.gradients = None\n        self.activations = None\n        target_layer.register_forward_hook(self._save_activation)\n        target_layer.register_full_backward_hook(self._save_gradient)\n\n    def _save_activation(self, module, input, output):\n        self.activations = output.detach()\n\n    def _save_gradient(self, module, grad_input, grad_output):\n        self.gradients = grad_output[0].detach()\n\n    def generate(self, input_tensor, class_idx=None):\n        self.model.eval()\n        output = self.model(input_tensor)\n        if class_idx is None:\n            class_idx = output.argmax(dim=1).item()\n        self.model.zero_grad()\n        output[0, class_idx].backward()\n\n        weights = self.gradients.mean(dim=(2, 3), keepdim=True)\n        cam = (weights * self.activations).sum(dim=1, keepdim=True)\n        cam = torch.relu(cam)\n        cam = cam.squeeze().cpu().numpy()\n        cam = (cam - cam.min()) / (cam.max() - cam.min() + 1e-8)\n        return cam, class_idx\n\ndef show_gradcam(filepath, model):\n    img = cv2.imread(filepath)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = crop_and_resize(img)\n    tensor = val_tf(img).unsqueeze(0).to(device)\n\n    cam_extractor = GradCAM(model, model.layer4[-1])\n    cam, pred_class = cam_extractor.generate(tensor)\n    cam_resized = cv2.resize(cam, (IMG_SIZE, IMG_SIZE))\n    heatmap = cv2.applyColorMap(np.uint8(255 * cam_resized), cv2.COLORMAP_JET)\n    overlay = cv2.addWeighted(img, 0.6, heatmap, 0.4, 0)\n\n    fig, axes = plt.subplots(1, 2, figsize=(8, 4))\n    axes[0].imshow(img); axes[0].set_title(\"Input\"); axes[0].axis(\"off\")\n    axes[1].imshow(overlay); axes[1].set_title(f\"Grad-CAM (pred={pred_class})\"); axes[1].axis(\"off\")\n    plt.tight_layout()\n    plt.savefig(os.path.join(OUT_DIR, \"gradcam_example.png\"))\n    plt.show()\n\n# Example usage on one validation image — swap in any filepath you like\nsample_fp = val_df.iloc[0][\"filepath\"]\nshow_gradcam(sample_fp, model)\n\nprint(\"\\nDone. Best model: /kaggle/working/best_resnet50_dr.pth\")\nprint(\"Training curve CSV: /kaggle/working/training_history.csv\")\nprint(\"Sample Grad-CAM: /kaggle/working/gradcam_example.png\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-09T18:01:50.165079Z","iopub.execute_input":"2026-09-09T18:01:50.165483Z","iopub.status.idle":"2026-09-09T20:11:30.497708Z","shell.execute_reply.started":"2026-09-09T18:01:50.165452Z","shell.execute_reply":"2026-09-09T20:11:30.496829Z"}},"outputs":[],"execution_count":null}]}