{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":""},"papermill":{"default_parameters":{},"duration":null,"end_time":null,"environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-07-05T19:28:52.900117+00:00","version":"2.7.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"DR Ordinal Grading — Journal Version (Final)\n**Authors:** Sarra Chtioui, Olfa Fakhfakh, Farah Jemili\n\nPipeline Overview\n1. **Setup** — Dual T4 MirroredStrategy + Mixed Precision\n2. **Data** — EyePACS / APTOS / Messidor-1 / IDRiD with patient-level splits\n3. **Model** — EfficientNetB4 + Ordinal BCE head\n4. **Training** — Progressive unfreezing, robust checkpointing every epoch\n5. **Evaluation** — Full metric suite saved to CSV + JSON\n6. **Ablations** — Loss / Backbone / Dataset / Resolution / SMOTE\n7. **XAI** — GradCAM, GradCAM++, EigenCAM, ScoreCAM, SHAP, LIME + Insertion/Deletion AUC","metadata":{"papermill":{"duration":0.011937,"end_time":"2026-07-05T19:28:55.224926+00:00","exception":false,"start_time":"2026-07-05T19:28:55.212989+00:00","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Cell 1 — Install Dependencies","metadata":{"papermill":{"duration":0.009861,"end_time":"2026-07-05T19:28:55.244766+00:00","exception":false,"start_time":"2026-07-05T19:28:55.234905+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import subprocess, sys\n\ndef pip(pkg):\n # -q suppresses the giant wall of text, but will still show brief loading bars\n # If the package has multiple parts, we split it so check_call doesn't fail\n args = [sys.executable, \"-m\", \"pip\", \"install\", \"-q\"] + pkg.split()\n subprocess.check_call(args)\n\npip(\"grad-cam\")\npip(\"tf-keras-vis\")\npip(\"mlflow\")\npip(\"imbalanced-learn\")\npip(\"shap\")\npip(\"lime\")\npip(\"keras-cv\")\npip(\"onnx\")\npip(\"tf2onnx\")\npip(\"bentoml\")","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:28:55.265997Z","iopub.status.busy":"2026-07-05T19:28:55.265712Z","iopub.status.idle":"2026-07-05T19:29:34.885735Z","shell.execute_reply":"2026-07-05T19:29:34.885075Z"},"papermill":{"duration":39.633018,"end_time":"2026-07-05T19:29:34.887501+00:00","exception":false,"start_time":"2026-07-05T19:28:55.254483+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 2 — Imports & Hardware Setup","metadata":{"papermill":{"duration":0.011136,"end_time":"2026-07-05T19:29:34.909883+00:00","exception":false,"start_time":"2026-07-05T19:29:34.898747+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os, json, random, shutil, time, glob, warnings, zipfile, gc\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib\nmatplotlib.use(\"Agg\")\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, callbacks\nfrom tensorflow.keras.applications import (\n EfficientNetB4, EfficientNetB0,\n ResNet50V2, # V2 outperforms plain ResNet50 — no deployment cost\n DenseNet121, # Medical imaging literature anchor\n MobileNetV3Large, # Edge / MLOps latency target\n)\nimport tensorflow.keras.applications.efficientnet as effnet_preprocess\nimport tensorflow.keras.applications.resnet_v2 as resnet_v2_preprocess # ResNet50V2\nimport tensorflow.keras.applications.densenet as densenet_preprocess\nimport tensorflow.keras.applications.mobilenet_v3 as mobilenet_preprocess\ntry:\n from tensorflow.keras.applications import ConvNeXtTiny\n import tensorflow.keras.applications.convnext as convnext_preprocess\n _HAS_CONVNEXT = True\nexcept ImportError:\n _HAS_CONVNEXT = False\ntry:\n from tensorflow.keras.applications import EfficientNetV2S, EfficientNetV2M\n import tensorflow.keras.applications.efficientnet_v2 as effnetv2_preprocess\n _HAS_EFFNETV2 = True\nexcept ImportError:\n _HAS_EFFNETV2 = False\ntry:\n import keras_cv\n _HAS_KERAS_CV = True\nexcept ImportError:\n _HAS_KERAS_CV = False\n# ── ViT (transformer baseline — isolated from main registry) ─────────────────\n# Self-installing: on a Save & Run All (Commit) run there is no separate\n# '!pip install transformers' cell, so a bare ImportError here would silently\n# set _HAS_VIT=False and run_vit_comparison() would just [SKIP] forever.\n# Instead we try the import, and only if that fails do we pip install\n# transformers right here, then retry the import once.\ntry:\n from transformers import TFViTModel\n _HAS_VIT = True\nexcept ImportError:\n import subprocess, sys, importlib\n print(\"[SETUP] 'transformers' not found — installing it now ...\")\n _result = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"transformers\"],\n capture_output=True, text=True)\n if _result.returncode == 0:\n importlib.invalidate_caches()\n try:\n from transformers import TFViTModel\n _HAS_VIT = True\n print(\"[OK] Installed transformers.\")\n except ImportError:\n _HAS_VIT = False\n print(\"[WARN] transformers installed but TFViTModel still not importable.\")\n else:\n _HAS_VIT = False\n print(f\"[WARN] pip install transformers failed:\\n{_result.stderr[-1000:]}\")\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.metrics import (\n accuracy_score, cohen_kappa_score, roc_auc_score,\n classification_report, confusion_matrix,\n precision_score, recall_score, f1_score, roc_curve\n)\nfrom sklearn.preprocessing import label_binarize\nfrom pathlib import Path\n\nwarnings.filterwarnings(\"ignore\")\n\n# ── Mixed Precision ──────────────────────────────────────────────────────────\n# Disabled (float32) because mixed_float16 causes NaN overflows on EfficientNetB4\nkeras.mixed_precision.set_global_policy(\"float32\")\nprint(\"[OK] Precision: float32 (Safe mode)\")\n\n# ── Dual T4 MirroredStrategy ─────────────────────────────────────────────────\nSTRATEGY = tf.distribute.MirroredStrategy()\nN_GPUS = STRATEGY.num_replicas_in_sync\nprint(f\"[OK] MirroredStrategy: {N_GPUS} GPU(s)\")\nprint(f\" GPUs: {tf.config.list_physical_devices('GPU')}\")\nprint(f\" TF : {tf.__version__}\")\n\n# ── tf.data global performance knobs ─────────────────────────────────────────\nAUTOTUNE = tf.data.AUTOTUNE\nNUM_WORKERS = min(8, os.cpu_count() or 4) # parallel map workers\nOPTIONS = tf.data.Options()\nOPTIONS.threading.max_intra_op_parallelism = 1 # avoid GIL contention\nOPTIONS.threading.private_threadpool_size = NUM_WORKERS","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:29:34.93289Z","iopub.status.busy":"2026-07-05T19:29:34.932388Z","iopub.status.idle":"2026-07-05T19:30:08.851111Z","shell.execute_reply":"2026-07-05T19:30:08.850131Z"},"papermill":{"duration":33.931709,"end_time":"2026-07-05T19:30:08.85259+00:00","exception":false,"start_time":"2026-07-05T19:29:34.920881+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 3 — Global Config","metadata":{"papermill":{"duration":0.010656,"end_time":"2026-07-05T19:30:08.874792+00:00","exception":false,"start_time":"2026-07-05T19:30:08.864136+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"SEED = 42\nrandom.seed(SEED); np.random.seed(SEED); tf.random.set_seed(SEED)\nos.environ[\"TF_DETERMINISTIC_OPS\"] = \"1\"\n\n# ── Dataset paths (Kaggle mount points) ─────────────────────────────────────\ndef _find(slugs):\n for base in [\"/kaggle/input/\", \"/kaggle/input/datasets/\", \"/kaggle/input/competitions/\"]:\n for s in slugs:\n p = os.path.join(base, s)\n if os.path.isdir(p):\n return p\n return f\"/kaggle/input/{slugs[0]}\"\n\nEYEPACS_BASE = _find([\"tanlikesmath/diabetic-retinopathy-resized\",\n \"diabetic-retinopathy-resized\"])\nAPTOS_BASE = _find([\"aptos2019-blindness-detection\"])\nIDRID_BASE = _find([\"mariaherrerot/idrid-dataset\", \"idrid-dataset\",\n \"idrid-diabetic-retinopathy-grading\"])\nMESSIDOR_BASE = _find([\"mariaherrerot/messidor2preprocess\",\n \"andrewmvd/diabetic-retinopathy-in-fundus-images\",\n \"diabetic-retinopathy-in-fundus-images\"])\n\n# ── Working directories ───────────────────────────────────────────────────────\nWORK_DIR = \"/kaggle/working\"\nCKPT_DIR = f\"{WORK_DIR}/checkpoints\"\nOUT_DIR = f\"{WORK_DIR}/outputs\"\nTFR_DIR = f\"{WORK_DIR}/tfrecords\"\nZIP_PATH = f\"{WORK_DIR}/DR_Journal_artifacts.zip\"\nSTATE_F = f\"{WORK_DIR}/training_state.json\"\nSPLITS_F = f\"{WORK_DIR}/patient_splits.json\"\nHISTORY_CSV = f\"{OUT_DIR}/training_history.csv\"\n\nfor d in [CKPT_DIR, OUT_DIR, TFR_DIR]:\n os.makedirs(d, exist_ok=True)\n\n# ── Backbone registry ─────────────────────────────────────────────────────────\n# Per-backbone: (img_size, batch_per_gpu, gradcam_layer, loader, preprocess_fn)\n# preprocess_fn: normalises [0,255] uint8 → backbone-specific input range\n# EfficientNet: [0,255] (no-op, handles internally)\n# ResNet/DenseNet: ImageNet mean subtraction (caffe/torch mode)\n# InceptionV3/MobileNet: [-1, 1] scaling\n# ConvNeXt: imagenet normalisation\n# ── Backbone Registry ────────────────────────────────────────────────────────\n# Curated set: accuracy-speed spectrum, ALL export to ONNX + TFLite FP32 + INT8\n# Removed: InceptionV3 (ONNX fragile), DenseNet169 (redundant), ResNet50 (plain)\ndef _lazy_loader(module_path, class_name, **extra_kwargs):\n \"\"\"\n Defensive factory: (re-)imports `class_name` from `module_path` at CALL\n time rather than trusting a bare name bound once when this cell first\n ran. Fixes the observed bug where BACKBONE_REGISTRY's lambdas closed\n over a global (e.g. `EfficientNetV2M`) that raised\n `NameError: name 'EfficientNetV2M' is not defined` when the ablation\n loop reached it, despite the corresponding import having appeared to\n succeed earlier in the same kernel session. A fresh import per call is\n a few milliseconds of overhead and removes an entire class of\n kernel-state bugs.\n \"\"\"\n def _load(sz):\n import importlib\n mod = importlib.import_module(module_path)\n cls = getattr(mod, class_name)\n return cls(include_top=False, weights=\"imagenet\",\n input_shape=(sz, sz, 3), **extra_kwargs)\n return _load\n\n\nBACKBONE_REGISTRY = {\n # ① Primary model — EfficientNet family @ native high-res\n \"EfficientNetB4\": {\n \"img_size\": 380,\n \"batch_gpu\": 24, # P100: 380px fp32 — safe at 24 (was 16 on T4)\n \"cam_layer\": \"block6a_expand_activation\",\n \"loader\": _lazy_loader(\"tensorflow.keras.applications\", \"EfficientNetB4\"),\n \"preprocess\": effnet_preprocess.preprocess_input,\n \"deploy\": \"ONNX ✅ TFLite-INT8 ✅ BentoML ✅\",\n },\n # ② Lightweight EfficientNet baseline\n \"EfficientNetB0\": {\n \"img_size\": 224,\n \"batch_gpu\": 48, # P100: 224px fp32 — plenty of headroom\n \"cam_layer\": \"block6a_expand_activation\",\n \"loader\": _lazy_loader(\"tensorflow.keras.applications\", \"EfficientNetB0\"),\n \"preprocess\": effnet_preprocess.preprocess_input,\n \"deploy\": \"ONNX ✅ TFLite-INT8 ✅ BentoML ✅\",\n },\n # ③ Classic residual net — literature anchor, replaced with V2 (strictly better)\n \"ResNet50V2\": {\n \"img_size\": 224,\n \"batch_gpu\": 32, # P100: 224px fp32 — bumped from 24\n \"cam_layer\": \"post_bn\",\n \"loader\": _lazy_loader(\"tensorflow.keras.applications\", \"ResNet50V2\"),\n \"preprocess\": resnet_v2_preprocess.preprocess_input,\n \"deploy\": \"ONNX ✅ TFLite-INT8 ✅ BentoML ✅\",\n },\n # ④ Dense connectivity — heavily cited in medical imaging\n \"DenseNet121\": {\n \"img_size\": 224,\n \"batch_gpu\": 32, # P100: 224px fp32 — bumped from 24\n \"cam_layer\": \"relu\",\n \"loader\": _lazy_loader(\"tensorflow.keras.applications\", \"DenseNet121\"),\n \"preprocess\": densenet_preprocess.preprocess_input,\n \"deploy\": \"ONNX ✅ TFLite-INT8 ✅ BentoML ✅\",\n },\n # ⑤ Edge / MLOps latency target (fastest inference)\n \"MobileNetV3Large\": {\n \"img_size\": 224,\n \"batch_gpu\": 48, # P100: 224px fp32 — very lightweight model, plenty of room\n \"cam_layer\": \"multiply_17\",\n \"loader\": _lazy_loader(\"tensorflow.keras.applications\", \"MobileNetV3Large\",\n minimalistic=False),\n \"preprocess\": mobilenet_preprocess.preprocess_input,\n \"deploy\": \"ONNX ✅ TFLite-INT8 ⚠️ BentoML ✅\",\n },\n}\n\n\n# ── Conditionally add newer architectures (TF 2.10+) ─────────────────────────\nif _HAS_CONVNEXT:\n # ⑥ Modern pure-CNN — ONNX/TFLite excellent, comparable accuracy to EfficientNet\n BACKBONE_REGISTRY[\"ConvNeXtTiny\"] = {\n \"img_size\": 224,\n \"batch_gpu\": 32, # P100: 224px fp32 — bumped from 24\n \"cam_layer\": \"convnext_tiny_stage3_block2_identity\",\n \"loader\": _lazy_loader(\"tensorflow.keras.applications\", \"ConvNeXtTiny\"),\n \"preprocess\": convnext_preprocess.preprocess_input,\n \"deploy\": \"ONNX ✅ TFLite-INT8 ✅ BentoML ✅\",\n }\nif _HAS_EFFNETV2:\n # ⑦ SOTA in EfficientNet family — top accuracy challenger\n BACKBONE_REGISTRY[\"EfficientNetV2S\"] = {\n \"img_size\": 384,\n \"batch_gpu\": 16, # P100: 384px fp32 — bumped from 12\n \"cam_layer\": \"block6a_expand_activation\",\n \"loader\": _lazy_loader(\"tensorflow.keras.applications\", \"EfficientNetV2S\"),\n \"preprocess\": effnetv2_preprocess.preprocess_input,\n \"deploy\": \"ONNX ✅ TFLite-INT8 ✅ BentoML ✅\",\n }\n BACKBONE_REGISTRY[\"EfficientNetV2M\"] = {\n \"img_size\": 384,\n \"batch_gpu\": 12, # P100: 384px fp32 — slightly heavier than V2S, safe at 12\n \"cam_layer\": \"block7a_project_bn\",\n \"loader\": _lazy_loader(\"tensorflow.keras.applications\", \"EfficientNetV2M\"),\n \"preprocess\": effnetv2_preprocess.preprocess_input,\n \"deploy\": \"ONNX ✅ TFLite-INT8 ✅ BentoML ✅\",\n }\nif _HAS_KERAS_CV:\n # ⑧ Hybrid CNN-Transformer (Window + Grid Attention)\n BACKBONE_REGISTRY[\"MaxViTTiny\"] = {\n \"img_size\": 224,\n \"batch_gpu\": 32, # P100: 224px fp32 — requires some memory for attention grids\n \"cam_layer\": \"auto\", # MaxViT spatial layers are deep, depends on exact block, auto/none for now\n \"loader\": _lazy_loader(\"keras_cv.models\", \"MaxViTTinyBackbone\",\n include_rescaling=False),\n \"preprocess\": lambda x: x, # keras-cv backbones often expect 0-255 natively\n \"deploy\": \"ONNX ✅ TFLite-INT8 ⚠️ BentoML ✅\",\n }\n\nprint(f\"[OK] Backbone registry: {list(BACKBONE_REGISTRY.keys())}\")\n\n# ── ViT-Small Ordinal Model (transformer baseline, isolated from CNN registry) ─\n# Separate because: different preprocessing (NCHW, [-1,1]), no GradCAM,\n# and TFLite-INT8 is NOT supported — this is intentional for the deployment story.\nif _HAS_VIT:\n class ViTOrdinalModel(keras.Model):\n \"\"\"\n ViT-Small-patch16-224 wrapped as an ordinal grading head.\n CLS token → LayerNorm → Dropout → Dense(4, sigmoid).\n NOTE: GradCAM is NOT applicable (no spatial feature maps).\n Deployment: ONNX ✅ TFLite-FP32 ✅ TFLite-INT8 ❌\n \"\"\"\n def __init__(self, vit_backbone, **kwargs):\n super().__init__(**kwargs)\n self.vit = vit_backbone\n self.norm = layers.LayerNormalization()\n self.drop = layers.Dropout(0.3)\n # FIX: was activation=\"sigmoid\" here while training used\n # ordinal_bce_loss(..., from_logits=True) below — that combination\n # silently double-squashes the output (BCE-from-logits re-applies\n # sigmoid to an already-sigmoided value), distorting both the loss\n # landscape and calibration. Emitting raw logits here matches\n # build_model()'s convention (\"No sigmoid! Output raw logits for\n # numerical stability\") and is what ordinal_bce_loss actually expects.\n self.head = layers.Dense(4, activation=None,\n name=\"ordinal_head\", dtype=\"float32\")\n\n def call(self, inputs, training=False):\n # inputs: (B, H, W, 3) float32 pixels [0-255]\n x = inputs / 127.5 - 1.0 # → [-1, 1]\n x = tf.transpose(x, perm=[0, 3, 1, 2]) # NHWC → NCHW (HF format)\n out = self.vit(pixel_values=x, training=training)\n cls = out.last_hidden_state[:, 0, :] # CLS token\n cls = self.norm(cls, training=training)\n cls = self.drop(cls, training=training)\n return self.head(cls)\n\n\ndef run_vit_comparison(train_df, val_df):\n \"\"\"\n ViT-Small-patch16-224 as the transformer baseline for the backbone table.\n Runs AFTER all CNN ablations. Isolated from BACKBONE_REGISTRY because:\n - Different preprocessing (NCHW, normalised to [-1,1])\n - No GradCAM (CLS token output)\n - Needs smaller batch (8/GPU) to avoid OOM\n - TFLite-INT8 deliberately unsupported → shows CNN advantage\n Saves results into backbone_ablation.json under key 'ViT-Small'.\n Fully resumable.\n \"\"\"\n # Defensive: re-verify right here instead of trusting the _HAS_VIT flag\n # set (possibly in a different kernel state) back in the imports cell.\n # This was observed to read False even in a run where the transformers\n # import earlier in the same session did not report needing its\n # pip-install fallback — re-checking at the point of use removes that\n # whole class of stale-flag bug.\n global _HAS_VIT, TFViTModel\n try:\n from transformers import TFViTModel as _TFViTModel_fresh\n TFViTModel = _TFViTModel_fresh\n _HAS_VIT = True\n except ImportError:\n _HAS_VIT = False\n if not _HAS_VIT:\n print(\"[SKIP] ViT: 'transformers' package / TFViTModel not importable on this kernel\")\n return {}\n\n out_file = f\"{OUT_DIR}/backbone_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n if \"ViT-Small\" in results:\n r = results[\"ViT-Small\"]\n print(f\"[SKIP] ViT-Small already done: acc={r['accuracy']:.4f} qwk={r['qwk']:.4f}\")\n return results\n\n # ── OOM Protection: clear everything from previous backbone runs ──────────\n keras.backend.clear_session()\n gc.collect()\n tf.random.set_seed(SEED)\n\n VIT_ID = \"google/vit-small-patch16-224\"\n IMG_SZ = 224\n BS = 8 * N_GPUS # 16 total on T4×2 — conservative to avoid OOM\n ckd = f\"{CKPT_DIR}/bb_ViT_Small\"\n best_p = f\"{ckd}/best.weights.h5\"\n os.makedirs(ckd, exist_ok=True)\n\n print(f\"\\n{'='*60}\")\n print(f\"VIT BASELINE: {VIT_ID} @ {IMG_SZ}px | batch={BS} (OOM-safe)\")\n print(f\"{'='*60}\")\n\n cw = get_class_weights(train_df)\n tds = make_tf_dataset(train_df, BS, IMG_SZ, augment=True, class_weights=cw)\n vds = make_tf_dataset(val_df, BS, IMG_SZ, augment=False, shuffle=False)\n\n best_acc = 0.0\n class _VitCB(keras.callbacks.Callback):\n def on_epoch_end(self, epoch, logs=None):\n nonlocal best_acc\n va = float((logs or {}).get(\"val_ordinal_accuracy\", 0.0))\n if va > best_acc:\n best_acc = va\n self.model.save_weights(best_p)\n print(f\" [BEST] ViT-Small ep={epoch} acc={va:.4f}\")\n\n common_cbs = [\n _VitCB(),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=4,\n restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5,\n patience=2, min_lr=1e-8),\n ]\n\n try:\n with STRATEGY.scope():\n vit_bb = TFViTModel.from_pretrained(VIT_ID, from_pt=True)\n m = ViTOrdinalModel(vit_bb, name=\"ViT_Small_Ordinal\")\n # Build graph with one dummy forward pass\n _ = m(tf.zeros((1, IMG_SZ, IMG_SZ, 3)), training=False)\n\n # Phase 1: frozen backbone — train head only\n vit_bb.trainable = False\n m.compile(optimizer=keras.optimizers.Adam(1e-3),\n loss=ordinal_bce_loss, metrics=[ordinal_accuracy])\n\n print(\"\\n Phase 1/2: Head-only (frozen ViT, 10 epochs)\")\n m.fit(tds, validation_data=vds, epochs=10, verbose=1, callbacks=common_cbs)\n\n # Phase 2: partial unfreeze — last 4 of 12 transformer encoder layers\n print(\"\\n Phase 2/2: Partial unfreeze (last 4 encoder layers, 15 epochs)\")\n with STRATEGY.scope():\n for enc_layer in vit_bb.vit.encoder.layer[-4:]:\n enc_layer.trainable = True\n m.compile(optimizer=keras.optimizers.Adam(5e-6),\n loss=ordinal_bce_loss, metrics=[ordinal_accuracy])\n m.fit(tds, validation_data=vds, epochs=15, verbose=1, callbacks=common_cbs)\n\n if os.path.exists(best_p):\n m.load_weights(best_p)\n\n # ── Evaluate ──────────────────────────────────────────────────────────\n vds_e = make_tf_dataset(val_df, BS, IMG_SZ, augment=False, shuffle=False)\n raw = m.predict(vds_e, verbose=0)\n true = val_df[\"label\"].values\n # FIX: was ordinal_to_grade(p) (thr=0.5, probability-space) applied to\n # `raw`, which is a raw-logit output (see head-activation fix above).\n # Use the same optimal-threshold logit decode as evaluate()/\n # run_resolution_ablation for a fair, comparable number.\n opt_thr, _ = find_optimal_threshold(raw, true)\n preds = np.array([ordinal_logits_to_grade(p, thr=opt_thr) for p in raw])\n acc = float(accuracy_score(true, preds))\n qwk = float(cohen_kappa_score(true, preds, weights=\"quadratic\"))\n f1_m = float(f1_score(true, preds, average=\"macro\", zero_division=0))\n\n # Latency\n samp = preprocess_image(val_df.iloc[0][\"path\"], IMG_SZ)[np.newaxis]\n t0 = time.time()\n for _ in range(20): m(samp, training=False)\n lat = (time.time()-t0)/20*1000\n\n n_p = sum(np.prod(v.shape) for v in m.trainable_variables)\n\n results[\"ViT-Small\"] = {\n \"accuracy\": round(acc, 4),\n \"qwk\": round(qwk, 4),\n \"f1_macro\": round(f1_m, 4),\n \"auc_macro\": 0.0, # GradCAM N/A on ViT — skipped intentionally\n \"latency_ms\": round(lat, 1),\n \"params_M\": round(n_p / 1e6, 1),\n \"img_size\": IMG_SZ,\n \"deploy\": \"ONNX ✅ TFLite-FP32 ✅ TFLite-INT8 ❌ BentoML ✅\",\n \"notes\": \"Transformer baseline. INT8 quant fails on attention layers.\",\n }\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n print(f\"\\n ViT-Small: acc={acc:.4f} qwk={qwk:.4f} lat={lat:.1f}ms\")\n auto_zip(tag=\"vit_comparison_done\")\n\n except tf.errors.ResourceExhaustedError:\n print(\"[OOM] ViT-Small ran out of GPU memory — skipping (reduce BS or try FP16)\")\n except Exception as e:\n print(f\"[ERROR] ViT comparison failed: {e}\")\n finally:\n # ── Guaranteed cleanup ────────────────────────────────────────────────\n try: del m, vit_bb\n except: pass\n keras.backend.clear_session()\n gc.collect()\n print(\"[OK] ViT GPU memory released\")\n\n return results\n\nACTIVE_BB = \"EfficientNetB4\"\n_bb = BACKBONE_REGISTRY[ACTIVE_BB]\n\nCFG = {\n # Backbone\n \"ACTIVE_BB\": ACTIVE_BB,\n \"IMG_SIZE\": _bb[\"img_size\"],\n \"BATCH_SIZE\": _bb[\"batch_gpu\"] * N_GPUS, # total across all GPUs\n \"CAM_LAYER\": _bb[\"cam_layer\"],\n # Preprocessing\n \"CLAHE_CLIP\": 2.0,\n \"CLAHE_TILE\": 8,\n # Training phases\n \"PHASES\": [\n {\"unfreeze\": 0, \"lr\": 1e-3, \"epochs\": 3, \"patience\": 2, \"rlr_patience\": 2},\n ],\n \"DROPOUT\": 0.5,\n \"WEIGHT_DECAY\": 1e-4,\n \"MC_SAMPLES\": 10,\n # Ablation flags\n \"USE_FOCAL\": False,\n \"FOCAL_GAMMA\": 2.0,\n \"USE_MIXUP\": False,\n \"MIXUP_ALPHA\": 0.4,\n \"DATASETS\": [\"eyepacs\", \"aptos\", \"messidor\", \"idrid\"],\n # Statistical validation\n \"N_RUNS\": 1,\n # XAI\n \"XAI_SAMPLES\": 50,\n # Misc\n \"SEED\": SEED,\n \"VERSION\": \"journal-v2.0\",\n}\n\nwith open(f\"{WORK_DIR}/config.json\", \"w\") as f:\n json.dump(CFG, f, indent=2)\n\nprint(f\"[CONFIG] Backbone={CFG['ACTIVE_BB']} | IMG={CFG['IMG_SIZE']}px \"\n f\"| BatchSize={CFG['BATCH_SIZE']} ({N_GPUS}×{_bb['batch_gpu']})\")","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:30:08.897642Z","iopub.status.busy":"2026-07-05T19:30:08.897175Z","iopub.status.idle":"2026-07-05T19:30:08.936893Z","shell.execute_reply":"2026-07-05T19:30:08.936208Z"},"papermill":{"duration":0.053513,"end_time":"2026-07-05T19:30:08.938651+00:00","exception":false,"start_time":"2026-07-05T19:30:08.885138+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 4 — Session Resume / Artifact Restore","metadata":{"papermill":{"duration":0.01095,"end_time":"2026-07-05T19:30:08.960679+00:00","exception":false,"start_time":"2026-07-05T19:30:08.949729+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def restore_from_input():\n \"\"\"Auto-restore checkpoints from a previously saved Kaggle dataset.\"\"\"\n prev_runs = [\n \"/kaggle/input/notebooks/sarrachtioui/dr-journal-final\",\n \"/kaggle/input/dr-journal-final\",\n \"/kaggle/input/DR_Journal_Final\"\n ]\n src = None\n \n states = glob.glob(\"/kaggle/input/**/training_state.json\", recursive=True)\n if states:\n best_state_file = None\n best_phase = -1\n best_epoch = -1\n best_stat_runs = -1\n for s_file in states:\n try:\n src_dir = os.path.dirname(s_file)\n with open(s_file, \"r\") as f:\n data = json.load(f)\n p = data.get(\"phase\", 0)\n e = data.get(\"epoch\", -1)\n stat_runs = 0\n stat_file = os.path.join(src_dir, \"outputs\", \"statistical_results.json\")\n if os.path.exists(stat_file):\n try:\n with open(stat_file, \"r\") as sf:\n stat_runs = len(json.load(sf))\n except:\n pass\n if p > best_phase or (p == best_phase and e > best_epoch) or (p == best_phase and e == best_epoch and stat_runs > best_stat_runs):\n best_phase = p\n best_epoch = e\n best_stat_runs = stat_runs\n best_state_file = s_file\n except:\n pass\n if best_state_file:\n src = os.path.dirname(best_state_file)\n print(f\"[RESTORE] Found multiple states. Selected most advanced: phase={best_phase}, epoch={best_epoch} from {src}\")\n else:\n for p in prev_runs:\n if os.path.exists(os.path.join(p, \"training_state.json\")):\n src = p\n break\n\n if src:\n print(f\"[RESTORE] Found previous run at: {src}\")\n for fname in [\"training_state.json\", \"patient_splits.json\"]:\n s = os.path.join(src, fname)\n d = os.path.join(WORK_DIR, fname)\n if os.path.exists(s) and not os.path.exists(d):\n shutil.copy2(s, d)\n print(f\" [RESTORE] copied {fname}\")\n\n src_ckpt = os.path.join(src, \"checkpoints\")\n if os.path.isdir(src_ckpt):\n for f in glob.glob(os.path.join(src_ckpt, \"*\")):\n dst = os.path.join(CKPT_DIR, os.path.basename(f))\n if not os.path.exists(dst):\n if os.path.isdir(f):\n shutil.copytree(f, dst)\n else:\n shutil.copy2(f, dst)\n\n src_outputs = os.path.join(src, \"outputs\")\n if os.path.isdir(src_outputs):\n for f in glob.glob(os.path.join(src_outputs, \"*\")):\n dst = os.path.join(OUT_DIR, os.path.basename(f))\n if not os.path.exists(dst):\n if os.path.isdir(f):\n shutil.copytree(f, dst)\n else:\n shutil.copy2(f, dst)\n print(\" [RESTORE] copied existing evaluation outputs\")\n\n # ── NEW: merge ablation JSONs across ALL attached inputs, not just `src` ──\n # This is the actual fix: the block above only pulls files from whichever\n # single `src` won the phase/epoch/stat_runs tiebreak. That tiebreak knows\n # nothing about backbone_ablation.json (or the other ablation files), so\n # a \"winning\" src with fewer/no ablation results can silently blank them\n # out. This scans every copy of each ablation file anywhere under\n # /kaggle/input and keeps whichever has the most completed entries.\n def _merge_json_artifact(fname):\n dst = os.path.join(OUT_DIR, fname)\n current = json.load(open(dst)) if os.path.exists(dst) else {}\n for cand in glob.glob(f\"/kaggle/input/**/{fname}\", recursive=True):\n try:\n data = json.load(open(cand))\n except Exception:\n continue\n if isinstance(data, dict) and len(data) > len(current):\n current = data\n elif isinstance(data, list) and len(data) > len(current):\n current = data\n if current:\n with open(dst, \"w\") as f:\n json.dump(current, f, indent=2)\n print(f\" [RESTORE] merged {fname}: {len(current)} entries\")\n\n for fname in [\"backbone_ablation.json\", \"loss_ablation.json\", \"dataset_ablation.json\",\n \"resolution_ablation.json\", \"statistical_results.json\",\n \"encoding_ablation.json\", \"preprocessing_ablation.json\",\n # Extended-ablation outputs — added so a fresh session that\n # attaches a prior session's output as input resumes ALL\n # ablation progress, not just the original 7 files.\n \"combo_ablation.json\", \"calibration_results.json\",\n \"eval_uncertainty.json\", \"error_analysis.json\",\n \"fusion_dualres_ablation.json\", \"fusion_multibackbone_ablation.json\",\n \"attention_ablation.json\", \"emd_ablation.json\",\n \"augmentation_family_ablation.json\", \"ensemble_ablation.json\",\n \"leakage_ablation.json\", \"smote_ablation.json\",\n \"deployment_benchmark.json\", \"io_throughput_benchmark.json\",\n \"bentoml_packaging.json\"]:\n _merge_json_artifact(fname)\n\n # ── NEW: also pull backbone checkpoints + ablation JSON out of the zip\n # archive, in case a given result only ever existed inside\n # DR_Journal_artifacts.zip and was never extracted as a loose file. ──\n if not os.path.exists(os.path.join(OUT_DIR, \"backbone_ablation.json\")):\n for zpath in glob.glob(\"/kaggle/input/**/DR_Journal_artifacts.zip\", recursive=True):\n try:\n with zipfile.ZipFile(zpath) as zf:\n names = zf.namelist()\n if \"outputs/backbone_ablation.json\" in names:\n zf.extract(\"outputs/backbone_ablation.json\", WORK_DIR)\n print(f\" [RESTORE] extracted backbone_ablation.json from zip: {zpath}\")\n for n in names:\n if n.startswith(\"checkpoints/bb_\") and not os.path.exists(os.path.join(WORK_DIR, n)):\n zf.extract(n, WORK_DIR)\n break\n except Exception as e:\n print(f\" [RESTORE] zip read failed for {zpath}: {e}\")\n\n if src or os.path.exists(os.path.join(OUT_DIR, \"backbone_ablation.json\")):\n print(\"[RESTORE] Done. Ready to resume Phase 3!\")\n else:\n print(\"[RESTORE] No prior artifacts found — fresh start.\")\n\nrestore_from_input()","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:30:08.984914Z","iopub.status.busy":"2026-07-05T19:30:08.984693Z","iopub.status.idle":"2026-07-05T19:35:30.332586Z","shell.execute_reply":"2026-07-05T19:35:30.331682Z"},"papermill":{"duration":321.361403,"end_time":"2026-07-05T19:35:30.334111+00:00","exception":false,"start_time":"2026-07-05T19:30:08.972708+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 5 — State Persistence Helpers","metadata":{"papermill":{"duration":0.011042,"end_time":"2026-07-05T19:35:30.356302+00:00","exception":false,"start_time":"2026-07-05T19:35:30.34526+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def save_state(state: dict):\n for path in [STATE_F, STATE_F.replace(\".json\", \"_backup.json\")]:\n with open(path, \"w\") as f:\n json.dump(state, f, indent=2)\n\ndef load_state() -> dict:\n for path in [STATE_F, STATE_F.replace(\".json\", \"_backup.json\")]:\n if os.path.exists(path):\n try:\n with open(path) as f:\n s = json.load(f)\n print(f\"[STATE] Loaded: phase={s.get('phase',0)}, \"\n f\"epoch={s.get('epoch',-1)}, \"\n f\"best_acc={s.get('best_val_acc',0):.4f}\")\n return s\n except json.JSONDecodeError:\n pass\n return {\"phase\": 0, \"epoch\": -1, \"best_val_acc\": 0.0, \"history\": []}\n\ndef auto_zip(tag=\"\"):\n tmp = ZIP_PATH + \".tmp\"\n with zipfile.ZipFile(tmp, \"w\", zipfile.ZIP_DEFLATED) as zf:\n for root, dirs, files in os.walk(WORK_DIR):\n dirs[:] = [d for d in dirs if d not in (\"mlruns\", \"__pycache__\", \"tfrecords\")]\n for fname in files:\n if fname.endswith(\".tmp\") or fname == os.path.basename(ZIP_PATH):\n continue\n fp = os.path.join(root, fname)\n zf.write(fp, os.path.relpath(fp, WORK_DIR))\n if os.path.exists(ZIP_PATH):\n os.remove(ZIP_PATH)\n os.rename(tmp, ZIP_PATH)\n mb = os.path.getsize(ZIP_PATH) / 1e6\n print(f\"[ZIP{' '+tag if tag else ''}] {ZIP_PATH} ({mb:.1f} MB)\")\n\ndef latest_ckpt(phase: int):\n files = sorted(glob.glob(f\"{CKPT_DIR}/phase{phase}_ep*.weights.h5\"))\n return files[-1] if files else None\n\ndef save_history_csv(history: list):\n if not history:\n return\n pd.DataFrame(history).to_csv(HISTORY_CSV, index=False)","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:30.379017Z","iopub.status.busy":"2026-07-05T19:35:30.378284Z","iopub.status.idle":"2026-07-05T19:35:30.387403Z","shell.execute_reply":"2026-07-05T19:35:30.386668Z"},"papermill":{"duration":0.02209,"end_time":"2026-07-05T19:35:30.389024+00:00","exception":false,"start_time":"2026-07-05T19:35:30.366934+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 6 — Data Loaders","metadata":{"papermill":{"duration":0.011077,"end_time":"2026-07-05T19:35:30.410642+00:00","exception":false,"start_time":"2026-07-05T19:35:30.399565+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def _detect_img_dir(base, candidates, ext=\"*\"):\n for c in candidates:\n p = os.path.join(base, c)\n if os.path.isdir(p) and glob.glob(os.path.join(p, ext)):\n return p\n return base\n\ndef load_eyepacs():\n csv = f\"{EYEPACS_BASE}/trainLabels.csv\"\n df = pd.read_csv(csv)\n df.columns = df.columns.str.strip()\n df = df.rename(columns={\"image\": \"image_id\", \"level\": \"label\"})\n img_dir = _detect_img_dir(EYEPACS_BASE,\n [\"train/train\", \"resized_train/resized_train\", \"resized_train\", \"\"],\n \"*.jpeg\")\n df[\"path\"] = df[\"image_id\"].apply(lambda x: os.path.join(img_dir, x + \".jpeg\"))\n df[\"patient_id\"] = (df[\"image_id\"].str.extract(r'^(.+?)_(?:left|right)$')[0]\n .fillna(df[\"image_id\"]))\n df[\"source\"] = \"eyepacs\"\n df = df[df[\"path\"].apply(os.path.exists)].reset_index(drop=True)\n print(f\" EyePACS : {len(df):,} images, {df['patient_id'].nunique():,} patients\")\n return df[[\"image_id\", \"path\", \"label\", \"patient_id\", \"source\"]]\n\ndef load_aptos():\n df = pd.read_csv(f\"{APTOS_BASE}/train.csv\")\n df = df.rename(columns={\"id_code\": \"image_id\", \"diagnosis\": \"label\"})\n img_dir = _detect_img_dir(APTOS_BASE, [\"train_images\", \"\"], \"*.png\")\n df[\"path\"] = df[\"image_id\"].apply(lambda x: os.path.join(img_dir, x + \".png\"))\n df[\"patient_id\"] = df[\"image_id\"]\n df[\"source\"] = \"aptos\"\n df = df[df[\"path\"].apply(os.path.exists)].reset_index(drop=True)\n print(f\" APTOS : {len(df):,} images\")\n return df[[\"image_id\", \"path\", \"label\", \"patient_id\", \"source\"]]\n\ndef load_messidor():\n csv_files = glob.glob(f\"{MESSIDOR_BASE}/*.csv\")\n if not csv_files:\n raise FileNotFoundError(f\"No CSV found in {MESSIDOR_BASE}\")\n df = pd.read_csv(csv_files[0])\n df.columns = df.columns.str.strip()\n img_col = next(c for c in df.columns if any(k in c.lower()\n for k in [\"image\",\"filename\",\"name\",\"file\",\"id_code\",\"code\"]))\n grade_col = next(c for c in df.columns if any(k in c.lower()\n for k in [\"retinopathy\",\"grade\",\"diagnosis\",\"label\"]))\n df = df.rename(columns={img_col: \"image_id\", grade_col: \"label\"})\n df[\"label\"] = pd.to_numeric(df[\"label\"], errors=\"coerce\").fillna(0).astype(int)\n if df[\"label\"].max() == 3: # R0-R3 → ICDR\n df[\"label\"] = df[\"label\"].map({0:0,1:1,2:2,3:4})\n img_dir = _detect_img_dir(MESSIDOR_BASE,\n [\"images\",\"messidor-2/messidor-2/preprocess\",\"messidor-2/preprocess\",\n \"preprocess\",\"\"], \"*.*\")\n def _find_path(img_id):\n img_id = str(img_id)\n # Try the id as-is first (it may already contain the extension)\n p = os.path.join(img_dir, img_id)\n if os.path.exists(p): return p\n # Strip any existing extension (case-insensitive) and retry\n stem = os.path.splitext(img_id)[0]\n for ext in [\".png\",\".jpg\",\".jpeg\",\".JPG\",\".PNG\"]:\n p = os.path.join(img_dir, stem + ext)\n if os.path.exists(p): return p\n return None\n df[\"path\"] = df[\"image_id\"].apply(_find_path)\n df[\"patient_id\"] = df[\"image_id\"]\n df[\"source\"] = \"messidor\"\n df = df[df[\"path\"].notna()].reset_index(drop=True)\n print(f\" Messidor: {len(df):,} images\")\n return df[[\"image_id\", \"path\", \"label\", \"patient_id\", \"source\"]]\n\ndef load_idrid(split=\"train\"):\n csv = f\"{IDRID_BASE}/idrid_labels.csv\"\n df = pd.read_csv(csv)\n df.columns = df.columns.str.strip()\n img_col = next((c for c in df.columns if any(k in c.lower()\n for k in [\"image\",\"id_code\"])), df.columns[0])\n grade_col = next((c for c in df.columns if any(k in c.lower()\n for k in [\"retinopathy\",\"grade\",\"diagnosis\"])), df.columns[1])\n df = df.rename(columns={img_col: \"image_id\", grade_col: \"label\"})\n img_dir = _detect_img_dir(IDRID_BASE, [\"Imagenes/Imagenes\",\"images\",\"\"], \"*.jpg\")\n def _path(img_id):\n for ext in [\".jpg\",\".png\",\".jpeg\"]:\n p = os.path.join(img_dir, str(img_id) + ext)\n if os.path.exists(p): return p\n return os.path.join(img_dir, str(img_id) + \".jpg\")\n df[\"path\"] = df[\"image_id\"].apply(_path)\n df[\"patient_id\"] = df[\"image_id\"]\n df[\"source\"] = f\"idrid_{split}\"\n df = df[df[\"path\"].apply(os.path.exists)].reset_index(drop=True)\n print(f\" IDRiD-{split}: {len(df):,} images\")\n return df[[\"image_id\", \"path\", \"label\", \"patient_id\", \"source\"]]\n\ndef build_dataset_df(datasets=None):\n datasets = datasets or CFG[\"DATASETS\"]\n loaders = {\"eyepacs\": load_eyepacs, \"aptos\": load_aptos,\n \"messidor\": load_messidor, \"idrid\": lambda: load_idrid(\"train\")}\n frames = []\n for ds in datasets:\n try:\n frames.append(loaders[ds]())\n except Exception as e:\n print(f\" [WARN] {ds} failed: {e}\")\n df = pd.concat(frames, ignore_index=True)\n df[\"label\"] = df[\"label\"].astype(int).clip(0, 4)\n print(f\"\\n TOTAL: {len(df):,} images\")\n print(f\" Class dist:\\n{df['label'].value_counts().sort_index().to_string()}\")\n return df","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:30.4332Z","iopub.status.busy":"2026-07-05T19:35:30.432957Z","iopub.status.idle":"2026-07-05T19:35:30.451439Z","shell.execute_reply":"2026-07-05T19:35:30.450666Z"},"papermill":{"duration":0.031576,"end_time":"2026-07-05T19:35:30.452892+00:00","exception":false,"start_time":"2026-07-05T19:35:30.421316+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 7 — Patient-Level Splits","metadata":{"papermill":{"duration":0.010745,"end_time":"2026-07-05T19:35:30.474381+00:00","exception":false,"start_time":"2026-07-05T19:35:30.463636+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def make_splits(df, val_frac=0.10, test_frac=0.10):\n \"\"\"Patient-level stratified split. Saves + restores from JSON.\"\"\"\n if os.path.exists(SPLITS_F):\n with open(SPLITS_F) as f:\n sp = json.load(f)\n tr = df[df[\"image_id\"].isin(set(sp[\"train\"]))].reset_index(drop=True)\n va = df[df[\"image_id\"].isin(set(sp[\"val\"]))].reset_index(drop=True)\n te = df[df[\"image_id\"].isin(set(sp[\"test\"]))].reset_index(drop=True)\n print(f\"[SPLITS] Loaded | train={len(tr):,} val={len(va):,} test={len(te):,}\")\n return tr, va, te\n\n pat = (df.groupby(\"patient_id\")\n .agg(label=(\"label\", lambda x: x.mode()[0]),\n source=(\"source\", \"first\"))\n .reset_index())\n eye = pat[pat[\"source\"] == \"eyepacs\"]\n oth = pat[pat[\"source\"] != \"eyepacs\"]\n\n tr_p, tmp_p = train_test_split(\n eye[\"patient_id\"].values, test_size=val_frac+test_frac,\n stratify=eye[\"label\"].values, random_state=SEED)\n va_p, te_p = train_test_split(\n tmp_p, test_size=test_frac/(val_frac+test_frac), random_state=SEED)\n\n all_train = set(tr_p) | set(oth[\"patient_id\"].values)\n tr = df[df[\"patient_id\"].isin(all_train)].reset_index(drop=True)\n va = df[df[\"patient_id\"].isin(va_p)].reset_index(drop=True)\n te = df[df[\"patient_id\"].isin(te_p)].reset_index(drop=True)\n\n with open(SPLITS_F, \"w\") as f:\n json.dump({\"train\": tr[\"image_id\"].tolist(),\n \"val\": va[\"image_id\"].tolist(),\n \"test\": te[\"image_id\"].tolist(),\n \"seed\": SEED, \"version\": CFG[\"VERSION\"]}, f)\n print(f\"[SPLITS] Created | train={len(tr):,} val={len(va):,} test={len(te):,}\")\n return tr, va, te","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:30.497316Z","iopub.status.busy":"2026-07-05T19:35:30.49681Z","iopub.status.idle":"2026-07-05T19:35:30.504781Z","shell.execute_reply":"2026-07-05T19:35:30.503991Z"},"papermill":{"duration":0.020764,"end_time":"2026-07-05T19:35:30.506167+00:00","exception":false,"start_time":"2026-07-05T19:35:30.485403+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 8 — Preprocessing & tf.data Pipeline","metadata":{"papermill":{"duration":0.010988,"end_time":"2026-07-05T19:35:30.527997+00:00","exception":false,"start_time":"2026-07-05T19:35:30.517009+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def preprocess_image(path: str, img_size: int, augment: bool = False) -> np.ndarray:\n \"\"\"CLAHE + black-border crop + optional augmentation. Returns float32 [0,255].\"\"\"\n img = cv2.imread(path)\n if img is None:\n return np.zeros((img_size, img_size, 3), dtype=np.float32)\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n # Black-border crop\n gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n _, thr = cv2.threshold(gray, 7, 255, cv2.THRESH_BINARY)\n coords = cv2.findNonZero(thr)\n if coords is not None:\n x, y, w, h = cv2.boundingRect(coords)\n img = img[y:y+h, x:x+w]\n img = cv2.resize(img, (img_size, img_size), interpolation=cv2.INTER_CUBIC)\n # CLAHE on L channel\n lab = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)\n clahe = cv2.createCLAHE(clipLimit=CFG[\"CLAHE_CLIP\"],\n tileGridSize=(CFG[\"CLAHE_TILE\"], CFG[\"CLAHE_TILE\"]))\n lab[:, :, 0] = clahe.apply(lab[:, :, 0])\n img = cv2.cvtColor(lab, cv2.COLOR_LAB2RGB)\n if augment:\n if random.random() > 0.5: img = cv2.flip(img, 1)\n if random.random() > 0.5: img = cv2.flip(img, 0)\n α = random.uniform(0.8, 1.2)\n β = random.uniform(-20, 20)\n img = np.clip(α * img.astype(np.float32) + β, 0, 255).astype(np.uint8)\n angle = random.uniform(-15, 15)\n M = cv2.getRotationMatrix2D((img_size//2, img_size//2), angle, 1.0)\n img = cv2.warpAffine(img, M, (img_size, img_size))\n return img.astype(np.float32) # EfficientNet expects [0,255]\n\ndef grade_to_ordinal(grade: int) -> np.ndarray:\n return np.array([1.0 if grade >= k else 0.0 for k in range(1, 5)],\n dtype=np.float32)\n\ndef ordinal_to_grade(probs: np.ndarray, thr: float = 0.5) -> int:\n return int(np.sum(probs > thr))\n\ndef ordinal_logits_to_grade(logits: np.ndarray, thr: float = 0.0) -> int:\n \"\"\"Convert raw logits to grade. logits > thr ↔ sigmoid(logits) > sigmoid(thr).\"\"\"\n return int(np.sum(logits > thr))\n\ndef find_optimal_threshold(logits: np.ndarray, true_labels: np.ndarray) -> float:\n \"\"\"Search for the logit threshold that maximizes QWK on the given data.\"\"\"\n best_thr, best_qwk = 0.0, -1.0\n for thr in np.arange(-3.0, 3.01, 0.1):\n preds = np.sum(logits > thr, axis=1)\n qwk = cohen_kappa_score(true_labels, preds, weights=\"quadratic\")\n if qwk > best_qwk:\n best_qwk = qwk\n best_thr = thr\n # Fine search around the best\n for thr in np.arange(best_thr - 0.1, best_thr + 0.11, 0.01):\n preds = np.sum(logits > thr, axis=1)\n qwk = cohen_kappa_score(true_labels, preds, weights=\"quadratic\")\n if qwk > best_qwk:\n best_qwk = qwk\n best_thr = thr\n return float(best_thr), float(best_qwk)\n\ndef _sample_beta(alpha):\n \"\"\"Beta(alpha, alpha) via two Gamma(alpha,1) samples (X/(X+Y)) — avoids\n needing tensorflow_probability as an extra dependency.\"\"\"\n g1 = tf.random.gamma([], alpha=alpha, beta=1.0)\n g2 = tf.random.gamma([], alpha=alpha, beta=1.0)\n return g1 / (g1 + g2 + 1e-8)\n\n\ndef _mixup_batch_fn(mixup_alpha=0.2):\n \"\"\"\n Returns a post-batch map fn implementing MixUp (Zhang et al., 2018) for\n ordinal multi-hot targets: pairs each batch element with the element\n tf.roll'd by 1 position (no second dataset pass needed), samples\n lambda ~ Beta(alpha,alpha) per batch (clipped to >=0.5 so the \"primary\"\n image always dominates, standard practice), and linearly interpolates\n both images and ordinal target vectors. The interpolated ordinal vector\n is a valid soft target for ordinal_bce_loss (binary cross-entropy\n accepts probabilistic targets natively — no special-casing needed\n elsewhere). Sample weights (if present) are left untouched.\n\n FIX: CFG[\"USE_MIXUP\"] previously had NO reader anywhere in the\n notebook — the \"MixUp\" loss-ablation variant silently trained\n identically to the baseline. This is what actually makes it do\n something.\n \"\"\"\n def _apply(*args):\n imgs, ords = args[0], args[1]\n lam = _sample_beta(mixup_alpha)\n lam = tf.maximum(lam, 1.0 - lam)\n imgs_r = tf.roll(imgs, shift=1, axis=0)\n ords_r = tf.roll(ords, shift=1, axis=0)\n mixed_imgs = lam * imgs + (1.0 - lam) * imgs_r\n mixed_ords = lam * ords + (1.0 - lam) * ords_r\n return (mixed_imgs, mixed_ords) + args[2:]\n return _apply\n\n\ndef make_tf_dataset(df: pd.DataFrame,\n batch_size: int,\n img_size: int,\n augment: bool = False,\n shuffle: bool = True,\n class_weights: dict = None,\n mixup: bool = False,\n mixup_alpha: float = 0.2) -> tf.data.Dataset:\n \"\"\"Fully optimised tf.data pipeline for dual T4.\"\"\"\n paths = df[\"path\"].values\n labels = df[\"label\"].values\n ord_labels = np.array([grade_to_ordinal(l) for l in labels], dtype=np.float32)\n\n if class_weights is not None:\n weights = np.array([class_weights[l] for l in labels], dtype=np.float32)\n ds = tf.data.Dataset.from_tensor_slices((paths, ord_labels, weights))\n else:\n ds = tf.data.Dataset.from_tensor_slices((paths, ord_labels))\n\n if shuffle:\n ds = ds.shuffle(buffer_size=len(paths), seed=SEED,\n reshuffle_each_iteration=True)\n\n sz_ = img_size\n aug_ = augment\n\n def _process(*args):\n path_py = args[0].numpy().decode(\"utf-8\")\n img = preprocess_image(path_py, sz_, aug_)\n return (img,) + args[1:]\n\n def _tf_wrap(*args):\n dtypes = [tf.float32] + [a.dtype for a in args[1:]]\n out = tf.py_function(_process, args, dtypes)\n out[0].set_shape([sz_, sz_, 3])\n for i, a in enumerate(args[1:], 1):\n out[i].set_shape(a.shape)\n return tuple(out)\n\n ds = (ds\n .with_options(OPTIONS)\n .map(_tf_wrap, num_parallel_calls=NUM_WORKERS)\n .batch(batch_size, drop_remainder=shuffle))\n if mixup:\n ds = ds.map(_mixup_batch_fn(mixup_alpha), num_parallel_calls=NUM_WORKERS)\n ds = ds.prefetch(2)\n return ds\n\ndef get_class_weights(df: pd.DataFrame, clip_max: float = 3.0) -> dict:\n y = df[\"label\"].values\n w = compute_class_weight(\"balanced\", classes=np.unique(y), y=y)\n w = np.clip(w, 0.1, clip_max)\n return dict(enumerate(w))\n\nprint(\"[OK] Data pipeline defined.\")\nprint(f\" NUM_WORKERS={NUM_WORKERS} | AUTOTUNE=enabled | shuffle_buffer=full\")","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:30.551481Z","iopub.status.busy":"2026-07-05T19:35:30.551227Z","iopub.status.idle":"2026-07-05T19:35:30.567398Z","shell.execute_reply":"2026-07-05T19:35:30.566537Z"},"papermill":{"duration":0.02961,"end_time":"2026-07-05T19:35:30.568839+00:00","exception":false,"start_time":"2026-07-05T19:35:30.539229+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 8b — TFRecord Builder & Fast Reader\n> **Preprocess all images ONCE and save to TFRecords.**\n> Training then reads pre-cooked tensors — no OpenCV overhead per epoch.\n> This alone can **2-3× your GPU utilisation**.","metadata":{"papermill":{"duration":0.010628,"end_time":"2026-07-05T19:35:30.590186+00:00","exception":false,"start_time":"2026-07-05T19:35:30.579558+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def _bytes_feature(value):\n return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _int64_feature(value):\n return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef find_tfrecords_dir(tag: str):\n \"\"\"Find directory containing TFRecords for a tag (either working dir or mounted input).\"\"\"\n if glob.glob(f\"{TFR_DIR}/{tag}_*.tfrecord\"):\n return TFR_DIR\n \n # Check recursive deep mounts (finds the exact folder instantly)\n found = glob.glob(f\"/kaggle/input/**/tfrecords/{tag}_*.tfrecord\", recursive=True)\n if found:\n return os.path.dirname(found[0])\n \n return TFR_DIR\n\ndef build_tfrecords(df: pd.DataFrame, tag: str, img_size: int,\n images_per_shard: int = 2000):\n \"\"\"\n Preprocess all images in df and write to sharded TFRecords.\n Each record: {image: raw bytes of float32 tensor, label: int, image_id: str}\n Skips if TFRecords for this tag already exist.\n \"\"\"\n tfr_path = find_tfrecords_dir(tag)\n pattern = f\"{tfr_path}/{tag}_*.tfrecord\"\n existing = sorted(glob.glob(pattern))\n manifest = f\"{tfr_path}/{tag}_manifest.json\"\n if existing and os.path.exists(manifest):\n with open(manifest) as f:\n m = json.load(f)\n # FIX: this used to check n_images only. TFRecords bake in a FIXED\n # resolution at write time (_parse in make_tf_dataset_fast does a\n # tf.reshape to exactly img_size — reading a 380px-encoded shard at\n # img_size=384 throws, it does not resize), so switching to a\n # winning backbone with a different native resolution needs this\n # check to also catch a resolution mismatch, not just a count\n # mismatch, or it would silently reuse wrong-resolution shards.\n if m.get(\"n_images\") == len(df) and m.get(\"img_size\") == img_size:\n print(f\"[TFR] {tag}: {len(existing)} shards already exist \"\n f\"({m['n_images']} images @ {m.get('img_size')}px). Skipping.\")\n return existing\n else:\n reason = (f\"count {m.get('n_images')} vs {len(df)}\"\n if m.get(\"n_images\") != len(df)\n else f\"resolution {m.get('img_size')}px vs requested {img_size}px\")\n print(f\"[TFR] {tag}: mismatch ({reason}). Rebuilding.\")\n if tfr_path == TFR_DIR:\n for fp in existing: os.remove(fp)\n\n paths = df[\"path\"].values\n labels = df[\"label\"].values\n n = len(df)\n n_shards = max(1, (n + images_per_shard - 1) // images_per_shard)\n shard_files = []\n\n print(f\"[TFR] Building {tag}: {n} images → {n_shards} shards @ {img_size}px...\")\n t0 = time.time()\n\n for shard_idx in range(n_shards):\n shard_path = f\"{TFR_DIR}/{tag}_{shard_idx:04d}.tfrecord\"\n shard_files.append(shard_path)\n start = shard_idx * images_per_shard\n end = min(start + images_per_shard, n)\n\n with tf.io.TFRecordWriter(shard_path) as writer:\n for i in range(start, end):\n img = preprocess_image(paths[i], img_size, augment=False)\n # Compress to JPEG to save ~90% disk space (prevents Kaggle 20GB crash)\n img_uint8 = img.astype(np.uint8)\n img_jpeg = tf.io.encode_jpeg(img_uint8, quality=95).numpy()\n example = tf.train.Example(features=tf.train.Features(feature={\n \"image\": _bytes_feature(img_jpeg),\n \"label\": _int64_feature(int(labels[i])),\n }))\n writer.write(example.SerializeToString())\n\n if (shard_idx + 1) % 5 == 0 or shard_idx == n_shards - 1:\n elapsed = time.time() - t0\n print(f\" [{shard_idx+1}/{n_shards}] {elapsed:.0f}s elapsed\")\n\n # Always save new manifest to TFR_DIR\n new_manifest = f\"{TFR_DIR}/{tag}_manifest.json\"\n with open(new_manifest, \"w\") as f:\n json.dump({\"tag\": tag, \"n_images\": n, \"n_shards\": n_shards,\n \"img_size\": img_size}, f)\n\n elapsed = time.time() - t0\n total_mb = sum(os.path.getsize(fp) for fp in shard_files) / 1e6\n print(f\"[TFR] {tag}: Done in {elapsed:.0f}s | {total_mb:.0f} MB on disk\")\n return shard_files\n\n\ndef make_tf_dataset_fast(tag: str,\n batch_size: int,\n img_size: int,\n augment: bool = False,\n shuffle: bool = True,\n class_weights_map: dict = None,\n n_images: int = None,\n mixup: bool = False,\n mixup_alpha: float = 0.2) -> tf.data.Dataset:\n \"\"\"\n Read pre-built TFRecords. ~10× faster than on-the-fly OpenCV.\n Augmentation (flip + brightness + rotation) done in pure TF ops on GPU.\n \"\"\"\n tfr_path = find_tfrecords_dir(tag)\n pattern = f\"{tfr_path}/{tag}_*.tfrecord\"\n files = sorted(glob.glob(pattern))\n if not files:\n raise FileNotFoundError(f\"No TFRecords found for tag '{tag}'. \"\n f\"Run build_tfrecords() first.\")\n\n sz = img_size\n\n # Build weight lookup tensor if class weights provided\n if class_weights_map is not None:\n _w = np.ones(5, dtype=np.float32)\n for k, v in class_weights_map.items():\n _w[int(k)] = float(v)\n _w_tensor = tf.constant(_w)\n else:\n _w_tensor = None\n\n def _parse(serialized):\n features = tf.io.parse_single_example(serialized, {\n \"image\": tf.io.FixedLenFeature([], tf.string),\n \"label\": tf.io.FixedLenFeature([], tf.int64),\n })\n img = tf.io.decode_jpeg(features[\"image\"], channels=3)\n img = tf.cast(tf.reshape(img, [sz, sz, 3]), tf.float32)\n label = tf.cast(features[\"label\"], tf.int32)\n # Convert scalar label → ordinal vector [4]\n ord_label = tf.cast(tf.range(1, 5) <= label, tf.float32)\n if _w_tensor is not None:\n sample_w = tf.gather(_w_tensor, label)\n return img, ord_label, sample_w\n return img, ord_label\n\n # Continuous rotation helper (pure TF — no external deps)\n def _tf_rotate(image, max_deg=15.0):\n \"\"\"Rotate image by a random angle in [-max_deg, +max_deg] using projective transform.\"\"\"\n angle_rad = tf.random.uniform([], -max_deg, max_deg) * (3.14159265 / 180.0)\n h = tf.cast(tf.shape(image)[0], tf.float32)\n w = tf.cast(tf.shape(image)[1], tf.float32)\n cy, cx = h / 2.0, w / 2.0\n cos_a = tf.cos(angle_rad)\n sin_a = tf.sin(angle_rad)\n # Inverse affine: map destination → source coords\n transforms = tf.expand_dims(tf.stack([\n cos_a, sin_a, cx - cos_a * cx - sin_a * cy,\n -sin_a, cos_a, cy + sin_a * cx - cos_a * cy,\n 0.0, 0.0\n ]), 0)\n image = tf.expand_dims(image, 0)\n image = tf.raw_ops.ImageProjectiveTransformV3(\n images=image,\n transforms=transforms,\n output_shape=tf.cast(tf.stack([h, w]), tf.int32),\n interpolation=\"BILINEAR\",\n fill_mode=\"CONSTANT\",\n fill_value=0.0,\n )\n return tf.squeeze(image, 0)\n\n def _augment(*args):\n img = args[0]\n # ── Geometric (matches original OpenCV pipeline) ──\n img = tf.image.random_flip_left_right(img)\n img = tf.image.random_flip_up_down(img)\n img = _tf_rotate(img, max_deg=15.0) # continuous ±15°\n # ── Photometric (matches original α∈[0.8,1.2], β∈[-20,20]) ──\n img = tf.image.random_contrast(img, lower=0.8, upper=1.2) # α\n img = tf.image.random_brightness(img, max_delta=20.0) # β\n img = tf.clip_by_value(img, 0.0, 255.0)\n return (img,) + args[1:]\n\n # CRITICAL: use num_parallel_reads=1 when not shuffling to guarantee\n # deterministic shard order — parallel reads interleave shards and break\n # label alignment in QWKCallback and evaluate().\n parallel_reads = NUM_WORKERS if shuffle else 1\n ds = tf.data.TFRecordDataset(files, num_parallel_reads=parallel_reads)\n ds = ds.with_options(OPTIONS)\n\n if shuffle:\n buf = min(n_images if n_images else 10000, 4096)\n ds = ds.shuffle(buffer_size=buf, seed=SEED, reshuffle_each_iteration=True)\n\n ds = ds.map(_parse, num_parallel_calls=NUM_WORKERS)\n\n if augment:\n ds = ds.map(_augment, num_parallel_calls=NUM_WORKERS)\n\n ds = ds.batch(batch_size, drop_remainder=augment)\n if mixup:\n ds = ds.map(_mixup_batch_fn(mixup_alpha), num_parallel_calls=NUM_WORKERS)\n ds = ds.prefetch(2)\n return ds\n\n\ndef tfrecords_exist(tag: str) -> bool:\n \"\"\"Check if TFRecords for a given tag exist and are complete.\"\"\"\n tfr_path = find_tfrecords_dir(tag)\n manifest = f\"{tfr_path}/{tag}_manifest.json\"\n if not os.path.exists(manifest):\n return False\n files = glob.glob(f\"{tfr_path}/{tag}_*.tfrecord\")\n return len(files) > 0\n\nprint(\"[OK] TFRecord builder & fast reader defined.\")","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:30.613223Z","iopub.status.busy":"2026-07-05T19:35:30.612927Z","iopub.status.idle":"2026-07-05T19:35:30.634715Z","shell.execute_reply":"2026-07-05T19:35:30.63394Z"},"papermill":{"duration":0.035266,"end_time":"2026-07-05T19:35:30.636137+00:00","exception":false,"start_time":"2026-07-05T19:35:30.600871+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 9 — Loss Functions","metadata":{"papermill":{"duration":0.011038,"end_time":"2026-07-05T19:35:30.657992+00:00","exception":false,"start_time":"2026-07-05T19:35:30.646954+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def ordinal_bce_loss(y_true, y_pred):\n \"\"\"Sum of 4 independent sigmoid BCE losses.\"\"\"\n bce = tf.keras.backend.binary_crossentropy(y_true, y_pred, from_logits=True)\n return tf.reduce_sum(bce, axis=-1)\n\ndef focal_ordinal_loss(gamma=2.0):\n def _loss(y_true, y_pred):\n bce = tf.keras.backend.binary_crossentropy(y_true, y_pred, from_logits=True)\n p_t = tf.math.exp(-bce)\n fl = bce * tf.pow(1.0 - p_t, gamma)\n return tf.reduce_mean(tf.reduce_sum(fl, axis=-1))\n return _loss\n\ndef get_loss():\n if CFG[\"USE_FOCAL\"]:\n print(f\"[LOSS] Focal Ordinal (gamma={CFG['FOCAL_GAMMA']})\")\n return focal_ordinal_loss(CFG[\"FOCAL_GAMMA\"])\n print(\"[LOSS] Ordinal BCE\")\n return ordinal_bce_loss\n\ndef ordinal_accuracy(y_true, y_pred):\n # y_pred are raw logits. > 0.0 is equivalent to sigmoid(y_pred) > 0.5\n pred_grades = tf.cast(tf.reduce_sum(tf.cast(y_pred > 0.0, tf.float32), axis=-1), tf.float32)\n true_grades = tf.cast(tf.reduce_sum(y_true, axis=-1), tf.float32)\n return tf.cast(tf.equal(pred_grades, true_grades), tf.float32)","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:30.6818Z","iopub.status.busy":"2026-07-05T19:35:30.681278Z","iopub.status.idle":"2026-07-05T19:35:30.687007Z","shell.execute_reply":"2026-07-05T19:35:30.686482Z"},"papermill":{"duration":0.019087,"end_time":"2026-07-05T19:35:30.68838+00:00","exception":false,"start_time":"2026-07-05T19:35:30.669293+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 10 — Model Builder","metadata":{"papermill":{"duration":0.011079,"end_time":"2026-07-05T19:35:30.710269+00:00","exception":false,"start_time":"2026-07-05T19:35:30.69919+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def build_model(unfreeze_last: int = 20, backbone_name: str = None) -> keras.Model:\n \"\"\"\n Ordinal DR classifier.\n Architecture: Backbone → GAP → BN → MC-Dropout(0.5) → Dense(4, sigmoid, float32)\n The float32 cast on the head is REQUIRED with mixed_float16 to prevent NaN loss.\n \"\"\"\n bb_name = backbone_name or CFG[\"ACTIVE_BB\"]\n reg = BACKBONE_REGISTRY[bb_name]\n sz = reg[\"img_size\"]\n base = reg[\"loader\"](sz)\n base.trainable = True\n\n # CRITICAL: Always freeze BatchNorm layers regardless of unfreeze depth.\n # Unfreezing BN during fine-tuning causes chaotic running-stat updates from\n # small medical-image batches, destabilising the loss. Frozen BN layers\n # use their stable ImageNet statistics as a fixed normaliser.\n for layer in base.layers:\n if isinstance(layer, layers.BatchNormalization):\n layer.trainable = False\n\n if unfreeze_last > 0:\n for layer in base.layers[:-unfreeze_last]:\n layer.trainable = False\n elif unfreeze_last == 0:\n base.trainable = False\n\n inp = keras.Input(shape=(sz, sz, 3), name=\"fundus_input\")\n # Apply backbone-specific normalisation (e.g. ImageNet mean subtraction)\n preprocess_fn = reg.get(\"preprocess\", lambda x: x)\n x = layers.Lambda(preprocess_fn, name=\"backbone_preprocess\")(inp)\n # Do NOT hardcode training=False — let Keras control it via model.fit().\n # Frozen layers (layer.trainable=False) already use running stats automatically.\n # Unfrozen BN layers correctly update from batch stats during fine-tuning.\n x = base(x)\n x = layers.GlobalAveragePooling2D(name=\"gap\")(x)\n x = layers.BatchNormalization(name=\"head_bn\")(x)\n x = layers.Dropout(CFG[\"DROPOUT\"], name=\"mc_dropout\")(x)\n # Cast to float32 BEFORE the final dense head (mixed_float16 safety)\n x = layers.Activation(\"linear\", dtype=\"float32\", name=\"fp32_cast\")(x)\n # No sigmoid! Output raw logits for numerical stability.\n out = layers.Dense(4, name=\"ordinal_head\", dtype=\"float32\")(x)\n model = keras.Model(inp, out, name=f\"DR_{bb_name}\")\n n_train = sum(np.prod(v.shape) for v in model.trainable_variables)\n print(f\"[MODEL] {bb_name} | {sz}px | trainable={n_train:,}\")\n return model","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:30.733155Z","iopub.status.busy":"2026-07-05T19:35:30.732959Z","iopub.status.idle":"2026-07-05T19:35:30.739907Z","shell.execute_reply":"2026-07-05T19:35:30.739278Z"},"papermill":{"duration":0.020031,"end_time":"2026-07-05T19:35:30.741358+00:00","exception":false,"start_time":"2026-07-05T19:35:30.721327+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 11 — Callbacks","metadata":{"papermill":{"duration":0.0108,"end_time":"2026-07-05T19:35:30.76315+00:00","exception":false,"start_time":"2026-07-05T19:35:30.75235+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class QWKCallback(keras.callbacks.Callback):\n def __init__(self, val_df, batch_size, img_size):\n super().__init__()\n self.val_df = val_df\n self.batch_size = batch_size\n self.img_size = img_size\n # Build the val dataset ONCE — rebuilding every epoch wastes time and\n # risks non-deterministic ordering if parallel reads are not controlled.\n safe_bs = min(batch_size, 8)\n self.val_ds = make_tf_dataset_fast(\n \"val\", safe_bs, img_size, augment=False, shuffle=False)\n self.true_grades = val_df[\"label\"].values # fixed reference order\n\n def on_epoch_end(self, epoch, logs=None):\n logs = logs or {}\n preds_list = []\n for x, y in self.val_ds:\n batch_preds = self.model(x, training=False)\n preds_list.append(batch_preds.numpy())\n\n preds = np.concatenate(preds_list, axis=0)\n # Align length (drop_remainder=False so last batch may be smaller)\n true_grades = self.true_grades[:len(preds)]\n # Find optimal threshold instead of hardcoding 0.0\n opt_thr, opt_qwk = find_optimal_threshold(preds, true_grades)\n # Also report the default threshold for reference\n qwk_default = cohen_kappa_score(\n true_grades, np.sum(preds > 0.0, axis=1), weights=\"quadratic\")\n logs[\"val_qwk\"] = float(opt_qwk)\n print(f\" [QWK] thr=0.0→{qwk_default:.4f} | opt thr={opt_thr:.2f}→{opt_qwk:.4f}\")\n\n\nclass MetricsLogger(keras.callbacks.Callback):\n \"\"\"\n Logs every metric every epoch to:\n - training_state.json (full resumable state)\n - training_history.csv (flat CSV for plotting)\n Tracks: train_loss, train_acc, val_loss, val_acc, val_qwk, lr\n \"\"\"\n def __init__(self, phase_idx: int, state: dict):\n super().__init__()\n self.phase_idx = phase_idx\n self.state = state\n self.best_acc = state.get(\"best_val_acc\", 0.0)\n\n def on_epoch_end(self, epoch, logs=None):\n logs = logs or {}\n lr = float(keras.backend.get_value(self.model.optimizer.learning_rate))\n row = {\n \"phase\": self.phase_idx,\n \"epoch\": epoch,\n \"lr\": round(lr, 8),\n \"train_loss\": round(float(logs.get(\"loss\", 0)), 6),\n \"train_acc\": round(float(logs.get(\"ordinal_accuracy\", 0)), 6),\n \"val_loss\": round(float(logs.get(\"val_loss\", 0)), 6),\n \"val_acc\": round(float(logs.get(\"val_ordinal_accuracy\", 0)), 6),\n \"val_qwk\": round(float(logs.get(\"val_qwk\", 0)), 6),\n }\n self.state[\"history\"].append(row)\n self.state[\"phase\"] = self.phase_idx\n self.state[\"epoch\"] = epoch\n self.state[\"best_val_acc\"] = max(self.best_acc, row[\"val_acc\"])\n save_state(self.state)\n save_history_csv(self.state[\"history\"])\n print(f\" [LOG] ph={self.phase_idx} ep={epoch:03d} \"\n f\"tr_acc={row['train_acc']:.4f} vl_acc={row['val_acc']:.4f} \"\n f\"vl_qwk={row['val_qwk']:.4f} lr={lr:.2e}\")\n\n\nclass RobustCheckpoint(keras.callbacks.Callback):\n \"\"\"Saves weights every epoch. Keeps best model. Prunes to last 3 per phase.\"\"\"\n def __init__(self, phase_idx: int, state: dict):\n super().__init__()\n self.phase_idx = phase_idx\n self.state = state\n self.best_acc = state.get(\"best_val_acc\", 0.0)\n self._zip_ctr = 0\n\n def on_epoch_end(self, epoch, logs=None):\n logs = logs or {}\n val_acc = float(logs.get(\"val_ordinal_accuracy\", 0.0))\n path = f\"{CKPT_DIR}/phase{self.phase_idx}_ep{epoch:03d}_acc{val_acc:.4f}.weights.h5\"\n self.model.save_weights(path)\n if val_acc > self.best_acc:\n self.best_acc = val_acc\n self.model.save_weights(f\"{CKPT_DIR}/best_model.weights.h5\")\n self.model.save(f\"{CKPT_DIR}/best_model_full.keras\")\n print(f\" [BEST] epoch={epoch} val_acc={val_acc:.4f} ✓\")\n # Prune old checkpoints (keep last 3 per phase)\n old = sorted(glob.glob(f\"{CKPT_DIR}/phase{self.phase_idx}_ep*.weights.h5\"))[:-3]\n for fp in old:\n try: os.remove(fp)\n except: pass\n # Auto-zip every 5 epochs\n self._zip_ctr += 1\n if self._zip_ctr >= 5:\n auto_zip(tag=f\"ph{self.phase_idx}_ep{epoch}\")\n self._zip_ctr = 0","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:30.786192Z","iopub.status.busy":"2026-07-05T19:35:30.785993Z","iopub.status.idle":"2026-07-05T19:35:30.798436Z","shell.execute_reply":"2026-07-05T19:35:30.79792Z"},"papermill":{"duration":0.025653,"end_time":"2026-07-05T19:35:30.799795+00:00","exception":false,"start_time":"2026-07-05T19:35:30.774142+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 12 — Main Training Loop","metadata":{"papermill":{"duration":0.011116,"end_time":"2026-07-05T19:35:30.822357+00:00","exception":false,"start_time":"2026-07-05T19:35:30.811241+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def train(train_df: pd.DataFrame, val_df: pd.DataFrame,\n tfrecord_tag: str = \"train\") -> keras.Model:\n \"\"\"\n 3-phase progressive unfreezing with full resumability.\n All metrics saved every epoch. Returns the best model.\n\n tfrecord_tag: which pre-built TFRecord tag to read from in the fast\n path (default \"train\", i.e. the original full training set built in\n STEP 1b). Pass a different tag (e.g. \"train_final\") when calling this\n with a `train_df` that reflects a winning dataset-composition/\n oversampling ablation choice — the fast path reads TFRecords by tag,\n NOT by inspecting train_df's actual contents, so without this override\n it would silently keep training on whatever \"train\" was built from,\n regardless of what train_df you passed in. See STEP 5b in Main\n Execution for where this matters.\n \"\"\"\n state = load_state()\n start_ph = state.get(\"phase\", 0)\n best_ckpt = f\"{CKPT_DIR}/best_model.weights.h5\"\n model = None\n cw = get_class_weights(train_df)\n print(f\"[TRAIN] Class weights: { {k: round(v,3) for k,v in cw.items()} }\")\n\n IMG = CFG[\"IMG_SIZE\"]\n BS = CFG[\"BATCH_SIZE\"]\n\n # Prefer TFRecords (10× faster) over on-the-fly preprocessing\n if tfrecords_exist(tfrecord_tag) and tfrecords_exist(\"val\"):\n print(f\"[TRAIN] Using pre-built TFRecords (fast path, tag='{tfrecord_tag}')\")\n train_ds = make_tf_dataset_fast(tfrecord_tag, BS, IMG, augment=True,\n shuffle=True, class_weights_map=cw,\n n_images=len(train_df),\n mixup=CFG[\"USE_MIXUP\"])\n val_ds = make_tf_dataset_fast(\"val\", BS, IMG, augment=False, shuffle=False)\n else:\n print(\"[TRAIN] Using on-the-fly preprocessing (slow path — consider building TFRecords)\")\n train_ds = make_tf_dataset(train_df, BS, IMG, augment=True,\n shuffle=True, class_weights=cw,\n mixup=CFG[\"USE_MIXUP\"])\n val_ds = make_tf_dataset(val_df, BS, IMG, augment=False, shuffle=False)\n\n for ph_idx, ph_cfg in enumerate(CFG[\"PHASES\"]):\n if ph_idx < start_ph:\n print(f\"[SKIP] Phase {ph_idx+1} already done.\")\n continue\n\n print(f\"\\n{'='*60}\")\n print(f\"PHASE {ph_idx+1}/3 | unfreeze={ph_cfg['unfreeze']} \"\n f\"| lr={ph_cfg['lr']} | max_epochs={ph_cfg['epochs']}\")\n print(f\"{'='*60}\")\n\n with STRATEGY.scope():\n model = build_model(unfreeze_last=ph_cfg[\"unfreeze\"])\n if ph_idx > 0 and os.path.exists(best_ckpt):\n model.load_weights(best_ckpt)\n print(f\" [RESUME] Loaded best weights from phase {ph_idx}\")\n elif ph_idx == 0:\n ckpt = latest_ckpt(0)\n if ckpt:\n model.load_weights(ckpt)\n print(f\" [RESUME] Continued from {ckpt}\")\n model.compile(\n optimizer=keras.optimizers.AdamW(\n learning_rate=ph_cfg[\"lr\"],\n weight_decay=CFG[\"WEIGHT_DECAY\"],\n clipnorm=1.0), # prevent gradient explosion during full unfreeze\n loss=get_loss(),\n metrics=[ordinal_accuracy],\n )\n\n # Determine start epoch for this phase\n ph_hist = [h for h in state.get(\"history\", []) if h[\"phase\"] == ph_idx]\n start_epoch = (ph_hist[-1][\"epoch\"] + 1) if (ph_hist and ph_idx == start_ph) else 0\n\n # Restore LR if resuming mid-phase (prevents LR jumping back to initial on restart)\n if ph_hist and ph_idx == start_ph and start_epoch > 0:\n last_lr = ph_hist[-1][\"lr\"]\n model.optimizer.learning_rate = float(last_lr)\n print(f\" [RESUME] Restored LR to {last_lr:.2e} (was at epoch {start_epoch-1})\")\n\n cbs = [\n QWKCallback(val_df, BS, IMG),\n MetricsLogger(ph_idx, state),\n RobustCheckpoint(ph_idx, state),\n keras.callbacks.ReduceLROnPlateau(\n monitor=\"val_loss\", factor=0.5,\n patience=ph_cfg.get(\"rlr_patience\", 4),\n min_lr=1e-8, verbose=1),\n keras.callbacks.EarlyStopping(\n monitor=\"val_loss\", patience=ph_cfg[\"patience\"],\n restore_best_weights=True, verbose=1),\n ]\n\n model.fit(\n train_ds,\n validation_data=val_ds,\n initial_epoch=start_epoch,\n epochs=ph_cfg[\"epochs\"],\n callbacks=cbs,\n verbose=1,\n )\n\n epochs_run = ph_cfg[\"epochs\"] - start_epoch\n if epochs_run > 0:\n state[\"phase\"] = ph_idx + 1\n state[\"epoch\"] = -1\n save_state(state)\n auto_zip(tag=f\"phase{ph_idx+1}_complete\")\n print(f\" [DONE] Phase {ph_idx+1} complete.\")\n else:\n print(f\" [WARN] Phase {ph_idx+1}: 0 epochs ran — state not advanced.\")\n\n # If all phases were skipped (full resume), model is still None — rebuild it\n if model is None:\n print(\"[TRAIN] All phases already complete — rebuilding model to load best weights.\")\n with STRATEGY.scope():\n model = build_model(unfreeze_last=-1)\n model.compile(\n optimizer=keras.optimizers.AdamW(\n learning_rate=CFG[\"PHASES\"][-1][\"lr\"],\n weight_decay=CFG[\"WEIGHT_DECAY\"]),\n loss=get_loss(),\n metrics=[ordinal_accuracy],\n )\n # Load best weights into final model\n if os.path.exists(best_ckpt):\n model.load_weights(best_ckpt)\n print(f\"\\n[TRAIN] Loaded best model weights: {best_ckpt}\")\n plot_training_curves(state)\n auto_zip(tag=\"training_complete\")\n print(\"\\n[TRAIN] Training complete.\")\n return model\n\n\ndef plot_training_curves(state: dict):\n hist = pd.DataFrame(state.get(\"history\", []))\n if hist.empty:\n return\n fig, axes = plt.subplots(1, 3, figsize=(18, 5))\n \n # Create continuous absolute epochs instead of phase-relative ones to avoid zig-zags\n abs_epochs = np.arange(len(hist))\n \n # Phase boundaries (find where phase changes)\n bounds = []\n if \"phase\" in hist.columns:\n phases = hist[\"phase\"].values\n for i in range(1, len(phases)):\n if phases[i] != phases[i-1]:\n bounds.append(i)\n\n def _vlines(ax):\n for b in bounds:\n ax.axvline(b, color=\"gray\", lw=1.2, ls=\":\", alpha=0.7)\n\n axes[0].plot(abs_epochs, hist[\"train_acc\"], label=\"Train Acc\", lw=2, color=\"#2196F3\")\n axes[0].plot(abs_epochs, hist[\"val_acc\"], label=\"Val Acc\", lw=2, color=\"#4CAF50\", ls=\"--\")\n _vlines(axes[0])\n axes[0].set_title(\"Accuracy\", fontweight=\"bold\"); axes[0].legend(); axes[0].grid(alpha=0.3)\n\n axes[1].plot(abs_epochs, hist[\"train_loss\"], label=\"Train Loss\", lw=2, color=\"#FF5722\")\n axes[1].plot(abs_epochs, hist[\"val_loss\"], label=\"Val Loss\", lw=2, color=\"#E91E63\", ls=\"--\")\n _vlines(axes[1])\n axes[1].set_title(\"Ordinal BCE Loss\", fontweight=\"bold\"); axes[1].legend(); axes[1].grid(alpha=0.3)\n\n if \"val_qwk\" in hist.columns:\n axes[2].plot(abs_epochs, hist[\"val_qwk\"], label=\"Val QWK\", lw=2, color=\"#9C27B0\")\n _vlines(axes[2])\n axes[2].set_title(\"Val QWK\", fontweight=\"bold\"); axes[2].legend(); axes[2].grid(alpha=0.3)\n\n for ax in axes:\n ax.set_xlabel(\"Global Epoch\")\n plt.suptitle(f\"Training Curves — {CFG['ACTIVE_BB']} @ {CFG['IMG_SIZE']}px\",\n fontsize=14, fontweight=\"bold\")\n plt.tight_layout()\n path = f\"{OUT_DIR}/fig_training_curves.png\"\n plt.savefig(path, dpi=150, bbox_inches=\"tight\")\n plt.close()\n print(f\"[SAVED] {path}\")\n\nprint(\"[OK] Model, loss, callbacks, training loop defined.\")","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:30.846823Z","iopub.status.busy":"2026-07-05T19:35:30.846567Z","iopub.status.idle":"2026-07-05T19:35:30.864711Z","shell.execute_reply":"2026-07-05T19:35:30.864063Z"},"papermill":{"duration":0.032046,"end_time":"2026-07-05T19:35:30.866181+00:00","exception":false,"start_time":"2026-07-05T19:35:30.834135+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 13 — Full Evaluation (Multi-class + Binary Triage)","metadata":{"papermill":{"duration":0.011439,"end_time":"2026-07-05T19:35:30.888918+00:00","exception":false,"start_time":"2026-07-05T19:35:30.877479+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def evaluate(model, df: pd.DataFrame, desc: str = \"\", save_tag: str = None) -> dict:\n \"\"\"\n MC-Dropout (10 passes) + TTA (H-flip) → final predictions.\n Saves ALL metrics to JSON. Returns result dict with probs/preds/labels.\n\n TRAINING metrics (per epoch): train_loss, train_acc (in training_history.csv)\n VALIDATION metrics (per epoch): val_loss, val_acc, val_qwk (in training_history.csv)\n EVALUATION metrics (final): accuracy, qwk, auc_macro, f1_macro, f1_weighted,\n precision_macro, precision_weighted, recall_macro, recall_weighted,\n f1_per_grade[0..4], precision_per_grade[0..4], recall_per_grade[0..4],\n sensitivity_per_grade[0..4], specificity_per_grade[0..4]\n \"\"\"\n # ── SKIP if already done ────────────────────────────────────────────────\n if save_tag:\n out_json = f\"{OUT_DIR}/eval_{save_tag}.json\"\n if os.path.exists(out_json):\n print(f\" [SKIP] Evaluation '{save_tag}' already exists → loading cached result.\")\n with open(out_json) as f:\n cached = json.load(f)\n cached[\"probs\"] = np.zeros((len(df), 5), dtype=np.float32)\n cached[\"preds\"] = np.zeros(len(df), dtype=np.int32)\n cached[\"labels\"] = df[\"label\"].values\n return cached\n # ────────────────────────────────────────────────────────────────────────\n\n import gc\n gc.collect() # Free Python objects — but do NOT call clear_session() here, it destroys the model\n\n IMG = CFG[\"IMG_SIZE\"]\n BS = 8 # Small fixed batch — prevents OOM on GPU after training fills memory\n paths, labels = df[\"path\"].tolist(), df[\"label\"].tolist()\n\n print(f\" [EVAL] {desc}: {len(paths)} images, BS={BS}, MC={CFG['MC_SAMPLES']}\")\n\n # MC-Dropout inference (training=True keeps Dropout active for uncertainty)\n # Use BS=4 to keep BN training-mode memory footprint minimal.\n MC_BS = 4\n mc_probs = []\n for mc_i in range(CFG[\"MC_SAMPLES\"]):\n batch_probs = []\n for i in range(0, len(paths), MC_BS):\n imgs = np.array([preprocess_image(p, IMG) for p in paths[i:i+MC_BS]])\n batch_probs.append(model(imgs, training=True).numpy())\n del imgs\n mc_probs.append(np.concatenate(batch_probs, axis=0))\n del batch_probs\n if (mc_i + 1) % 5 == 0:\n gc.collect()\n print(f\" MC pass {mc_i+1}/{CFG['MC_SAMPLES']} done\")\n mean_mc = np.mean(mc_probs, axis=0)\n del mc_probs\n gc.collect()\n\n # TTA (horizontal flip)\n tta_probs = []\n for i in range(0, len(paths), BS):\n imgs = np.array([preprocess_image(p, IMG) for p in paths[i:i+BS]])\n flipped = imgs[:, :, ::-1, :]\n p1 = model(imgs, training=False).numpy()\n p2 = model(flipped, training=False).numpy()\n tta_probs.append((p1 + p2) / 2.0)\n tta = np.concatenate(tta_probs, axis=0)\n\n final_logits = (mean_mc + tta) / 2.0\n final_probs = 1.0 / (1.0 + np.exp(-final_logits))\n\n # Find optimal threshold for ordinal→grade conversion\n y_true_arr = np.array(labels)\n opt_thr, opt_qwk = find_optimal_threshold(final_logits, y_true_arr)\n print(f\" [EVAL] Optimal logit threshold: {opt_thr:.2f} (QWK={opt_qwk:.4f})\")\n print(f\" [EVAL] Default threshold 0.0 QWK: \"\n f\"{cohen_kappa_score(y_true_arr, np.sum(final_logits > 0.0, axis=1), weights='quadratic'):.4f}\")\n preds = np.array([ordinal_logits_to_grade(l, thr=opt_thr) for l in final_logits])\n y_true = np.array(labels)\n\n # Core metrics\n acc = accuracy_score(y_true, preds)\n qwk = cohen_kappa_score(y_true, preds, weights=\"quadratic\")\n\n # Per-class + aggregated\n f1_per = f1_score(y_true, preds, average=None, labels=list(range(5)), zero_division=0)\n pr_per = precision_score(y_true, preds, average=None, labels=list(range(5)), zero_division=0)\n re_per = recall_score(y_true, preds, average=None, labels=list(range(5)), zero_division=0)\n\n f1_mac = float(f1_score(y_true, preds, average=\"macro\", zero_division=0))\n f1_wt = float(f1_score(y_true, preds, average=\"weighted\", zero_division=0))\n pr_mac = float(precision_score(y_true, preds, average=\"macro\", zero_division=0))\n pr_wt = float(precision_score(y_true, preds, average=\"weighted\", zero_division=0))\n re_mac = float(recall_score(y_true, preds, average=\"macro\", zero_division=0))\n re_wt = float(recall_score(y_true, preds, average=\"weighted\", zero_division=0))\n\n # Multiclass AUC\n try:\n y_bin = label_binarize(y_true, classes=list(range(5)))\n cp = np.zeros((len(final_probs), 5))\n cp[:, 0] = 1 - final_probs[:, 0]\n cp[:, 1] = final_probs[:, 0] - final_probs[:, 1]\n cp[:, 2] = final_probs[:, 1] - final_probs[:, 2]\n cp[:, 3] = final_probs[:, 2] - final_probs[:, 3]\n cp[:, 4] = final_probs[:, 3]\n auc_mac = float(roc_auc_score(y_bin, np.clip(cp, 0, 1),\n multi_class=\"ovr\", average=\"macro\"))\n except Exception:\n auc_mac = None\n\n # Per-grade sensitivity / specificity\n sens, spec = [], []\n for g in range(5):\n tp = int(((y_true == g) & (preds == g)).sum())\n fn = int(((y_true == g) & (preds != g)).sum())\n tn = int(((y_true != g) & (preds != g)).sum())\n fp = int(((y_true != g) & (preds == g)).sum())\n sens.append(round(tp / (tp + fn + 1e-8), 4))\n spec.append(round(tn / (tn + fp + 1e-8), 4))\n\n print(f\"\\n{'='*55}\\n{desc}\\n{'='*55}\")\n print(f\"Accuracy : {acc:.4f}\")\n print(f\"QWK : {qwk:.4f}\")\n print(f\"AUC macro : {auc_mac:.4f}\" if auc_mac else \"AUC: N/A\")\n print(f\"F1 mac/wt: {f1_mac:.4f} / {f1_wt:.4f}\")\n print(f\"Prec mac/wt:{pr_mac:.4f} / {pr_wt:.4f}\")\n print(f\"Rec mac/wt:{re_mac:.4f} / {re_wt:.4f}\")\n print(\"\\nPer-grade Prec Rec F1 Sens Spec n\")\n for g in range(5):\n n = int((y_true == g).sum())\n print(f\" G{g} {pr_per[g]:.4f} {re_per[g]:.4f} \"\n f\"{f1_per[g]:.4f} {sens[g]:.4f} {spec[g]:.4f} {n}\")\n print(classification_report(y_true, preds,\n target_names=[f\"G{i}\" for i in range(5)]))\n\n result = {\n \"desc\": desc, \"n_samples\": int(len(y_true)),\n \"accuracy\": round(float(acc), 4), \"qwk\": round(float(qwk), 4),\n \"auc_macro_ovr\": round(auc_mac, 4) if auc_mac else None,\n \"f1_macro\": round(f1_mac, 4), \"f1_weighted\": round(f1_wt, 4),\n \"precision_macro\": round(pr_mac, 4), \"precision_weighted\": round(pr_wt, 4),\n \"recall_macro\": round(re_mac, 4), \"recall_weighted\": round(re_wt, 4),\n \"f1_per_grade\": [round(x, 4) for x in f1_per.tolist()],\n \"precision_per_grade\": [round(x, 4) for x in pr_per.tolist()],\n \"recall_per_grade\": [round(x, 4) for x in re_per.tolist()],\n \"sensitivity_per_grade\": sens,\n \"specificity_per_grade\": spec,\n }\n if save_tag:\n path = f\"{OUT_DIR}/eval_{save_tag}.json\"\n with open(path, \"w\") as f:\n json.dump(result, f, indent=2)\n print(f\"[SAVED] {path}\")\n\n result.update({\"probs\": final_probs, \"preds\": preds, \"labels\": y_true})\n return result\n\n\ndef binary_triage(model, val_df: pd.DataFrame, test_df: pd.DataFrame = None):\n \"\"\"Youden-J optimal threshold + full binary metrics.\"\"\"\n out_path = f\"{OUT_DIR}/eval_binary_triage.json\"\n if os.path.exists(out_path):\n print(f\" [SKIP] Found existing binary triage: {out_path}\")\n with open(out_path, \"r\") as f:\n cached = json.load(f)\n rho_cached = float(cached.get(\"val\", {}).get(\"threshold\", 0.5))\n return rho_cached, cached\n\n res = evaluate(model, val_df, desc=\"Val — binary threshold tuning\")\n bin_true = (res[\"labels\"] >= 2).astype(int)\n bin_score = res[\"probs\"][:, 1] # P(grade >= 2)\n\n fpr, tpr, thrs = roc_curve(bin_true, bin_score)\n j_idx = int(np.argmax(tpr - fpr))\n rho = float(thrs[j_idx])\n auc_v = float(roc_auc_score(bin_true, bin_score))\n\n def _bin_metrics(true_bin, score, thr, tag):\n pred_bin = (score >= thr).astype(int)\n tn, fp, fn, tp = confusion_matrix(true_bin, pred_bin, labels=[0,1]).ravel()\n m = {\n \"tag\": tag, \"threshold\": round(thr, 4), \"auc\": round(auc_v, 4),\n \"accuracy\": round(accuracy_score(true_bin, pred_bin), 4),\n \"sensitivity\": round(tp/(tp+fn+1e-8), 4),\n \"specificity\": round(tn/(tn+fp+1e-8), 4),\n \"ppv\": round(tp/(tp+fp+1e-8), 4),\n \"npv\": round(tn/(tn+fn+1e-8), 4),\n \"f1\": round(float(f1_score(true_bin, pred_bin)), 4),\n \"tp\": int(tp), \"tn\": int(tn), \"fp\": int(fp), \"fn\": int(fn),\n }\n print(f\"\\n[BINARY {tag}] thr={thr:.4f} AUC={auc_v:.4f} \"\n f\"Sens={m['sensitivity']:.4f} Spec={m['specificity']:.4f} \"\n f\"F1={m['f1']:.4f}\")\n return m\n\n val_bin = _bin_metrics(bin_true, bin_score, rho, \"Val\")\n out = {\"val\": val_bin}\n\n if test_df is not None and len(test_df) > 0:\n res_te = evaluate(model, test_df, desc=\"Test\")\n te_bin = (res_te[\"labels\"] >= 2).astype(int)\n te_score = res_te[\"probs\"][:, 1]\n te_auc = float(roc_auc_score(te_bin, te_score))\n\n # ROC curve figure\n fpr_t, tpr_t, _ = roc_curve(te_bin, te_score)\n plt.figure(figsize=(6, 5))\n plt.plot(fpr_t, tpr_t, lw=2, color=\"darkorange\",\n label=f\"AUC = {te_auc:.3f}\")\n plt.plot([0,1],[0,1],\"k--\")\n plt.xlabel(\"FPR\"); plt.ylabel(\"TPR\")\n plt.title(\"Binary Triage ROC (IDRiD test)\")\n plt.legend(); plt.grid(alpha=0.3)\n plt.tight_layout()\n p = f\"{OUT_DIR}/fig_binary_roc.png\"\n plt.savefig(p, dpi=150, bbox_inches=\"tight\"); plt.close()\n print(f\"[SAVED] {p}\")\n\n out[\"test\"] = _bin_metrics(te_bin, te_score, rho, \"Test\")\n\n with open(f\"{OUT_DIR}/eval_binary_triage.json\", \"w\") as f:\n json.dump(out, f, indent=2)\n return rho, out","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:30.912792Z","iopub.status.busy":"2026-07-05T19:35:30.912201Z","iopub.status.idle":"2026-07-05T19:35:30.939405Z","shell.execute_reply":"2026-07-05T19:35:30.938865Z"},"papermill":{"duration":0.04075,"end_time":"2026-07-05T19:35:30.940719+00:00","exception":false,"start_time":"2026-07-05T19:35:30.899969+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 14 — Statistical Validation (5 Runs)","metadata":{"papermill":{"duration":0.011294,"end_time":"2026-07-05T19:35:30.963244+00:00","exception":false,"start_time":"2026-07-05T19:35:30.95195+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def run_statistical_validation(train_df, val_df, n_runs=5):\n \"\"\"Train N independent runs. Resumable. Saves mean±std.\"\"\"\n out_file = f\"{OUT_DIR}/statistical_results.json\"\n\n # ── Primary source: completed results JSON ─────────────────────────────\n results = json.load(open(out_file)) if os.path.exists(out_file) else []\n\n # ── Fallback: statistical_results.json may be missing from the input\n # dataset even though stat_run*_best.weights.h5 checkpoints were restored.\n # In that case, re-evaluate from the existing checkpoints so we don't\n # re-train runs that already completed.\n if not results:\n # Also check directly in kaggle input (in case outputs/ wasn't copied)\n for inp_stat in glob.glob(\"/kaggle/input/**/statistical_results.json\", recursive=True):\n try:\n with open(inp_stat) as f:\n results = json.load(f)\n print(f\"[STAT] Loaded {len(results)} prior results from input: {inp_stat}\")\n # Copy it to OUT_DIR so future restores find it\n with open(out_file, \"w\") as f:\n json.dump(results, f, indent=2)\n break\n except Exception:\n pass\n\n # ── Second fallback: checkpoints exist but no JSON at all ─────────────\n # Re-evaluate each completed run from its saved checkpoint weights.\n if len(results) < n_runs:\n completed_runs = set(r[\"run\"] - 1 for r in results) # 0-indexed run ids\n for run_idx in range(n_runs):\n if run_idx in completed_runs:\n continue\n ck = f\"{CKPT_DIR}/stat_run{run_idx}_best.weights.h5\"\n if not os.path.exists(ck):\n continue\n # Checkpoint exists but result is missing → re-evaluate without re-training\n print(f\"\\n[STAT] Checkpoint found for run {run_idx+1} but result missing — \"\n f\"re-evaluating from {os.path.basename(ck)}\")\n with STRATEGY.scope():\n m = build_model(unfreeze_last=-1)\n m.compile(optimizer=keras.optimizers.AdamW(1e-5,\n weight_decay=CFG[\"WEIGHT_DECAY\"], clipnorm=1.0),\n loss=get_loss(), metrics=[ordinal_accuracy])\n m.load_weights(ck)\n res = evaluate(m, val_df, desc=f\"Run {run_idx+1} (re-eval)\")\n results.append({\"run\": run_idx + 1, \"seed\": SEED + run_idx,\n \"accuracy\": res[\"accuracy\"], \"qwk\": res[\"qwk\"]})\n with open(out_file, \"w\") as f:\n json.dump(results, f, indent=2)\n del m\n import gc; gc.collect()\n\n start = len(results)\n if start >= n_runs:\n print(f\"[STAT] All {n_runs} runs already complete — skipping training.\")\n else:\n print(f\"[STAT] Resuming from run {start+1}/{n_runs}.\")\n\n IMG, BS = CFG[\"IMG_SIZE\"], CFG[\"BATCH_SIZE\"]\n cw = get_class_weights(train_df)\n\n for run in range(start, n_runs):\n print(f\"\\n{'='*55}\\nSTAT RUN {run+1}/{n_runs} (seed={SEED+run})\\n{'='*55}\")\n tf.random.set_seed(SEED + run); np.random.seed(SEED + run)\n\n if tfrecords_exist(\"train\") and tfrecords_exist(\"val\"):\n print(f\" [TFR] Using fast TFRecords for stat run {run+1}\")\n tds = make_tf_dataset_fast(\"train\", BS, IMG, augment=True,\n class_weights_map=cw, n_images=len(train_df))\n vds = make_tf_dataset_fast(\"val\", BS, IMG, augment=False, shuffle=False)\n else:\n print(f\" [TFR] Warning: TFRecords not found, using slow path for stat run {run+1}\")\n tds = make_tf_dataset(train_df, BS, IMG, augment=True, class_weights=cw)\n vds = make_tf_dataset(val_df, BS, IMG, augment=False, shuffle=False)\n ck = f\"{CKPT_DIR}/stat_run{run}_best.weights.h5\"\n\n with STRATEGY.scope():\n m = build_model(unfreeze_last=-1)\n m.compile(optimizer=keras.optimizers.AdamW(1e-5, weight_decay=CFG[\"WEIGHT_DECAY\"], clipnorm=1.0),\n loss=get_loss(), metrics=[ordinal_accuracy])\n\n m.fit(tds, validation_data=vds, epochs=20, verbose=0, callbacks=[\n QWKCallback(val_df, BS, IMG),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=5,\n restore_best_weights=True),\n keras.callbacks.ModelCheckpoint(ck, save_best_only=True,\n save_weights_only=True,\n monitor=\"val_ordinal_accuracy\", mode=\"max\"),\n ])\n if os.path.exists(ck): m.load_weights(ck)\n res = evaluate(m, val_df, desc=f\"Run {run+1}\")\n results.append({\"run\": run+1, \"seed\": SEED+run,\n \"accuracy\": res[\"accuracy\"], \"qwk\": res[\"qwk\"]})\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n\n accs = [r[\"accuracy\"] for r in results]\n qwks = [r[\"qwk\"] for r in results]\n summary = {\"n_runs\": n_runs,\n \"accuracy_mean\": round(float(np.mean(accs)), 4),\n \"accuracy_std\": round(float(np.std(accs)), 4),\n \"qwk_mean\": round(float(np.mean(qwks)), 4),\n \"qwk_std\": round(float(np.std(qwks)), 4)}\n print(f\"\\n[STATS] Accuracy: {summary['accuracy_mean']:.4f} ± {summary['accuracy_std']:.4f}\")\n print(f\"[STATS] QWK : {summary['qwk_mean']:.4f} ± {summary['qwk_std']:.4f}\")\n with open(f\"{OUT_DIR}/statistical_summary.json\", \"w\") as f:\n json.dump(summary, f, indent=2)\n auto_zip(tag=\"stat_validation_done\")\n return results","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:30.986971Z","iopub.status.busy":"2026-07-05T19:35:30.986761Z","iopub.status.idle":"2026-07-05T19:35:31.001921Z","shell.execute_reply":"2026-07-05T19:35:31.001152Z"},"papermill":{"duration":0.028738,"end_time":"2026-07-05T19:35:31.003406+00:00","exception":false,"start_time":"2026-07-05T19:35:30.974668+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 15 — Ablation Studies","metadata":{"papermill":{"duration":0.011247,"end_time":"2026-07-05T19:35:31.025999+00:00","exception":false,"start_time":"2026-07-05T19:35:31.014752+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def _run_variant(name, train_df, val_df, use_focal=False, use_mixup=False,\n oversample=False, ckpt_subdir=None):\n \"\"\"Generic single-variant trainer used by all ablation functions.\"\"\"\n ckpt_dir = ckpt_subdir or f\"{CKPT_DIR}/ablation_{name}\"\n best_ckpt = f\"{ckpt_dir}/best.weights.h5\"\n st_file = f\"{ckpt_dir}/state.json\"\n os.makedirs(ckpt_dir, exist_ok=True)\n\n CFG[\"USE_FOCAL\"] = use_focal\n CFG[\"USE_MIXUP\"] = use_mixup\n IMG, BS = CFG[\"IMG_SIZE\"], CFG[\"BATCH_SIZE\"]\n\n eff_train = train_df\n if oversample:\n max_c = train_df[\"label\"].value_counts().max()\n parts = [train_df]\n for c in train_df[\"label\"].unique():\n cls = train_df[train_df[\"label\"] == c]\n n_rep = max_c - len(cls)\n if n_rep > 0:\n parts.append(cls.sample(n_rep, replace=True, random_state=SEED))\n eff_train = pd.concat(parts).sample(frac=1, random_state=SEED).reset_index(drop=True)\n\n cw = get_class_weights(eff_train)\n # FIX: CFG[\"USE_MIXUP\"] was being set above but never actually passed to\n # make_tf_dataset — the \"MixUp\" variant silently trained identically to\n # the baseline. Now actually applies real MixUp (see _mixup_batch_fn,\n # Cell 8).\n tds = make_tf_dataset(eff_train, BS, IMG, augment=True, class_weights=cw,\n mixup=use_mixup)\n vds = make_tf_dataset(val_df, BS, IMG, augment=False, shuffle=False)\n\n best_acc = 0.0\n if os.path.exists(st_file):\n with open(st_file) as f: best_acc = json.load(f).get(\"best_val_acc\", 0.0)\n\n class _VarCB(keras.callbacks.Callback):\n def on_epoch_end(self, epoch, logs=None):\n nonlocal best_acc\n va = float((logs or {}).get(\"val_ordinal_accuracy\", 0.0))\n self.model.save_weights(f\"{ckpt_dir}/ep{epoch:03d}.weights.h5\")\n if va > best_acc:\n best_acc = va\n self.model.save_weights(best_ckpt)\n old = sorted(glob.glob(f\"{ckpt_dir}/ep*.weights.h5\"))[:-2]\n for fp in old:\n try: os.remove(fp)\n except: pass\n with open(st_file, \"w\") as f:\n json.dump({\"epoch\": epoch, \"best_val_acc\": best_acc}, f)\n\n with STRATEGY.scope():\n m = build_model(unfreeze_last=-1)\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n m.compile(optimizer=keras.optimizers.AdamW(1e-5, weight_decay=CFG[\"WEIGHT_DECAY\"]),\n loss=get_loss(), metrics=[ordinal_accuracy])\n\n m.fit(tds, validation_data=vds, epochs=25, verbose=1, callbacks=[\n QWKCallback(val_df, BS, IMG), _VarCB(),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=6,\n restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5,\n patience=3, verbose=0),\n ])\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n CFG[\"USE_FOCAL\"] = False; CFG[\"USE_MIXUP\"] = False\n return evaluate(m, val_df, desc=name)\n\n\ndef run_loss_ablation(train_df, val_df):\n out_file = f\"{OUT_DIR}/loss_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n variants = [\n (\"OrdinalBCE\", False, False, False),\n (\"FocalLoss\", True, False, False),\n (\"MixUp\", False, True, False),\n (\"Oversampling\",False, False, True),\n ]\n for name, focal, mixup, oversamp in variants:\n if name in results:\n print(f\"[SKIP] {name}: acc={results[name]['accuracy']:.4f}\"); continue\n print(f\"\\n{'='*50}\\nLOSS ABLATION: {name}\\n{'='*50}\")\n r = _run_variant(name, train_df, val_df, focal, mixup, oversamp)\n results[name] = {\"accuracy\": r[\"accuracy\"], \"qwk\": r[\"qwk\"]}\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n print(\"\\n[LOSS ABLATION]\")\n for k, v in results.items():\n print(f\" {k:<20} acc={v['accuracy']:.4f} qwk={v['qwk']:.4f}\")\n auto_zip(tag=\"loss_ablation_done\")\n return results\n\n\ndef run_dataset_ablation(val_df):\n out_file = f\"{OUT_DIR}/dataset_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n configs = {\n \"EyePACS_only\": [\"eyepacs\"],\n \"EyePACS_APTOS\": [\"eyepacs\", \"aptos\"],\n \"EyePACS_Messidor\": [\"eyepacs\", \"messidor\"],\n \"Multi_source_All\": [\"eyepacs\", \"aptos\", \"messidor\", \"idrid\"],\n }\n for name, ds_list in configs.items():\n if name in results:\n print(f\"[SKIP] {name}: acc={results[name]['accuracy']:.4f}\"); continue\n print(f\"\\n{'='*50}\\nDATASET ABLATION: {name}\\n{'='*50}\")\n orig = CFG[\"DATASETS\"]; CFG[\"DATASETS\"] = ds_list\n sub = build_dataset_df()\n CFG[\"DATASETS\"] = orig\n sub_tr = sub[~sub[\"image_id\"].isin(val_df[\"image_id\"])].reset_index(drop=True)\n r = _run_variant(name, sub_tr, val_df, ckpt_subdir=f\"{CKPT_DIR}/ds_{name}\")\n results[name] = {\"accuracy\": r[\"accuracy\"], \"qwk\": r[\"qwk\"], \"n_train\": len(sub_tr)}\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n print(\"\\n[DATASET ABLATION]\")\n for k, v in results.items():\n print(f\" {k:<25} n={v['n_train']:>6} acc={v['accuracy']:.4f} qwk={v['qwk']:.4f}\")\n auto_zip(tag=\"dataset_ablation_done\")\n return results\n\n\ndef run_resolution_ablation(train_df, val_df, resolutions=(224, 380, 512)):\n out_file = f\"{OUT_DIR}/resolution_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n bs_map = {224: 32, 380: 16, 512: 8}\n for sz in resolutions:\n key = f\"{sz}px\"\n if key in results:\n print(f\"[SKIP] {key}: acc={results[key]['accuracy']:.4f}\"); continue\n print(f\"\\n{'='*50}\\nRESOLUTION: {sz}×{sz}\\n{'='*50}\")\n bs = bs_map.get(sz, 16) * N_GPUS\n ckd = f\"{CKPT_DIR}/res_{sz}\"\n best = f\"{ckd}/best.weights.h5\"\n os.makedirs(ckd, exist_ok=True)\n cw = get_class_weights(train_df)\n tds = make_tf_dataset(train_df, bs, sz, augment=True, class_weights=cw)\n vds = make_tf_dataset(val_df, bs, sz, augment=False, shuffle=False)\n\n best_acc = 0.0\n class _ResCB(keras.callbacks.Callback):\n def on_epoch_end(self, epoch, logs=None):\n nonlocal best_acc\n va = float((logs or {}).get(\"val_ordinal_accuracy\", 0.0))\n self.model.save_weights(f\"{ckd}/ep{epoch:03d}.weights.h5\")\n if va > best_acc:\n best_acc = va; self.model.save_weights(best)\n old = sorted(glob.glob(f\"{ckd}/ep*.weights.h5\"))[:-2]\n for fp in old:\n try: os.remove(fp)\n except: pass\n\n tf.random.set_seed(SEED)\n # Use B4 for >=380, B0 otherwise\n from tensorflow.keras.applications import EfficientNetB0, EfficientNetB4\n base = (EfficientNetB4 if sz >= 380 else EfficientNetB0)(\n include_top=False, weights=\"imagenet\", input_shape=(sz, sz, 3))\n inp = keras.Input(shape=(sz, sz, 3))\n x = layers.Lambda(effnet_preprocess.preprocess_input,\n name=\"res_preprocess\")(inp)\n x = base(x); x = layers.GlobalAveragePooling2D()(x)\n x = layers.BatchNormalization()(x); x = layers.Dropout(0.5)(x)\n x = layers.Activation(\"linear\", dtype=\"float32\")(x)\n out = layers.Dense(4, name=\"ordinal_head\", dtype=\"float32\")(x)\n m = keras.Model(inp, out)\n if os.path.exists(best): m.load_weights(best)\n m.compile(optimizer=keras.optimizers.AdamW(1e-5, weight_decay=1e-4),\n loss=get_loss(), metrics=[ordinal_accuracy])\n\n # QWK callback using correct size\n class _QWK(keras.callbacks.Callback):\n def on_epoch_end(self, epoch, logs=None):\n vds2 = make_tf_dataset(val_df, bs, sz, augment=False, shuffle=False)\n raw = self.model.predict(vds2, verbose=0)\n true_grades = val_df[\"label\"].values\n opt_thr, opt_qwk = find_optimal_threshold(raw, true_grades)\n logs[\"val_qwk\"] = float(opt_qwk)\n\n m.fit(tds, validation_data=vds, epochs=25, verbose=1, callbacks=[\n _QWK(), _ResCB(),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=6,\n restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=3),\n ])\n if os.path.exists(best): m.load_weights(best)\n\n # Eval + latency\n vds2 = make_tf_dataset(val_df, bs, sz, augment=False, shuffle=False)\n raw = m.predict(vds2, verbose=0)\n true_grades = val_df[\"label\"].values\n opt_thr, opt_qwk = find_optimal_threshold(raw, true_grades)\n preds = np.array([ordinal_logits_to_grade(p, thr=opt_thr) for p in raw])\n acc = float(accuracy_score(true_grades, preds))\n qwk = float(opt_qwk)\n sample = preprocess_image(val_df.iloc[0][\"path\"], sz)[np.newaxis]\n t0 = time.time()\n for _ in range(20): m(sample, training=False)\n lat_ms = (time.time()-t0)/20*1000\n\n results[key] = {\"accuracy\": round(acc,4), \"qwk\": round(qwk,4),\n \"latency_ms\": round(lat_ms,1)}\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n print(f\" {key}: acc={acc:.4f} qwk={qwk:.4f} lat={lat_ms:.1f}ms\")\n\n print(f\"\\n{'Resolution':<12} {'Acc':<10} {'QWK':<10} {'Latency ms'}\")\n for k, v in results.items():\n print(f\" {k:<10} {v['accuracy']:.4f} {v['qwk']:.4f} {v['latency_ms']}\")\n auto_zip(tag=\"resolution_ablation_done\")\n return results\n\nprint(\"[OK] Evaluation & ablation functions defined.\")\n\n\ndef run_backbone_ablation(train_df, val_df,\n backbones=None,\n frozen_epochs=10, unfrozen_epochs=15):\n \"\"\"\n Backbone comparison ablation: trains each architecture with a standardised\n 2-phase schedule (frozen → unfrozen) and evaluates on val_df.\n Results include: accuracy, QWK, AUC macro, F1 macro, latency.\n Saves to backbone_ablation.json (resumable — skips already-done backbones).\n \"\"\"\n out_file = f\"{OUT_DIR}/backbone_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n backbones = backbones or list(BACKBONE_REGISTRY.keys())\n\n for bb_name in backbones:\n if bb_name in results:\n r = results[bb_name]\n print(f\"[SKIP] {bb_name}: acc={r['accuracy']:.4f} \"\n f\"qwk={r['qwk']:.4f}\"); continue\n\n reg = BACKBONE_REGISTRY[bb_name]\n sz = reg[\"img_size\"]\n bs = reg[\"batch_gpu\"] * N_GPUS\n ckd = f\"{CKPT_DIR}/bb_{bb_name}\"\n best_path = f\"{ckd}/best.weights.h5\"\n os.makedirs(ckd, exist_ok=True)\n\n print(f\"\\n{'='*60}\")\n print(f\"BACKBONE ABLATION: {bb_name} @ {sz}px | batch={bs}\")\n print(f\"{'='*60}\")\n\n cw = get_class_weights(train_df)\n # Build TFRecords for this resolution if needed\n tfr_tag_tr = f\"bb_{bb_name}_train\"\n tfr_tag_va = f\"bb_{bb_name}_val\"\n if sz != CFG[\"IMG_SIZE\"]:\n build_tfrecords(train_df, tfr_tag_tr, sz)\n build_tfrecords(val_df, tfr_tag_va, sz)\n tds = make_tf_dataset_fast(tfr_tag_tr, bs, sz, augment=True,\n n_images=len(train_df))\n vds = make_tf_dataset_fast(tfr_tag_va, bs, sz, augment=False,\n shuffle=False)\n elif tfrecords_exist(\"train\") and tfrecords_exist(\"val\"):\n tds = make_tf_dataset_fast(\"train\", bs, sz, augment=True,\n n_images=len(train_df))\n vds = make_tf_dataset_fast(\"val\", bs, sz, augment=False,\n shuffle=False)\n else:\n tds = make_tf_dataset(train_df, bs, sz, augment=True,\n class_weights=cw)\n vds = make_tf_dataset(val_df, bs, sz, augment=False, shuffle=False)\n\n best_acc = 0.0\n class _BbCB(keras.callbacks.Callback):\n def on_epoch_end(self, epoch, logs=None):\n nonlocal best_acc\n va = float((logs or {}).get(\"val_ordinal_accuracy\", 0.0))\n if va > best_acc:\n best_acc = va\n self.model.save_weights(best_path)\n print(f\" [BEST] {bb_name} ep={epoch} acc={va:.4f}\")\n\n class _BbQWK(keras.callbacks.Callback):\n def on_epoch_end(self, epoch, logs=None):\n vds2 = make_tf_dataset(val_df, bs, sz, augment=False, shuffle=False)\n raw = self.model.predict(vds2, verbose=0)\n # FIX: build_model() emits raw logits (no sigmoid — see its\n # docstring), so decoding must compare against 0.0 in logit\n # space, not 0.5 (which was silently treating logits as if\n # they were already probabilities). A full per-epoch optimal-\n # threshold search would be correct but slow; thr=0.0 is the\n # right *default* decision boundary for a logit output and is\n # cheap enough to run every epoch.\n pred = np.array([ordinal_logits_to_grade(p, thr=0.0) for p in raw])\n logs[\"val_qwk\"] = float(cohen_kappa_score(\n val_df[\"label\"].values, pred, weights=\"quadratic\"))\n\n common_cbs = [\n _BbQWK(), _BbCB(),\n keras.callbacks.EarlyStopping(\n monitor=\"val_loss\", patience=5, restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(\n monitor=\"val_loss\", factor=0.5, patience=2, min_lr=1e-8),\n ]\n\n # ── Phase 1: Frozen backbone ──\n print(f\"\\n Phase 1/2: Frozen backbone ({frozen_epochs} epochs)\")\n with STRATEGY.scope():\n m = build_model(unfreeze_last=0, backbone_name=bb_name)\n m.compile(optimizer=keras.optimizers.AdamW(1e-3, weight_decay=1e-4),\n loss=ordinal_bce_loss, metrics=[ordinal_accuracy])\n m.fit(tds, validation_data=vds, epochs=frozen_epochs,\n verbose=1, callbacks=common_cbs)\n\n # ── Phase 2: Full unfreeze ──\n print(f\"\\n Phase 2/2: Full unfreeze ({unfrozen_epochs} epochs)\")\n with STRATEGY.scope():\n m2 = build_model(unfreeze_last=-1, backbone_name=bb_name)\n if os.path.exists(best_path):\n m2.load_weights(best_path)\n m2.compile(optimizer=keras.optimizers.AdamW(1e-5, weight_decay=1e-4),\n loss=ordinal_bce_loss, metrics=[ordinal_accuracy])\n m2.fit(tds, validation_data=vds, epochs=unfrozen_epochs,\n verbose=1, callbacks=common_cbs)\n if os.path.exists(best_path):\n m2.load_weights(best_path)\n\n # ── Evaluate ──\n # FIX (important — this is the block that produced every number in\n # backbone_ablation.json to date): `raw` is a raw-logit output from\n # build_model() (\"No sigmoid! Output raw logits for numerical\n # stability\" — see that function's docstring). The previous code\n # here (a) decoded with ordinal_to_grade(p) at the default thr=0.5,\n # which is a probability-space threshold silently applied to\n # unbounded logits instead of the correct thr=0.0, and (b) computed\n # AUC by treating `raw` itself as if it were already a probability\n # in the cumulative-class-prob formula (1 - raw[:,0], etc.), which\n # is only valid after a sigmoid. Both are fixed below to match the\n # exact convention evaluate() and run_resolution_ablation already use.\n vds_eval = make_tf_dataset(val_df, bs, sz, augment=False, shuffle=False)\n raw = m2.predict(vds_eval, verbose=0)\n true = val_df[\"label\"].values\n opt_thr, _ = find_optimal_threshold(raw, true)\n preds = np.array([ordinal_logits_to_grade(p, thr=opt_thr) for p in raw])\n acc = float(accuracy_score(true, preds))\n qwk = float(cohen_kappa_score(true, preds, weights=\"quadratic\"))\n f1_m = float(f1_score(true, preds, average=\"macro\", zero_division=0))\n try:\n true_bin = label_binarize(true, classes=list(range(5)))\n probs = 1.0 / (1.0 + np.exp(-raw)) # sigmoid BEFORE the cumulative-diff formula\n cp = np.zeros((len(probs), 5))\n cp[:, 0] = 1 - probs[:, 0]\n cp[:, 1] = probs[:, 0] - probs[:, 1]\n cp[:, 2] = probs[:, 1] - probs[:, 2]\n cp[:, 3] = probs[:, 2] - probs[:, 3]\n cp[:, 4] = probs[:, 3]\n auc_macro = float(roc_auc_score(true_bin, np.clip(cp, 0, 1),\n multi_class=\"ovr\", average=\"macro\"))\n except Exception:\n auc_macro = 0.0\n\n # Latency\n sample = preprocess_image(val_df.iloc[0][\"path\"], sz)[np.newaxis]\n t0 = time.time()\n for _ in range(20): m2(sample, training=False)\n lat_ms = (time.time()-t0)/20*1000\n\n # Count params\n n_params = sum(np.prod(v.shape) for v in m2.trainable_variables)\n\n results[bb_name] = {\n \"accuracy\": round(acc, 4),\n \"qwk\": round(qwk, 4),\n \"f1_macro\": round(f1_m, 4),\n \"auc_macro\": round(auc_macro, 4),\n \"latency_ms\": round(lat_ms, 1),\n \"params_M\": round(n_params / 1e6, 1),\n \"img_size\": sz,\n \"decode_threshold\": round(float(opt_thr), 3),\n \"eval_protocol\": \"raw predict + optimal logit threshold (no MC-Dropout/TTA) — \"\n \"NOT directly comparable to evaluate()'s MC-Dropout+TTA numbers; \"\n \"see Cell 13 for the MC-Dropout+TTA protocol used for the headline result.\",\n }\n with open(out_file, \"w\") as f:\n json.dump(results, f, indent=2)\n print(f\"\\n {bb_name}: acc={acc:.4f} qwk={qwk:.4f} \"\n f\"auc={auc_macro:.4f} lat={lat_ms:.1f}ms\")\n auto_zip(tag=f\"bb_abl_{bb_name}\")\n\n # Clean up GPU memory\n del m, m2\n keras.backend.clear_session()\n tf.random.set_seed(SEED)\n\n # ── Print comparison table ──\n print(f\"\\n{'='*80}\")\n print(f\"{'Backbone':<20} {'Params':>8} {'Img':>5} {'Acc':>7} \"\n f\"{'QWK':>7} {'AUC':>7} {'F1':>7} {'Lat ms':>8}\")\n print(f\"{'-'*80}\")\n for bb, r in sorted(results.items(), key=lambda x: -x[1][\"qwk\"]):\n print(f\" {bb:<18} {r['params_M']:>6.1f}M {r['img_size']:>4}px \"\n f\"{r['accuracy']:>6.4f} {r['qwk']:>6.4f} \"\n f\"{r['auc_macro']:>6.4f} {r['f1_macro']:>6.4f} \"\n f\"{r['latency_ms']:>7.1f}\")\n print(f\"{'='*80}\")\n auto_zip(tag=\"backbone_ablation_done\")\n return results","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:31.049793Z","iopub.status.busy":"2026-07-05T19:35:31.049284Z","iopub.status.idle":"2026-07-05T19:35:31.092464Z","shell.execute_reply":"2026-07-05T19:35:31.091834Z"},"papermill":{"duration":0.056783,"end_time":"2026-07-05T19:35:31.093882+00:00","exception":false,"start_time":"2026-07-05T19:35:31.037099+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 16 — XAI: Saliency Maps (GradCAM / GradCAM++ / EigenCAM / ScoreCAM / SHAP / LIME)","metadata":{"papermill":{"duration":0.01113,"end_time":"2026-07-05T19:35:31.116329+00:00","exception":false,"start_time":"2026-07-05T19:35:31.105199+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def _overlay(img_np: np.ndarray, cam: np.ndarray, alpha: float = 0.45) -> np.ndarray:\n hm = cv2.applyColorMap(np.uint8(255 * cam), cv2.COLORMAP_JET)\n hm = cv2.cvtColor(hm, cv2.COLOR_BGR2RGB) / 255.0\n img = img_np / 255.0 if img_np.max() > 1.0 else img_np\n return np.clip(alpha * hm + (1 - alpha) * img, 0, 1)\n\ndef get_grad_model(model: keras.Model) -> keras.Model:\n return keras.Model(\n inputs=model.inputs,\n outputs=[model.get_layer(CFG[\"CAM_LAYER\"]).output, model.output],\n )\n\ndef saliency_gradcam(model, img_np, head_idx=1, grad_model=None):\n gm = grad_model or get_grad_model(model)\n img = img_np[np.newaxis]\n with tf.GradientTape() as tape:\n tape.watch(img)\n conv_out, preds = gm(img, training=False)\n score = preds[0, head_idx]\n grads = tape.gradient(score, conv_out)\n weights = tf.reduce_mean(grads, axis=(1, 2))[0].numpy()\n cam = (weights * conv_out[0].numpy()).sum(axis=-1)\n cam = np.maximum(cam, 0)\n cam = cv2.resize(cam, (img_np.shape[0], img_np.shape[1]))\n return (cam - cam.min()) / (cam.max() - cam.min() + 1e-8)\n\ndef saliency_gradcampp(model, img_np, head_idx=1, grad_model=None):\n gm = grad_model or get_grad_model(model)\n img = img_np[np.newaxis]\n with tf.GradientTape() as tape:\n tape.watch(img)\n conv_out, preds = gm(img, training=False)\n score = preds[0, head_idx]\n grads = tape.gradient(score, conv_out)\n g2 = grads ** 2\n g3 = grads ** 3\n co_np = conv_out[0].numpy()\n alpha = g2[0].numpy() / (2*g2[0].numpy() +\n (co_np * g3[0].numpy()).sum(axis=(0,1), keepdims=True) + 1e-8)\n weights = (alpha * tf.nn.relu(grads[0]).numpy()).sum(axis=(0, 1))\n cam = (weights * co_np).sum(axis=-1)\n cam = np.maximum(cam, 0)\n cam = cv2.resize(cam, (img_np.shape[0], img_np.shape[1]))\n return (cam - cam.min()) / (cam.max() - cam.min() + 1e-8)\n\ndef saliency_eigencam(model, img_np, grad_model=None):\n gm = grad_model or get_grad_model(model)\n img = img_np[np.newaxis]\n co, _ = gm(img, training=False)\n feat = co[0].numpy().reshape(-1, co.shape[-1])\n feat -= feat.mean(axis=0)\n _, _, vt = np.linalg.svd(feat, full_matrices=False)\n cam = (feat @ vt[0]).reshape(co.shape[1], co.shape[2])\n cam = cv2.resize(np.maximum(cam, 0), (img_np.shape[0], img_np.shape[1]))\n return (cam - cam.min()) / (cam.max() - cam.min() + 1e-8)\n\ndef saliency_scorecam(model, img_np, head_idx=1, n_masks=48, grad_model=None):\n gm = grad_model or get_grad_model(model)\n sz = img_np.shape[0]\n img = img_np[np.newaxis]\n co, _ = gm(img, training=False)\n n_ch = co.shape[-1]\n step = max(1, n_ch // n_masks)\n chs = list(range(0, n_ch, step))[:n_masks]\n ws = []\n for ch in chs:\n act = cv2.resize(co[0, :, :, ch].numpy(), (sz, sz))\n act = (act - act.min()) / (act.max() - act.min() + 1e-8)\n msk = img_np * act[:, :, np.newaxis]\n sc = float(model(msk[np.newaxis], training=False).numpy()[0, head_idx])\n ws.append(sc)\n cam = np.zeros((co.shape[1], co.shape[2]))\n for i, ch in enumerate(chs):\n cam += ws[i] * co[0, :, :, ch].numpy()\n cam = cv2.resize(np.maximum(cam, 0), (sz, sz))\n return (cam - cam.min()) / (cam.max() - cam.min() + 1e-8)\n\ndef saliency_shap(model, img_np, head_idx=1):\n import shap\n sz = img_np.shape[0]\n inp = model.inputs[0]\n out = model.output[:, head_idx:head_idx+1]\n sm = keras.Model(inp, out)\n bg = np.zeros((1, sz, sz, 3), dtype=np.float32)\n try:\n ex = shap.DeepExplainer(sm, bg)\n sv = ex.shap_values(img_np[np.newaxis])\n cam = np.abs(sv[0][0]).mean(axis=-1) if isinstance(sv, list) else np.abs(sv[0]).mean(axis=-1)\n except Exception:\n ex = shap.GradientExplainer(sm, bg)\n sv = ex.shap_values(img_np[np.newaxis])\n cam = np.abs(sv[0]).mean(axis=-1) if not isinstance(sv, list) else np.abs(sv[0][0]).mean(axis=-1)\n cam = cv2.resize(cam.astype(np.float32), (sz, sz))\n return (cam - cam.min()) / (cam.max() - cam.min() + 1e-8)\n\ndef saliency_lime(model, img_np, head_idx=1, n_segs=50, n_samp=300):\n from lime import lime_image\n sz = img_np.shape[0]\n def pfn(imgs):\n imgs_f = imgs.astype(np.float32)\n if imgs_f.max() <= 1.0: imgs_f *= 255.0\n logits = model(imgs_f, training=False).numpy()\n # FIX: model outputs raw logits (build_model has no sigmoid on its\n # head), so convert to probabilities before the cumulative-diff\n # formula below — otherwise this feeds LIME's linear surrogate a\n # meaningless \"probability\" derived straight from an unbounded logit.\n probs = 1.0 / (1.0 + np.exp(-logits))\n gp = np.zeros((len(imgs_f), 5))\n gp[:,0]=1-probs[:,0]; gp[:,1]=probs[:,0]-probs[:,1]\n gp[:,2]=probs[:,1]-probs[:,2]; gp[:,3]=probs[:,2]-probs[:,3]\n gp[:,4]=probs[:,3]\n return np.clip(gp, 0, 1)\n img_u8 = img_np.astype(np.uint8) if img_np.max() > 1.0 else (img_np*255).astype(np.uint8)\n # FIX: ordinal_to_grade(..., default thr=0.5) is the probability-space\n # decoder; the model's raw output here is logits, so use the logit-space\n # decoder at its correct thr=0.0 boundary instead (this only picks which\n # label LIME explains, not a reported metric, but should still be right).\n label = min(ordinal_logits_to_grade(model(img_np[np.newaxis], training=False).numpy()[0], thr=0.0), 4)\n exp = lime_image.LimeImageExplainer(random_state=SEED)\n expl = exp.explain_instance(img_u8, pfn, top_labels=5, hide_color=0,\n num_samples=n_samp, num_features=n_segs, batch_size=32)\n d = dict(expl.local_exp[label])\n cam = np.vectorize(d.get)(expl.segments).astype(np.float32)\n cam = cv2.resize(cam, (sz, sz))\n return (cam - cam.min()) / (cam.max() - cam.min() + 1e-8)\n\n\nSALIENCY_FNS = {\n \"GradCAM\": saliency_gradcam,\n \"GradCAM++\": saliency_gradcampp,\n \"EigenCAM\": lambda m, img, hi, gm: saliency_eigencam(m, img, gm),\n \"ScoreCAM\": saliency_scorecam,\n \"SHAP\": lambda m, img, hi, gm: saliency_shap(m, img, hi),\n \"LIME\": lambda m, img, hi, gm: saliency_lime(m, img, hi),\n}\n\ndef get_saliency(model, img_np, method: str, head_idx: int = 1,\n grad_model=None) -> np.ndarray:\n try:\n return SALIENCY_FNS[method](model, img_np, head_idx, grad_model)\n except Exception as e:\n print(f\" [WARN] {method} failed: {e} — falling back to GradCAM\")\n return saliency_gradcam(model, img_np, head_idx, grad_model)","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:31.140296Z","iopub.status.busy":"2026-07-05T19:35:31.139819Z","iopub.status.idle":"2026-07-05T19:35:31.162856Z","shell.execute_reply":"2026-07-05T19:35:31.162078Z"},"papermill":{"duration":0.03679,"end_time":"2026-07-05T19:35:31.164302+00:00","exception":false,"start_time":"2026-07-05T19:35:31.127512+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 17 — XAI Grid (5 Grades × 4 CAM Methods)","metadata":{"papermill":{"duration":0.011311,"end_time":"2026-07-05T19:35:31.186847+00:00","exception":false,"start_time":"2026-07-05T19:35:31.175536+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def plot_xai_grid(model, df: pd.DataFrame):\n \"\"\"5 DR grades × (Original + GradCAM + GradCAM++ + EigenCAM + ScoreCAM).\"\"\"\n gm = get_grad_model(model)\n methods = [\"GradCAM\", \"GradCAM++\", \"EigenCAM\", \"ScoreCAM\"]\n n_cols = len(methods) + 1\n fig, axes = plt.subplots(5, n_cols, figsize=(3.5*n_cols, 4*5))\n\n for grade in range(5):\n gdf = df[df[\"label\"] == grade]\n if len(gdf) == 0:\n for ax in axes[grade]: ax.axis(\"off\")\n continue\n row = gdf.sample(1, random_state=SEED).iloc[0]\n img = preprocess_image(row[\"path\"], CFG[\"IMG_SIZE\"])\n hi = min(grade, 3)\n\n axes[grade, 0].imshow(img.astype(np.uint8))\n axes[grade, 0].set_title(f\"Grade {grade}\\n(Original)\", fontsize=11)\n axes[grade, 0].axis(\"off\")\n\n for j, meth in enumerate(methods):\n cam = get_saliency(model, img, meth, hi, gm)\n axes[grade, j+1].imshow(_overlay(img, cam))\n axes[grade, j+1].set_title(meth, fontsize=11)\n axes[grade, j+1].axis(\"off\")\n\n plt.suptitle(f\"Multi-Method XAI — {CFG['ACTIVE_BB']} — All DR Grades\",\n fontsize=14, fontweight=\"bold\")\n plt.tight_layout()\n path = f\"{OUT_DIR}/fig_xai_all_grades.png\"\n plt.savefig(path, dpi=150, bbox_inches=\"tight\"); plt.close()\n print(f\"[SAVED] {path}\")","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:31.211213Z","iopub.status.busy":"2026-07-05T19:35:31.210676Z","iopub.status.idle":"2026-07-05T19:35:31.21742Z","shell.execute_reply":"2026-07-05T19:35:31.216872Z"},"papermill":{"duration":0.020431,"end_time":"2026-07-05T19:35:31.218772+00:00","exception":false,"start_time":"2026-07-05T19:35:31.198341+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 18 — Insertion / Deletion AUC (Quantitative XAI Faithfulness)","metadata":{"papermill":{"duration":0.011357,"end_time":"2026-07-05T19:35:31.241428+00:00","exception":false,"start_time":"2026-07-05T19:35:31.230071+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def _ins_del_single(model, img_np, saliency, head_idx, steps=20):\n \"\"\"Insertion and Deletion AUC for one image.\"\"\"\n n_px = img_np.shape[0] ** 2\n order = np.argsort(saliency.ravel())[::-1] # most important first\n ins_sc, del_sc = [], []\n for s in range(steps + 1):\n n = int(n_px * s / steps)\n mask = np.zeros(n_px, dtype=bool); mask[order[:n]] = True\n mask2d = mask.reshape(img_np.shape[0], img_np.shape[0])\n ins = np.zeros_like(img_np); ins[mask2d] = img_np[mask2d]\n dlt = img_np.copy(); dlt[mask2d] = 0\n ins_sc.append(float(model(ins[np.newaxis], training=False).numpy()[0, head_idx]))\n del_sc.append(float(model(dlt[np.newaxis], training=False).numpy()[0, head_idx]))\n return float(np.trapz(ins_sc)/steps), float(np.trapz(del_sc)/steps)\n\n\ndef compute_xai_faithfulness(model, df: pd.DataFrame, n_samples=30, steps=20):\n \"\"\"\n Insertion / Deletion AUC for all 6 XAI methods on Grade>=2 images.\n Faithfulness = Insertion_AUC - Deletion_AUC (higher = better).\n Fully resumable.\n \"\"\"\n out_file = f\"{OUT_DIR}/xai_faithfulness.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n\n methods = [\"GradCAM\", \"GradCAM++\", \"EigenCAM\", \"ScoreCAM\", \"SHAP\", \"LIME\"]\n pos_df = df[df[\"label\"] >= 2].sample(\n min(n_samples, len(df[df[\"label\"]>=2])), random_state=SEED)\n gm = get_grad_model(model)\n\n for meth in methods:\n if meth in results:\n v = results[meth]\n print(f\"[SKIP] {meth}: ins={v['insertion_auc']:.3f} \"\n f\"del={v['deletion_auc']:.3f}\"); continue\n\n print(f\"\\n[XAI FAITH] {meth} | {len(pos_df)} images | {steps} steps\")\n ins_l, del_l = [], []\n for _, row in pos_df.iterrows():\n img = preprocess_image(row[\"path\"], CFG[\"IMG_SIZE\"])\n hi = min(int(row[\"label\"])-1, 3)\n try:\n sal = get_saliency(model, img, meth, hi, gm)\n ins, dl = _ins_del_single(model, img, sal, hi, steps)\n ins_l.append(ins); del_l.append(dl)\n except Exception as e:\n print(f\" [WARN] {row['image_id']}: {e}\")\n\n if ins_l:\n results[meth] = {\n \"insertion_auc\": round(float(np.mean(ins_l)), 4),\n \"deletion_auc\": round(float(np.mean(del_l)), 4),\n \"insertion_std\": round(float(np.std(ins_l)), 4),\n \"deletion_std\": round(float(np.std(del_l)), 4),\n \"faithfulness\": round(float(np.mean(ins_l)-np.mean(del_l)), 4),\n \"n_images\": len(ins_l),\n }\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n\n # Summary table\n print(f\"\\n{'Method':<14} {'Ins AUC':>9} {'Del AUC':>9} {'Faithful':>10} {'N':>5}\")\n print(\"-\"*55)\n for m, v in sorted(results.items(), key=lambda x: -x[1][\"faithfulness\"]):\n print(f\"{m:<14} {v['insertion_auc']:>9.3f} {v['deletion_auc']:>9.3f} \"\n f\"{v['faithfulness']:>10.3f} {v['n_images']:>5}\")\n\n # Bar chart\n labs = list(results.keys())\n fig, axes = plt.subplots(1, 3, figsize=(15, 4))\n axes[0].bar(labs, [results[m][\"insertion_auc\"] for m in labs], color=\"#2196F3\")\n axes[0].set_title(\"Insertion AUC ↑\"); axes[0].set_ylim(0,1); axes[0].tick_params(axis=\"x\", rotation=30)\n axes[1].bar(labs, [results[m][\"deletion_auc\"] for m in labs], color=\"#F44336\")\n axes[1].set_title(\"Deletion AUC ↓\"); axes[1].set_ylim(0,1); axes[1].tick_params(axis=\"x\", rotation=30)\n axes[2].bar(labs, [results[m][\"faithfulness\"] for m in labs], color=\"#4CAF50\")\n axes[2].set_title(\"Faithfulness (Ins−Del) ↑\"); axes[2].tick_params(axis=\"x\", rotation=30)\n plt.suptitle(\"Quantitative XAI Faithfulness — Grade ≥ 2\", fontsize=13)\n plt.tight_layout()\n path = f\"{OUT_DIR}/fig_xai_faithfulness.png\"\n plt.savefig(path, dpi=150, bbox_inches=\"tight\"); plt.close()\n print(f\"[SAVED] {path}\")\n auto_zip(tag=\"xai_faithfulness_done\")\n return results","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:31.266441Z","iopub.status.busy":"2026-07-05T19:35:31.266187Z","iopub.status.idle":"2026-07-05T19:35:31.281181Z","shell.execute_reply":"2026-07-05T19:35:31.280373Z"},"papermill":{"duration":0.028622,"end_time":"2026-07-05T19:35:31.282855+00:00","exception":false,"start_time":"2026-07-05T19:35:31.254233+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 19 — SHAP Grid + LIME Grid (qualitative)","metadata":{"papermill":{"duration":0.011238,"end_time":"2026-07-05T19:35:31.305279+00:00","exception":false,"start_time":"2026-07-05T19:35:31.294041+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def plot_shap_grid(model, val_df):\n import shap\n sz = CFG[\"IMG_SIZE\"]\n bg_paths = val_df[val_df[\"label\"]==0].sample(\n min(30, len(val_df[val_df[\"label\"]==0])), random_state=SEED)[\"path\"].tolist()\n bg = np.array([preprocess_image(p, sz) for p in bg_paths])\n inp = model.inputs[0]; out = model.output[:, 1:2]\n sm = keras.Model(inp, out)\n\n try:\n ex = shap.DeepExplainer(sm, bg)\n except Exception:\n ex = shap.GradientExplainer(sm, bg)\n\n imgs, lbls = [], []\n for g in range(5):\n gdf = val_df[val_df[\"label\"]==g]\n if len(gdf)==0: continue\n row = gdf.sample(1, random_state=SEED).iloc[0]\n imgs.append(preprocess_image(row[\"path\"], sz)); lbls.append(g)\n\n try:\n sv = ex.shap_values(np.array(imgs))\n if isinstance(sv, list): sv = sv[0]\n\n fig, axes = plt.subplots(len(lbls), 3, figsize=(12, 4*len(lbls)))\n if len(lbls)==1: axes = axes[np.newaxis]\n for i, (img, g) in enumerate(zip(imgs, lbls)):\n imp = np.abs(sv[i]).mean(axis=-1)\n imp = (imp - imp.min()) / (imp.max()-imp.min()+1e-8)\n axes[i,0].imshow(img.astype(np.uint8)); axes[i,0].set_title(f\"Grade {g}\", fontsize=11); axes[i,0].axis(\"off\")\n axes[i,1].imshow(img.astype(np.uint8)); axes[i,1].imshow(imp, alpha=0.6, cmap=\"hot\"); axes[i,1].set_title(\"SHAP Overlay\", fontsize=11); axes[i,1].axis(\"off\")\n axes[i,2].imshow(imp, cmap=\"hot\"); axes[i,2].set_title(\"SHAP Map\", fontsize=11); axes[i,2].axis(\"off\")\n plt.suptitle(\"SHAP DeepExplainer — DR Grades\", fontsize=14)\n plt.tight_layout()\n path = f\"{OUT_DIR}/fig_shap_grid.png\"\n plt.savefig(path, dpi=150, bbox_inches=\"tight\"); plt.close()\n print(f\"[SAVED] {path}\")\n except Exception as e:\n print(f\"[WARN] SHAP grid failed: {e}\")\n\n\ndef plot_lime_grid(model, val_df):\n from lime import lime_image\n from skimage.segmentation import mark_boundaries\n sz = CFG[\"IMG_SIZE\"]\n exp = lime_image.LimeImageExplainer(random_state=SEED)\n\n def pfn(imgs):\n imgs_f = imgs.astype(np.float32)\n if imgs_f.max() <= 1.0: imgs_f *= 255.0\n logits = model(imgs_f, training=False).numpy()\n # FIX: sigmoid before the cumulative-diff formula — same issue as\n # saliency_lime() in Cell 16 (model emits raw logits, not probs).\n probs = 1.0 / (1.0 + np.exp(-logits))\n gp = np.zeros((len(imgs_f), 5))\n gp[:,0]=1-probs[:,0]; gp[:,1]=probs[:,0]-probs[:,1]\n gp[:,2]=probs[:,1]-probs[:,2]; gp[:,3]=probs[:,2]-probs[:,3]; gp[:,4]=probs[:,3]\n return np.clip(gp, 0, 1)\n\n fig, axes = plt.subplots(5, 3, figsize=(12, 20))\n for g in range(5):\n gdf = val_df[val_df[\"label\"]==g]\n if len(gdf)==0:\n for ax in axes[g]: ax.axis(\"off\"); continue\n row = gdf.sample(1, random_state=SEED).iloc[0]\n img = preprocess_image(row[\"path\"], sz)\n img_u8 = img.astype(np.uint8)\n # FIX: logit-space decode at thr=0.0 (model emits raw logits).\n pred_g = ordinal_logits_to_grade(model(img[np.newaxis], training=False).numpy()[0], thr=0.0)\n label = min(pred_g, 4)\n try:\n expl = exp.explain_instance(img_u8, pfn, top_labels=5, hide_color=0,\n num_samples=300, num_features=40, batch_size=32)\n temp, mask = expl.get_image_and_mask(label, positive_only=True,\n num_features=10, hide_rest=False)\n d = dict(expl.local_exp[label])\n hm = np.vectorize(d.get)(expl.segments).astype(np.float32)\n hm = (hm-hm.min())/(hm.max()-hm.min()+1e-8)\n axes[g,0].imshow(img_u8); axes[g,0].set_title(f\"Grade {g}\", fontsize=11); axes[g,0].axis(\"off\")\n axes[g,1].imshow(mark_boundaries(temp/255.0, mask)); axes[g,1].set_title(f\"LIME (pred G{pred_g})\", fontsize=11); axes[g,1].axis(\"off\")\n axes[g,2].imshow(hm, cmap=\"RdYlGn\"); axes[g,2].set_title(\"LIME Heatmap\", fontsize=11); axes[g,2].axis(\"off\")\n except Exception as e:\n print(f\" [WARN] LIME Grade {g}: {e}\")\n for ax in axes[g]: ax.axis(\"off\")\n\n plt.suptitle(\"LIME Explainability — All DR Grades\", fontsize=14)\n plt.tight_layout()\n path = f\"{OUT_DIR}/fig_lime_grid.png\"\n plt.savefig(path, dpi=150, bbox_inches=\"tight\"); plt.close()\n print(f\"[SAVED] {path}\")\n\nprint(\"[OK] XAI functions defined.\")","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:31.329308Z","iopub.status.busy":"2026-07-05T19:35:31.328869Z","iopub.status.idle":"2026-07-05T19:35:31.346166Z","shell.execute_reply":"2026-07-05T19:35:31.345407Z"},"papermill":{"duration":0.031154,"end_time":"2026-07-05T19:35:31.347738+00:00","exception":false,"start_time":"2026-07-05T19:35:31.316584+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 20 — Confusion Matrix & Convergence Plots","metadata":{"papermill":{"duration":0.01105,"end_time":"2026-07-05T19:35:31.370004+00:00","exception":false,"start_time":"2026-07-05T19:35:31.358954+00:00","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def plot_confusion_matrix(y_true, y_pred, title=\"Confusion Matrix\", save_tag=\"cm\"):\n from sklearn.metrics import ConfusionMatrixDisplay\n cm = confusion_matrix(y_true, y_pred, labels=list(range(5)))\n cm_n = cm.astype(float) / (cm.sum(axis=1, keepdims=True) + 1e-8)\n fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n ConfusionMatrixDisplay(cm, display_labels=[f\"G{i}\" for i in range(5)]).plot(ax=axes[0], colorbar=False)\n ConfusionMatrixDisplay(cm_n, display_labels=[f\"G{i}\" for i in range(5)]).plot(ax=axes[1], colorbar=False, values_format=\".2f\")\n axes[0].set_title(f\"{title} (counts)\"); axes[1].set_title(f\"{title} (normalised)\")\n plt.suptitle(title, fontweight=\"bold\"); plt.tight_layout()\n path = f\"{OUT_DIR}/fig_{save_tag}.png\"\n plt.savefig(path, dpi=150, bbox_inches=\"tight\"); plt.close()\n print(f\"[SAVED] {path}\")","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:31.393327Z","iopub.status.busy":"2026-07-05T19:35:31.393131Z","iopub.status.idle":"2026-07-05T19:35:31.398956Z","shell.execute_reply":"2026-07-05T19:35:31.398385Z"},"papermill":{"duration":0.019159,"end_time":"2026-07-05T19:35:31.40033+00:00","exception":false,"start_time":"2026-07-05T19:35:31.381171+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cell 21 — ═══════════════ MAIN EXECUTION ═══════════════\n> **Run all cells above first, then execute this cell.**\n> The pipeline is fully resumable — re-running picks up from where it stopped.","metadata":{"papermill":{"duration":0.011423,"end_time":"2026-07-05T19:35:31.423292+00:00","exception":false,"start_time":"2026-07-05T19:35:31.411869+00:00","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"> **⚠️ Note on the output already attached to the next cell (Main Execution):** the printed log below this line (if present) is from a run captured *before* the fixes applied above — most importantly, the backbone-ablation accuracy/QWK/AUC numbers in that log were computed with a decode-threshold bug (probability-space threshold applied to raw logits) that has now been corrected in Cell 15. Treat any numbers in the old output as superseded, and re-run this notebook top-to-bottom to regenerate them correctly. This cell's own code is unchanged — only the functions it calls were fixed.","metadata":{}},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────────────────────\n# STEP 1: Load datasets & create patient-level splits\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"=\"*60)\nprint(\"STEP 1 — Dataset loading & patient-level splits\")\nprint(\"=\"*60)\n\nprint(\"\\nLoading datasets:\")\ndf_all = build_dataset_df()\n\nprint(\"\\nCreating/loading patient-level splits:\")\ntrain_df, val_df, test_df = make_splits(df_all)\n\n# IDRiD test set (external holdout)\ntry:\n idrid_test_df = load_idrid(\"test\")\n if len(idrid_test_df) == 0:\n idrid_test_df = None\n else:\n print(f\" IDRiD external test: {len(idrid_test_df)} images\")\nexcept Exception:\n idrid_test_df = None\n print(\" [WARN] No separate IDRiD test set found — using internal test split.\")\n\n# ─────────────────────────────────────────────────────────────────────────────\n# STEP 1b: Build TFRecords (default resolution — one-time preprocessing)\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 1b — Building TFRecords (preprocess once, train fast)\")\nprint(\"=\"*60)\nIMG = CFG[\"IMG_SIZE\"]\nbuild_tfrecords(train_df, \"train\", IMG)\nbuild_tfrecords(val_df, \"val\", IMG)\nbuild_tfrecords(test_df, \"test\", IMG)\nif idrid_test_df is not None:\n build_tfrecords(idrid_test_df, \"idrid_test\", IMG)\nauto_zip(tag=\"tfrecords_built\")\n\n# ═══════════════════════════════════════════════════════════════════════════\n# SEARCH PHASE — everything below runs BEFORE the expensive 3-phase\n# progressive-unfreeze commitment. Rationale: the full schedule (8+25+40\n# epochs) plus 5-run statistical validation is by far the most expensive\n# thing this notebook does — it should only be spent on a configuration\n# that's already been shown competitive by cheaper, shorter runs, not on\n# whatever CFG[\"ACTIVE_BB\"] happened to default to.\n#\n# Two distinct kinds of comparison happen here, and it matters which is\n# which:\n# BENCHMARK (across categories) — fundamentally different modelling\n# strategies compared against each other: fine-tuned CNN/hybrid\n# backbones vs. fine-tuned Transformers vs. frozen foundation-\n# model linear probes. These are not apples-to-apples with each\n# other (frozen linear-probing is a deliberately weaker regime\n# than full fine-tuning), so results are reported side by side\n# but NOT blindly ranked as if paradigm didn't matter.\n# ABLATION (within a category) — variants of the SAME strategy compared\n# against each other: which of the 9 CNN backbones, which loss\n# function, which training-data composition. These ARE\n# apples-to-apples and a single winner is meaningful.\n#\n# What can run here, mechanically: Category A (9 CNN/hybrid backbones, via\n# BACKBONE_REGISTRY) and Category B's ViT-Small are both DEFINED earlier in\n# this notebook (Cell 3), so they're available now. Swin-Tiny (Category B)\n# and the foundation-model linear probes (Category C) are defined in\n# Extended Cells J and N, which — by plain top-to-bottom notebook order —\n# don't exist yet at this point in a fresh run. They're still compared,\n# just later (Extended Main Execution has a Cross-Category Leaderboard step\n# that pulls ALL of this together once those cells have run too). Because\n# of that ordering constraint, only Category A can mechanically feed back\n# into CFG[\"ACTIVE_BB\"] here — ViT/Swin/foundation probes use different\n# code paths (ViTOrdinalModel / SwinOrdinalModel / a linear-probe head) that\n# this pipeline's train()/evaluate()/statistical-validation functions don't\n# know how to drive. If one of them scores higher, that's valuable\n# information (printed clearly below) — treat it as \"worth a dedicated\n# fine-tuning run,\" not as something this cell will silently swap in.\n# ═══════════════════════════════════════════════════════════════════════════\n\n# ─────────────────────────────────────────────────────────────────────────────\n# STEP 2: CATEGORY A — CNN/Hybrid Backbone Architectures (within-category ablation)\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 2 — CATEGORY A benchmark: CNN/Hybrid backbones (9 architectures)\")\nprint(\"=\"*60)\ntry:\n bb_abl = run_backbone_ablation(train_df, val_df)\nexcept Exception as e:\n print(f\"[ERROR] Backbone ablation failed: {e}\")\n bb_abl = {}\nauto_zip(tag=\"after_backbone_ablation\")\n\n# ─────────────────────────────────────────────────────────────────────────────\n# STEP 3: CATEGORY B (partial) — ViT-Small Transformer baseline\n# Swin-Tiny joins this category later (Extended Cell J); see the header note\n# above for why it can't run from this cell on a fresh top-to-bottom pass.\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 3 — CATEGORY B benchmark (partial): ViT-Small transformer baseline\")\nprint(\"=\"*60)\ntry:\n bb_abl = run_vit_comparison(train_df, val_df)\nexcept Exception as e:\n print(f\"[ERROR] ViT comparison failed: {e}\")\nauto_zip(tag=\"after_vit_comparison\")\n\n# ─────────────────────────────────────────────────────────────────────────────\n# STEP 4: WINNER SELECTION — pick the architecture that gets the expensive\n# training run. Constrained to Category A (BACKBONE_REGISTRY entries) for\n# the mechanical reason above; ViT-Small (and later Swin / foundation\n# probes) are reported for comparison regardless.\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 4 — Cross-category leaderboard so far + winner selection\")\nprint(\"=\"*60)\n\ndef _select_winning_backbone(all_results, registry_keys, metric=\"qwk\"):\n \"\"\"\n Ranks every completed entry in `all_results` by `metric`, prints a\n clearly categorised leaderboard (fine-tuned CNN/hybrid vs. fine-tuned\n Transformer vs. frozen linear-probe, inferred from the key name and\n whether it's a BACKBONE_REGISTRY member), and returns the best backbone\n NAME that is actually in `registry_keys` (i.e. actually usable as\n CFG[\"ACTIVE_BB\"]). Falls back to the current CFG[\"ACTIVE_BB\"] if no\n registry-eligible result exists yet.\n \"\"\"\n valid = {k: v for k, v in all_results.items() if isinstance(v, dict) and metric in v}\n if not valid:\n print(f\" [WARN] No usable results yet — keeping default \"\n f\"CFG['ACTIVE_BB']={CFG['ACTIVE_BB']!r}.\")\n return CFG[\"ACTIVE_BB\"]\n\n def _category(name):\n if name.endswith(\"-LinearProbe\"):\n return \"C: frozen foundation-model linear-probe\"\n if name in registry_keys:\n return \"A: fine-tuned CNN/hybrid backbone\"\n return \"B: fine-tuned Transformer\"\n\n ranked = sorted(valid.items(), key=lambda kv: -kv[1][metric])\n print(f\"\\n {'Rank':<5}{'Model':<26}{'Category':<38}{metric.upper():<8}Registry-eligible?\")\n registry_eligible_winner = None\n for i, (name, r) in enumerate(ranked, 1):\n cat = _category(name)\n eligible = name in registry_keys\n if eligible and registry_eligible_winner is None:\n registry_eligible_winner = name\n marker = \" <== CHOSEN for expensive training\" if name == registry_eligible_winner and eligible else \"\"\n print(f\" {i:<5}{name:<26}{cat:<38}{r[metric]:<8.4f}{'yes' if eligible else 'no'}{marker}\")\n\n if registry_eligible_winner is None:\n print(f\" [WARN] No registry-eligible entries yet — keeping default \"\n f\"CFG['ACTIVE_BB']={CFG['ACTIVE_BB']!r}.\")\n return CFG[\"ACTIVE_BB\"]\n\n if ranked[0][0] != registry_eligible_winner:\n print(f\"\\n NOTE: '{ranked[0][0]}' scored higher than the chosen backbone but \"\n f\"isn't a BACKBONE_REGISTRY entry, so this pipeline can't fine-tune it \"\n f\"automatically. If that margin holds up, it's worth a dedicated \"\n f\"fine-tuning run outside this notebook (full fine-tune of a foundation \"\n f\"model needs its native PyTorch training code, not a frozen linear probe).\")\n return registry_eligible_winner\n\n\nwinning_bb = _select_winning_backbone(bb_abl, set(BACKBONE_REGISTRY.keys()), metric=\"qwk\")\nprev_bb, prev_img_size = CFG[\"ACTIVE_BB\"], CFG[\"IMG_SIZE\"]\nCFG[\"ACTIVE_BB\"] = winning_bb\nCFG[\"IMG_SIZE\"] = BACKBONE_REGISTRY[winning_bb][\"img_size\"]\n\nprint(f\"\\n >>> WINNER: {winning_bb} (was default {prev_bb}) — this is what gets the \"\n f\"full 3-phase progressive-unfreeze training + 5-run statistical validation. <<<\")\n\nif CFG[\"IMG_SIZE\"] != prev_img_size:\n print(f\"\\n [RESOLUTION CHANGE] {prev_bb}@{prev_img_size}px -> \"\n f\"{winning_bb}@{CFG['IMG_SIZE']}px. Rebuilding the standard \"\n f\"train/val/test/idrid_test TFRecords at the new resolution \"\n f\"(build_tfrecords() skips automatically once a matching-resolution \"\n f\"build already exists).\")\n IMG = CFG[\"IMG_SIZE\"]\n build_tfrecords(train_df, \"train\", IMG)\n build_tfrecords(val_df, \"val\", IMG)\n build_tfrecords(test_df, \"test\", IMG)\n if idrid_test_df is not None:\n build_tfrecords(idrid_test_df, \"idrid_test\", IMG)\n auto_zip(tag=\"tfrecords_rebuilt_for_winner\")\nelse:\n print(f\"\\n Winning backbone uses the same {CFG['IMG_SIZE']}px resolution as the \"\n f\"default TFRecords already built in STEP 1b — no rebuild needed.\")\n\n# ─────────────────────────────────────────────────────────────────────────────\n# STEP 5: Within-winning-architecture ablations — Loss/Imbalance, Dataset\n# composition. These now test variants of the WINNING backbone (build_model()\n# defaults to CFG[\"ACTIVE_BB\"], already updated above). Findings are reported\n# here to inform the final recipe before the expensive commit below; picking\n# a non-default loss/dataset composition based on these results is currently\n# a manual decision (re-run STEP 7 with an adjusted train() call / train_df\n# if you want to act on them) — ask if you'd like this auto-wired too.\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"\\n\" + \"=\"*60)\nprint(f\"STEP 5 — Within-{winning_bb}-architecture ablations: Loss & Dataset composition\")\nprint(\"=\"*60)\n\nprint(\"\\n--- Loss / Imbalance Ablation ---\")\ntry:\n loss_abl = run_loss_ablation(train_df, val_df)\nexcept Exception as e:\n print(f\"[ERROR] Loss ablation failed: {e}\")\n loss_abl = {}\nauto_zip(tag=\"after_loss_ablation\")\n\nprint(\"\\n--- Dataset Composition Ablation ---\")\ntry:\n ds_abl = run_dataset_ablation(val_df)\nexcept Exception as e:\n print(f\"[ERROR] Dataset ablation failed: {e}\")\n ds_abl = {}\nauto_zip(tag=\"after_dataset_ablation\")\n\n# ─────────────────────────────────────────────────────────────────────────────\n# STEP 5b: Apply the loss & dataset winners — build the actual recipe the\n# expensive training run (STEP 7) will use.\n#\n# This is the part that makes STEP 5 more than a report: it determines the\n# winning loss strategy and winning dataset composition by QWK, constructs\n# the resulting training set, and — because the fast TFRecord path reads by\n# a FIXED tag rather than by inspecting whatever `train_df` gets passed to\n# train() — rebuilds a dedicated \"train_final\" TFRecord set whenever the\n# winning recipe actually differs from the plain original train_df. Skipped\n# automatically (falls back to the existing \"train\" tag) if the winners are\n# just the defaults, so this costs nothing when nothing changed.\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 5b — Applying loss & dataset winners to build the final training recipe\")\nprint(\"=\"*60)\n\ndef _select_winner_by_qwk(results_dict):\n valid = {k: v for k, v in results_dict.items() if isinstance(v, dict) and \"qwk\" in v}\n if not valid:\n return None, None\n winner = max(valid, key=lambda k: valid[k][\"qwk\"])\n return winner, valid[winner][\"qwk\"]\n\nloss_winner, loss_winner_qwk = _select_winner_by_qwk(loss_abl)\nds_winner, ds_winner_qwk = _select_winner_by_qwk(ds_abl)\nprint(f\" Loss ablation winner : {loss_winner or '(no usable result — defaulting to OrdinalBCE)'}\"\n + (f\" (qwk={loss_winner_qwk:.4f})\" if loss_winner else \"\"))\nprint(f\" Dataset ablation winner : {ds_winner or '(no usable result — defaulting to full train_df)'}\"\n + (f\" (qwk={ds_winner_qwk:.4f})\" if ds_winner else \"\"))\n\n# ── Build the winning dataset composition ──\nDATASET_SOURCE_CONFIGS = {\n \"EyePACS_only\": [\"eyepacs\"],\n \"EyePACS_APTOS\": [\"eyepacs\", \"aptos\"],\n \"EyePACS_Messidor\": [\"eyepacs\", \"messidor\"],\n \"Multi_source_All\": [\"eyepacs\", \"aptos\", \"messidor\", \"idrid\"],\n}\nfinal_train_df = train_df\nif ds_winner and ds_winner in DATASET_SOURCE_CONFIGS:\n _orig_datasets = CFG[\"DATASETS\"]\n CFG[\"DATASETS\"] = DATASET_SOURCE_CONFIGS[ds_winner]\n _sub = build_dataset_df()\n CFG[\"DATASETS\"] = _orig_datasets\n # Properly exclude val/test/idrid_test — more careful than\n # run_dataset_ablation's own internal filtering (which only excludes\n # val_df), since this dataframe feeds the model that gets reported.\n _excluded_ids = set(val_df[\"image_id\"]) | set(test_df[\"image_id\"])\n if idrid_test_df is not None:\n _excluded_ids |= set(idrid_test_df[\"image_id\"])\n final_train_df = _sub[~_sub[\"image_id\"].isin(_excluded_ids)].reset_index(drop=True)\n print(f\" -> Dataset composition '{ds_winner}': {len(final_train_df):,} images \"\n f\"(vs. {len(train_df):,} in the original full training set)\")\nelse:\n print(f\" -> Keeping the original full training set ({len(train_df):,} images).\")\n\n# ── Apply the winning loss strategy ──\nCFG[\"USE_FOCAL\"] = False\nCFG[\"USE_MIXUP\"] = False\nif loss_winner == \"FocalLoss\":\n CFG[\"USE_FOCAL\"] = True\n print(f\" -> Final training will use Focal Ordinal Loss (gamma={CFG['FOCAL_GAMMA']}).\")\nelif loss_winner == \"MixUp\":\n CFG[\"USE_MIXUP\"] = True\n print(f\" -> Final training will use MixUp augmentation (alpha=0.2).\")\nelif loss_winner == \"Oversampling\":\n _max_c = final_train_df[\"label\"].value_counts().max()\n _parts = [final_train_df]\n for _c in sorted(final_train_df[\"label\"].unique()):\n _cls = final_train_df[final_train_df[\"label\"] == _c]\n _n_rep = _max_c - len(_cls)\n if _n_rep > 0:\n _parts.append(_cls.sample(_n_rep, replace=True, random_state=SEED))\n final_train_df = (pd.concat(_parts, ignore_index=True)\n .sample(frac=1, random_state=SEED)\n .reset_index(drop=True))\n print(f\" -> Final training will use random oversampling (class-balanced via \"\n f\"replacement): {len(final_train_df):,} images.\")\nelse:\n print(f\" -> Final training will use standard Ordinal BCE loss.\")\n\n# ── Decide whether the fast path needs a dedicated TFRecord tag ──\n_final_ids = set(final_train_df[\"image_id\"])\n_orig_ids = set(train_df[\"image_id\"])\n_recipe_changed = (_final_ids != _orig_ids) or (len(final_train_df) != len(train_df))\n\nif _recipe_changed:\n print(f\"\\n Final recipe differs from the default TFRecord-backed 'train' set \"\n f\"({len(final_train_df):,} vs {len(train_df):,} images) — building a \"\n f\"dedicated 'train_final' TFRecord tag so the fast path actually reflects it.\")\n build_tfrecords(final_train_df, \"train_final\", CFG[\"IMG_SIZE\"])\n final_tfrecord_tag = \"train_final\"\n auto_zip(tag=\"tfrecords_built_for_final_recipe\")\nelse:\n print(f\"\\n Final recipe matches the default training set exactly — reusing the \"\n f\"existing 'train' TFRecords, no rebuild needed.\")\n final_tfrecord_tag = \"train\"\n\nprint(f\"\\n >>> FINAL RECIPE: backbone={CFG['ACTIVE_BB']} loss_winner={loss_winner or 'OrdinalBCE'} \"\n f\"dataset_winner={ds_winner or 'Multi_source_All'} n_train={len(final_train_df):,} <<<\")\n\n# ─────────────────────────────────────────────────────────────────────────────\n# STEP 6: Deployment-awareness snapshot — before committing, surface what's\n# already known about the winner's deployment profile (full ONNX/TFLite/INT8\n# numbers come later, in Extended Cell R, once this model is fully trained;\n# this is the cheap early signal from the backbone-ablation run itself).\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 6 — Deployment-awareness snapshot for the chosen winner\")\nprint(\"=\"*60)\n_winner_entry = bb_abl.get(winning_bb, {})\nif _winner_entry:\n print(f\" {winning_bb}: params={_winner_entry.get('params_M','?')}M \"\n f\"latency={_winner_entry.get('latency_ms','?')}ms (ablation-time, single \"\n f\"forward pass, this session's accelerator)\")\n print(f\" A definitive ONNX/TFLite-FP32/TFLite-INT8 export + latency/QWK-tradeoff \"\n f\"benchmark for this exact winner runs later, in Extended Cell R.\")\nelse:\n print(f\" No params/latency recorded for {winning_bb} in this run \"\n f\"(likely restored from a prior session without those fields) — \"\n f\"Extended Cell R will still produce full deployment numbers.\")\n\n# ─────────────────────────────────────────────────────────────────────────────\n# STEP 7: Progressive-unfreezing training (3 phases) — ON THE WINNER\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"\\n\" + \"=\"*60)\nprint(f\"STEP 7 — Progressive-unfreezing training (3 phases) on WINNER: {CFG['ACTIVE_BB']}\")\nprint(\"=\"*60)\nmodel = train(final_train_df, val_df, tfrecord_tag=final_tfrecord_tag)\n\n# ─────────────────────────────────────────────────────────────────────────────\n# STEP 8: Full evaluation — val, internal test, IDRiD test\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 8 — Multi-set evaluation\")\nprint(\"=\"*60)\n\nres_val = evaluate(model, val_df, desc=\"Validation set\", save_tag=\"val\")\nres_test = evaluate(model, test_df, desc=\"Internal test set\", save_tag=\"test_internal\")\n\nplot_confusion_matrix(res_val[\"labels\"], res_val[\"preds\"], \"Val Confusion Matrix\", \"cm_val\")\nplot_confusion_matrix(res_test[\"labels\"], res_test[\"preds\"], \"Test Confusion Matrix\", \"cm_test\")\n\nif idrid_test_df is not None:\n res_idrid = evaluate(model, idrid_test_df, desc=\"IDRiD external holdout\",\n save_tag=\"test_idrid\")\n plot_confusion_matrix(res_idrid[\"labels\"], res_idrid[\"preds\"],\n \"IDRiD Confusion Matrix\", \"cm_idrid\")\n\n# ─────────────────────────────────────────────────────────────────────────────\n# STEP 9: Binary triage (Youden-J optimal threshold)\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 9 — Binary triage (Grade >= 2 = Referable DR)\")\nprint(\"=\"*60)\nrho_star, binary_results = binary_triage(\n model, val_df, test_df=idrid_test_df if idrid_test_df is not None else test_df)\nprint(f\"\\n Optimal threshold ρ* = {rho_star:.4f}\")\n\n# ─────────────────────────────────────────────────────────────────────────────\n# STEP 10: Statistical validation (5 independent runs) — ON THE WINNER\n# CFG[\"ACTIVE_BB\"] is the winner selected in STEP 4, so every one of these 5\n# runs (and the mean±std they produce) is for the final chosen architecture,\n# not whatever the pre-search default happened to be.\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"\\n\" + \"=\"*60)\nprint(f\"STEP 10 — Statistical validation (5 runs) on WINNER: {CFG['ACTIVE_BB']}\")\nprint(\"=\"*60)\nstat_results = run_statistical_validation(train_df, val_df, n_runs=CFG[\"N_RUNS\"])\n\n# ─────────────────────────────────────────────────────────────────────────────\n# STEP 11: Resolution Ablation — kept here rather than in the search phase.\n# Unlike backbone/loss/dataset, this one isn't a clean independent axis: it\n# necessarily swaps architecture per resolution (EfficientNetB0 @224 vs.\n# EfficientNetB4 @380/512, per its own internal logic), so \"the winning\n# resolution\" is entangled with \"which backbone runs at that resolution\" —\n# it's an exploratory comparison, not a pre-commit search input.\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 11 — Resolution Ablation (exploratory; backbone-paired by design)\")\nprint(\"=\"*60)\ntry:\n res_abl = run_resolution_ablation(train_df, val_df, resolutions=(224, 380, 512))\nexcept Exception as e:\n print(f\"[ERROR] Resolution ablation failed: {e}\")\n res_abl = {}\nauto_zip(tag=\"after_resolution_ablation\")\n\n# ─────────────────────────────────────────────────────────────────────────────\n# STEP 12: XAI\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 12 — Explainability (XAI)\")\nprint(\"=\"*60)\n\nprint(\"\\n--- CAM grid (5 grades × 4 methods) ---\")\ntry:\n plot_xai_grid(model, val_df)\nexcept Exception as e:\n print(f\"[ERROR] XAI grid failed: {e}\")\n\nprint(\"\\n--- SHAP grid ---\")\ntry:\n plot_shap_grid(model, val_df)\nexcept Exception as e:\n print(f\"[ERROR] SHAP grid failed: {e}\")\n\nprint(\"\\n--- LIME grid ---\")\ntry:\n plot_lime_grid(model, val_df)\nexcept Exception as e:\n print(f\"[ERROR] LIME grid failed: {e}\")\n\nprint(\"\\n--- Quantitative faithfulness (Insertion/Deletion AUC) ---\")\ntry:\n faith_results = compute_xai_faithfulness(model, val_df,\n n_samples=CFG[\"XAI_SAMPLES\"], steps=20)\nexcept Exception as e:\n print(f\"[ERROR] Faithfulness failed: {e}\")\nauto_zip(tag=\"after_xai\")\n\n# ─────────────────────────────────────────────────────────────────────────────\n# STEP 13: Final summary\n# ─────────────────────────────────────────────────────────────────────────────\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 13 — Final Summary\")\nprint(\"=\"*60)\n\nsummary = {\n \"backbone\": CFG[\"ACTIVE_BB\"],\n \"backbone_selection\": \"chosen by STEP 4 architecture search (highest QWK among \"\n \"BACKBONE_REGISTRY entries in backbone_ablation.json)\",\n \"loss_winner\": loss_winner or \"OrdinalBCE (default)\",\n \"dataset_winner\": ds_winner or \"Multi_source_All (default, full train_df)\",\n \"n_train_final\": int(len(final_train_df)),\n \"img_size\": CFG[\"IMG_SIZE\"],\n \"batch_size\": CFG[\"BATCH_SIZE\"],\n \"n_gpus\": N_GPUS,\n \"val_accuracy\": res_val[\"accuracy\"],\n \"val_qwk\": res_val[\"qwk\"],\n \"val_auc_macro\": res_val[\"auc_macro_ovr\"],\n \"val_f1_macro\": res_val[\"f1_macro\"],\n \"binary_threshold\": round(rho_star, 4),\n \"binary_val_sensitivity\": binary_results[\"val\"][\"sensitivity\"],\n \"binary_val_specificity\": binary_results[\"val\"][\"specificity\"],\n \"binary_val_auc\": binary_results[\"val\"][\"auc\"],\n \"stat_acc_mean\": float(np.mean([r[\"accuracy\"] for r in stat_results])),\n \"stat_acc_std\": float(np.std( [r[\"accuracy\"] for r in stat_results])),\n \"stat_qwk_mean\": float(np.mean([r[\"qwk\"] for r in stat_results])),\n \"stat_qwk_std\": float(np.std( [r[\"qwk\"] for r in stat_results])),\n}\nwith open(f\"{OUT_DIR}/final_summary.json\", \"w\") as f:\n json.dump(summary, f, indent=2)\n\nprint(f\"\\n Backbone : {summary['backbone']} @ {summary['img_size']}px \"\n f\"(selected by architecture search, see backbone_ablation.json)\")\nprint(f\" GPUs : {summary['n_gpus']}x accelerator(s)\")\nprint(f\" Val Accuracy : {summary['val_accuracy']:.4f}\")\nprint(f\" Val QWK : {summary['val_qwk']:.4f}\")\nprint(f\" Val AUC macro : {summary['val_auc_macro']}\")\nprint(f\" Val F1 macro : {summary['val_f1_macro']:.4f}\")\nprint(f\" Binary thr ρ* : {summary['binary_threshold']}\")\nprint(f\" Sensitivity : {summary['binary_val_sensitivity']:.4f}\")\nprint(f\" Specificity : {summary['binary_val_specificity']:.4f}\")\nprint(f\" Stat Val Acc : {summary['stat_acc_mean']:.4f} ± {summary['stat_acc_std']:.4f}\")\nprint(f\" Stat Val QWK : {summary['stat_qwk_mean']:.4f} ± {summary['stat_qwk_std']:.4f}\")\nprint(f\"\\n Metrics saved to : {OUT_DIR}/\")\nprint(f\" History CSV : {HISTORY_CSV}\")\nprint(f\" Artifact ZIP : {ZIP_PATH}\")\n\n# Final auto-zip of everything\nauto_zip(tag=\"FINAL_COMPLETE\")\nprint(\"\\n[✓] Pipeline complete. Download DR_Journal_artifacts.zip from Kaggle output.\")\nprint(\"[i] Loss/Dataset ablation results (STEP 5) and Resolution ablation (STEP 11) are\")\nprint(\" diagnostic — see loss_ablation.json / dataset_ablation.json / resolution_ablation.json\")\nprint(\" if you want to manually act on a non-default choice for a future run.\")\n","metadata":{"execution":{"iopub.execute_input":"2026-07-05T19:35:31.447833Z","iopub.status.busy":"2026-07-05T19:35:31.447315Z"},"papermill":{"duration":null,"end_time":null,"exception":false,"start_time":"2026-07-05T19:35:31.434728+00:00","status":"running"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ═══════════════════════════════════════════════════════════════════════\n# EXTENDED ABLATIONS (Added)\n# ═══════════════════════════════════════════════════════════════════════\n\n**Everything below is additive.** No existing cell above this point was\nmodified — `preprocess_image`, `make_tf_dataset`, `build_model`,\n`train()`, `evaluate()`, and every original ablation function are untouched\nand remain fully compatible with any checkpoints/outputs you already have.\n\nRun the cells in this section **after** the original Cell 21 main execution\nhas completed at least once (you need `model`, `train_df`, `val_df`,\n`test_df` in memory — or just re-run Cell 21 first, it will skip anything\nalready done).\n\nNew sections added:\n1. Ordinal encoding ablation — Softmax vs. CORAL vs. CORN vs. your existing\n extended-binary Ordinal-BCE head\n2. Combinatorial loss/imbalance ablation (focal × class-weights × oversampling)\n3. Preprocessing ablation — None / CLAHE / Ben Graham / Both\n4. Calibration — temperature scaling, ECE, Brier score\n5. Uncertainty quantification — MC-Dropout variance/entropy vs. correctness\n6. Error analysis — adjacent-grade vs. severe-grade errors\n7. Statistical significance testing helper (Wilcoxon)\n8. Dual-resolution fusion ablation (single CFP source, two input scales)\n9. Multi-backbone (CNN + Transformer/Hybrid) late-fusion ablation\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"markdown","source":"### Extended Cell A — Ordinal Encoding Variants: Softmax / CORAL / CORN (model + loss definitions)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL A — Ordinal Encoding Variants: Softmax / CORAL / CORN\n# Compares alternative output-head formulations against the existing\n# extended-binary ordinal head (Dense(4) + ordinal_bce_loss) already used\n# throughout this notebook. Architecture-only additions — no changes to\n# preprocess_image, make_tf_dataset, or build_model.\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef build_softmax_model(backbone_name=None, unfreeze_last=-1):\n \"\"\"Standard 5-class softmax head, same backbone/preprocessing as build_model.\"\"\"\n bb_name = backbone_name or CFG[\"ACTIVE_BB\"]\n reg = BACKBONE_REGISTRY[bb_name]\n sz = reg[\"img_size\"]\n base = reg[\"loader\"](sz)\n base.trainable = True\n for layer in base.layers:\n if isinstance(layer, layers.BatchNormalization):\n layer.trainable = False\n if unfreeze_last > 0:\n for layer in base.layers[:-unfreeze_last]:\n layer.trainable = False\n elif unfreeze_last == 0:\n base.trainable = False\n\n inp = keras.Input(shape=(sz, sz, 3), name=\"fundus_input\")\n x = layers.Lambda(reg.get(\"preprocess\", lambda t: t), name=\"backbone_preprocess\")(inp)\n x = base(x)\n x = layers.GlobalAveragePooling2D(name=\"gap\")(x)\n x = layers.BatchNormalization(name=\"head_bn\")(x)\n x = layers.Dropout(CFG[\"DROPOUT\"], name=\"mc_dropout\")(x)\n x = layers.Activation(\"linear\", dtype=\"float32\", name=\"fp32_cast\")(x)\n out = layers.Dense(5, activation=\"softmax\", dtype=\"float32\", name=\"softmax_head\")(x)\n m = keras.Model(inp, out, name=f\"DR_{bb_name}_softmax\")\n return m\n\n\nclass CoralLayer(layers.Layer):\n \"\"\"CORAL (Cao et al. 2020): a single shared weight vector projects features\n to one scalar logit; per-threshold biases are added on top. This guarantees\n rank-monotonic predictions by construction, unlike the extended-binary head\n (independent Dense(4)) used elsewhere in this notebook.\"\"\"\n def __init__(self, n_classes=5, **kwargs):\n super().__init__(**kwargs)\n self.n_thresholds = n_classes - 1\n\n def build(self, input_shape):\n self.w = self.add_weight(\"coral_weight\", shape=(input_shape[-1], 1),\n initializer=\"glorot_uniform\", dtype=\"float32\")\n self.b = self.add_weight(\"coral_bias\", shape=(self.n_thresholds,),\n initializer=\"zeros\", dtype=\"float32\")\n super().build(input_shape)\n\n def call(self, x):\n shared_logit = tf.matmul(tf.cast(x, tf.float32), self.w) # [batch, 1]\n return shared_logit + self.b # -> [batch, 4]\n\n\ndef build_coral_model(backbone_name=None, unfreeze_last=-1):\n \"\"\"Same backbone as build_model, CoralLayer replaces the independent Dense(4).\n Uses the SAME ordinal-encoded targets and SAME loss (ordinal_bce_loss) as\n the rest of this notebook — only the head architecture differs.\"\"\"\n bb_name = backbone_name or CFG[\"ACTIVE_BB\"]\n reg = BACKBONE_REGISTRY[bb_name]\n sz = reg[\"img_size\"]\n base = reg[\"loader\"](sz)\n base.trainable = True\n for layer in base.layers:\n if isinstance(layer, layers.BatchNormalization):\n layer.trainable = False\n if unfreeze_last > 0:\n for layer in base.layers[:-unfreeze_last]:\n layer.trainable = False\n elif unfreeze_last == 0:\n base.trainable = False\n\n inp = keras.Input(shape=(sz, sz, 3), name=\"fundus_input\")\n x = layers.Lambda(reg.get(\"preprocess\", lambda t: t), name=\"backbone_preprocess\")(inp)\n x = base(x)\n x = layers.GlobalAveragePooling2D(name=\"gap\")(x)\n x = layers.BatchNormalization(name=\"head_bn\")(x)\n x = layers.Dropout(CFG[\"DROPOUT\"], name=\"mc_dropout\")(x)\n x = layers.Activation(\"linear\", dtype=\"float32\", name=\"fp32_cast\")(x)\n out = CoralLayer(n_classes=5, dtype=\"float32\", name=\"coral_head\")(x)\n m = keras.Model(inp, out, name=f\"DR_{bb_name}_coral\")\n return m\n\n\ndef corn_loss(y_true, y_pred):\n \"\"\"CORN (Shi et al. 2021). y_true: integer grade [N] (NOT ordinal-encoded).\n y_pred: [N, 4] logits. Trains P(y>k | y>=k) via conditional masking, so each\n threshold only learns from examples that actually reached that rank.\"\"\"\n y_true = tf.cast(y_true, tf.float32)\n losses, masks = [], []\n for k in range(4):\n mask = tf.cast(y_true >= k, tf.float32)\n target = tf.cast(y_true > k, tf.float32)\n bce = tf.keras.backend.binary_crossentropy(target, y_pred[:, k], from_logits=True)\n losses.append(mask * bce)\n masks.append(mask)\n total_loss = tf.add_n(losses)\n total_mask = tf.add_n(masks)\n return total_loss / (total_mask + 1e-8)\n\n\ndef make_tf_dataset_integer_labels(df, batch_size, img_size, augment=False,\n shuffle=True, class_weights=None):\n \"\"\"Identical to make_tf_dataset, except the label tensor is the raw integer\n grade instead of the 4-way ordinal encoding — required for CORN's\n conditional loss. Preprocessing (preprocess_image) is untouched.\"\"\"\n paths = df[\"path\"].values\n labels = df[\"label\"].values.astype(np.float32)\n\n if class_weights is not None:\n weights = np.array([class_weights[int(l)] for l in labels], dtype=np.float32)\n ds = tf.data.Dataset.from_tensor_slices((paths, labels, weights))\n else:\n ds = tf.data.Dataset.from_tensor_slices((paths, labels))\n\n if shuffle:\n ds = ds.shuffle(buffer_size=len(paths), seed=SEED, reshuffle_each_iteration=True)\n\n sz_, aug_ = img_size, augment\n\n def _process(*args):\n path_py = args[0].numpy().decode(\"utf-8\")\n img = preprocess_image(path_py, sz_, aug_)\n return (img,) + args[1:]\n\n def _tf_wrap(*args):\n dtypes = [tf.float32] + [a.dtype for a in args[1:]]\n out = tf.py_function(_process, args, dtypes)\n out[0].set_shape([sz_, sz_, 3])\n for i, a in enumerate(args[1:], 1):\n out[i].set_shape(a.shape)\n return tuple(out)\n\n ds = (ds.with_options(OPTIONS)\n .map(_tf_wrap, num_parallel_calls=NUM_WORKERS)\n .batch(batch_size, drop_remainder=shuffle)\n .prefetch(2))\n return ds\n\nprint(\"[OK] Ordinal encoding variants defined: Softmax, CORAL, CORN\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell B — Ordinal Encoding Ablation Runner","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL B — Ordinal Encoding Ablation Runner\n# Trains Softmax / CORAL / CORN variants (fixed schedule, same backbone/img size\n# as CFG[\"ACTIVE_BB\"]) and compares them against the existing OrdinalBCE result\n# already saved in loss_ablation.json. Fully resumable — skips completed variants.\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef _decode_softmax(probs):\n return np.argmax(probs, axis=1)\n\ndef run_ordinal_encoding_ablation(train_df, val_df, epochs=25, patience=6):\n out_file = f\"{OUT_DIR}/encoding_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n\n IMG, BS = CFG[\"IMG_SIZE\"], CFG[\"BATCH_SIZE\"]\n cw = get_class_weights(train_df)\n\n # Pull the existing OrdinalBCE baseline from loss_ablation.json if present\n loss_abl_file = f\"{OUT_DIR}/loss_ablation.json\"\n if os.path.exists(loss_abl_file) and \"OrdinalBCE(existing)\" not in results:\n loss_abl = json.load(open(loss_abl_file))\n if \"OrdinalBCE\" in loss_abl:\n results[\"OrdinalBCE(existing)\"] = loss_abl[\"OrdinalBCE\"]\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n\n # ── Softmax (5-class) ──────────────────────────────────────────────────\n if \"Softmax5class\" not in results:\n print(f\"\\n{'='*55}\\nENCODING ABLATION: Softmax5class\\n{'='*55}\")\n tds_sm = make_tf_dataset_integer_labels(train_df, BS, IMG, augment=True, class_weights=cw)\n vds_sm = make_tf_dataset_integer_labels(val_df, BS, IMG, augment=False, shuffle=False)\n ckpt_dir = f\"{CKPT_DIR}/encoding_softmax\"; os.makedirs(ckpt_dir, exist_ok=True)\n best_ckpt = f\"{ckpt_dir}/best.weights.h5\"\n with STRATEGY.scope():\n m = build_softmax_model()\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n m.compile(optimizer=keras.optimizers.AdamW(1e-5, weight_decay=CFG[\"WEIGHT_DECAY\"], clipnorm=1.0),\n loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])\n m.fit(tds_sm, validation_data=vds_sm, epochs=epochs, verbose=1, callbacks=[\n keras.callbacks.ModelCheckpoint(best_ckpt, save_best_only=True, save_weights_only=True,\n monitor=\"val_loss\", mode=\"min\"),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=patience, restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=3, min_lr=1e-8),\n ])\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n probs = m.predict(vds_sm, verbose=0)\n preds = _decode_softmax(probs)\n true = val_df[\"label\"].values\n results[\"Softmax5class\"] = {\n \"accuracy\": round(float(accuracy_score(true, preds)), 4),\n \"qwk\": round(float(cohen_kappa_score(true, preds, weights=\"quadratic\")), 4),\n }\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n del m; keras.backend.clear_session(); gc.collect()\n auto_zip(tag=\"encoding_softmax_done\")\n else:\n print(f\"[SKIP] Softmax5class: qwk={results['Softmax5class']['qwk']:.4f}\")\n\n # ── CORAL ────────────────────────────────────────────────────────────\n if \"CORAL\" not in results:\n print(f\"\\n{'='*55}\\nENCODING ABLATION: CORAL\\n{'='*55}\")\n tds = make_tf_dataset(train_df, BS, IMG, augment=True, class_weights=cw)\n vds = make_tf_dataset(val_df, BS, IMG, augment=False, shuffle=False)\n ckpt_dir = f\"{CKPT_DIR}/encoding_coral\"; os.makedirs(ckpt_dir, exist_ok=True)\n best_ckpt = f\"{ckpt_dir}/best.weights.h5\"\n with STRATEGY.scope():\n m = build_coral_model()\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n m.compile(optimizer=keras.optimizers.AdamW(1e-5, weight_decay=CFG[\"WEIGHT_DECAY\"], clipnorm=1.0),\n loss=ordinal_bce_loss, metrics=[ordinal_accuracy])\n m.fit(tds, validation_data=vds, epochs=epochs, verbose=1, callbacks=[\n keras.callbacks.ModelCheckpoint(best_ckpt, save_best_only=True, save_weights_only=True,\n monitor=\"val_loss\", mode=\"min\"),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=patience, restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=3, min_lr=1e-8),\n ])\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n raw = m.predict(vds, verbose=0)\n true = val_df[\"label\"].values\n opt_thr, opt_qwk = find_optimal_threshold(raw, true)\n preds = np.array([ordinal_logits_to_grade(l, thr=opt_thr) for l in raw])\n results[\"CORAL\"] = {\n \"accuracy\": round(float(accuracy_score(true, preds)), 4),\n \"qwk\": round(float(opt_qwk), 4),\n }\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n del m; keras.backend.clear_session(); gc.collect()\n auto_zip(tag=\"encoding_coral_done\")\n else:\n print(f\"[SKIP] CORAL: qwk={results['CORAL']['qwk']:.4f}\")\n\n # ── CORN ─────────────────────────────────────────────────────────────\n if \"CORN\" not in results:\n print(f\"\\n{'='*55}\\nENCODING ABLATION: CORN\\n{'='*55}\")\n tds = make_tf_dataset_integer_labels(train_df, BS, IMG, augment=True, class_weights=cw)\n vds = make_tf_dataset_integer_labels(val_df, BS, IMG, augment=False, shuffle=False)\n ckpt_dir = f\"{CKPT_DIR}/encoding_corn\"; os.makedirs(ckpt_dir, exist_ok=True)\n best_ckpt = f\"{ckpt_dir}/best.weights.h5\"\n with STRATEGY.scope():\n m = build_model(unfreeze_last=-1) # same Dense(4)-logit architecture; loss/targets differ\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n m.compile(optimizer=keras.optimizers.AdamW(1e-5, weight_decay=CFG[\"WEIGHT_DECAY\"], clipnorm=1.0),\n loss=corn_loss)\n m.fit(tds, validation_data=vds, epochs=epochs, verbose=1, callbacks=[\n keras.callbacks.ModelCheckpoint(best_ckpt, save_best_only=True, save_weights_only=True,\n monitor=\"val_loss\", mode=\"min\"),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=patience, restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=3, min_lr=1e-8),\n ])\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n raw = m.predict(vds, verbose=0)\n true = val_df[\"label\"].values\n opt_thr, opt_qwk = find_optimal_threshold(raw, true)\n preds = np.array([ordinal_logits_to_grade(l, thr=opt_thr) for l in raw])\n results[\"CORN\"] = {\n \"accuracy\": round(float(accuracy_score(true, preds)), 4),\n \"qwk\": round(float(opt_qwk), 4),\n }\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n del m; keras.backend.clear_session(); gc.collect()\n auto_zip(tag=\"encoding_corn_done\")\n else:\n print(f\"[SKIP] CORN: qwk={results['CORN']['qwk']:.4f}\")\n\n print(f\"\\n{'='*55}\\nORDINAL ENCODING ABLATION SUMMARY\\n{'='*55}\")\n for k, v in results.items():\n print(f\" {k:<22} acc={v['accuracy']:.4f} qwk={v['qwk']:.4f}\")\n return results\n\nprint(\"[OK] run_ordinal_encoding_ablation defined.\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell C — Combinatorial Loss/Imbalance Ablation","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL C — Combinatorial Loss/Imbalance Ablation\n# Isolates which component (focal weighting vs. oversampling vs. class weights,\n# which are always-on via get_class_weights) actually drives improvement,\n# rather than only comparing single flags in isolation (as run_loss_ablation does).\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef run_combo_ablation(train_df, val_df):\n out_file = f\"{OUT_DIR}/combo_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n variants = [\n (\"OrdinalBCE_CW\", False, False),\n (\"OrdinalFocal_CW\", True, False),\n (\"OrdinalBCE_CW_Oversample\", False, True),\n (\"OrdinalFocal_CW_Oversample\", True, True),\n ]\n for name, focal, oversamp in variants:\n if name in results:\n print(f\"[SKIP] {name}: qwk={results[name]['qwk']:.4f}\")\n continue\n print(f\"\\n{'='*55}\\nCOMBO ABLATION: {name}\\n{'='*55}\")\n r = _run_variant(name, train_df, val_df, use_focal=focal, oversample=oversamp,\n ckpt_subdir=f\"{CKPT_DIR}/combo_{name}\")\n results[name] = {\"accuracy\": r[\"accuracy\"], \"qwk\": r[\"qwk\"]}\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n auto_zip(tag=f\"combo_{name}_done\")\n\n print(f\"\\n{'='*55}\\nCOMBO ABLATION SUMMARY\\n{'='*55}\")\n for k, v in results.items():\n print(f\" {k:<28} acc={v['accuracy']:.4f} qwk={v['qwk']:.4f}\")\n return results\n\nprint(\"[OK] run_combo_ablation defined.\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell D — Probability Calibration (Temperature Scaling, ECE, Brier)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL D — Probability Calibration (Temperature Scaling, ECE, Brier)\n# Consumes the existing evaluate() output — no changes to training required.\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef temperature_scale(logits, ord_targets, max_iter=200, lr=0.01):\n \"\"\"Fit one scalar T that minimises ordinal BCE NLL on validation logits.\n logits: [N,4] raw ordinal logits. ord_targets: [N,4] binary ordinal targets.\"\"\"\n T = tf.Variable(1.0, dtype=tf.float32)\n opt = tf.keras.optimizers.Adam(lr)\n logits_t = tf.constant(logits, dtype=tf.float32)\n y = tf.constant(ord_targets, dtype=tf.float32)\n for _ in range(max_iter):\n with tf.GradientTape() as tape:\n scaled = logits_t / T\n loss = tf.reduce_mean(tf.reduce_sum(\n tf.keras.backend.binary_crossentropy(y, scaled, from_logits=True), axis=-1))\n grads = tape.gradient(loss, [T])\n opt.apply_gradients(zip(grads, [T]))\n T.assign(tf.clip_by_value(T, 0.05, 10.0))\n return float(T.numpy())\n\n\ndef expected_calibration_error(probs_5class, labels, n_bins=15):\n \"\"\"ECE over max-class confidence.\"\"\"\n confidences = probs_5class.max(axis=1)\n predictions = probs_5class.argmax(axis=1)\n accuracies = (predictions == labels).astype(float)\n bins = np.linspace(0, 1, n_bins + 1)\n ece = 0.0\n for lo, hi in zip(bins[:-1], bins[1:]):\n mask = (confidences > lo) & (confidences <= hi)\n if mask.sum() > 0:\n ece += (mask.sum() / len(confidences)) * abs(accuracies[mask].mean() - confidences[mask].mean())\n return float(ece)\n\n\ndef brier_score(probs_5class, labels, n_classes=5):\n onehot = np.eye(n_classes)[labels]\n return float(np.mean(np.sum((probs_5class - onehot) ** 2, axis=1)))\n\n\ndef ordinal_probs_to_class_probs(probs_ordinal):\n \"\"\"Convert the 4 cumulative sigmoid probs into per-class probabilities,\n same construction already used inside evaluate() for AUC.\"\"\"\n cp = np.zeros((len(probs_ordinal), 5))\n cp[:, 0] = 1 - probs_ordinal[:, 0]\n cp[:, 1] = probs_ordinal[:, 0] - probs_ordinal[:, 1]\n cp[:, 2] = probs_ordinal[:, 1] - probs_ordinal[:, 2]\n cp[:, 3] = probs_ordinal[:, 2] - probs_ordinal[:, 3]\n cp[:, 4] = probs_ordinal[:, 3]\n return np.clip(cp, 0, 1)\n\n\ndef run_calibration_analysis(model, val_df, test_df=None):\n \"\"\"Reports ECE/Brier before and after temperature scaling on val,\n then reports calibration on held-out test using the val-fitted T.\"\"\"\n out_file = f\"{OUT_DIR}/calibration_results.json\"\n if os.path.exists(out_file):\n print(f\"[SKIP] Calibration already computed.\")\n return json.load(open(out_file))\n\n res_val = evaluate(model, val_df, desc=\"Calibration — val\")\n probs_5c_val = ordinal_probs_to_class_probs(res_val[\"probs\"])\n ece_before = expected_calibration_error(probs_5c_val, res_val[\"labels\"])\n brier_before = brier_score(probs_5c_val, res_val[\"labels\"])\n\n logits_val = np.log(np.clip(res_val[\"probs\"], 1e-7, 1 - 1e-7) /\n np.clip(1 - res_val[\"probs\"], 1e-7, 1 - 1e-7))\n ord_targets_val = np.array([grade_to_ordinal(l) for l in res_val[\"labels\"]])\n T = temperature_scale(logits_val, ord_targets_val)\n\n scaled_probs_val = 1.0 / (1.0 + np.exp(-logits_val / T))\n probs_5c_val_scaled = ordinal_probs_to_class_probs(scaled_probs_val)\n ece_after = expected_calibration_error(probs_5c_val_scaled, res_val[\"labels\"])\n brier_after = brier_score(probs_5c_val_scaled, res_val[\"labels\"])\n\n out = {\n \"temperature\": round(T, 4),\n \"val_ece_before\": round(ece_before, 4),\n \"val_ece_after\": round(ece_after, 4),\n \"val_brier_before\": round(brier_before, 4),\n \"val_brier_after\": round(brier_after, 4),\n }\n\n if test_df is not None and len(test_df) > 0:\n res_test = evaluate(model, test_df, desc=\"Calibration — test\")\n logits_test = np.log(np.clip(res_test[\"probs\"], 1e-7, 1 - 1e-7) /\n np.clip(1 - res_test[\"probs\"], 1e-7, 1 - 1e-7))\n probs_5c_test = ordinal_probs_to_class_probs(res_test[\"probs\"])\n scaled_probs_test = 1.0 / (1.0 + np.exp(-logits_test / T))\n probs_5c_test_scaled = ordinal_probs_to_class_probs(scaled_probs_test)\n out[\"test_ece_before\"] = round(expected_calibration_error(probs_5c_test, res_test[\"labels\"]), 4)\n out[\"test_ece_after\"] = round(expected_calibration_error(probs_5c_test_scaled, res_test[\"labels\"]), 4)\n out[\"test_brier_before\"] = round(brier_score(probs_5c_test, res_test[\"labels\"]), 4)\n out[\"test_brier_after\"] = round(brier_score(probs_5c_test_scaled, res_test[\"labels\"]), 4)\n\n with open(out_file, \"w\") as f: json.dump(out, f, indent=2)\n print(f\"[CALIBRATION] T={T:.3f} Val ECE {ece_before:.4f}→{ece_after:.4f} \"\n f\"Brier {brier_before:.4f}→{brier_after:.4f}\")\n return out\n\nprint(\"[OK] Calibration functions defined.\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell E — Uncertainty Quantification (MC-Dropout variance/entropy)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL E — Uncertainty Quantification (MC-Dropout variance/entropy)\n# Runs its own MC-Dropout pass (independent of evaluate()'s cached results)\n# and correlates predictive uncertainty with correctness.\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef run_uncertainty_analysis(model, df, mc_samples=None, save_tag=\"uncertainty\"):\n out_file = f\"{OUT_DIR}/eval_{save_tag}.json\"\n if os.path.exists(out_file):\n print(f\"[SKIP] Uncertainty analysis already computed.\")\n return json.load(open(out_file))\n\n mc_samples = mc_samples or CFG[\"MC_SAMPLES\"]\n IMG = CFG[\"IMG_SIZE\"]\n MC_BS = 4\n paths, labels = df[\"path\"].tolist(), df[\"label\"].values\n\n mc_probs = []\n for mc_i in range(mc_samples):\n batch_probs = []\n for i in range(0, len(paths), MC_BS):\n imgs = np.array([preprocess_image(p, IMG) for p in paths[i:i+MC_BS]])\n logits = model(imgs, training=True).numpy()\n batch_probs.append(1.0 / (1.0 + np.exp(-logits)))\n del imgs\n mc_probs.append(np.concatenate(batch_probs, axis=0))\n if (mc_i + 1) % 5 == 0:\n gc.collect()\n print(f\" MC pass {mc_i+1}/{mc_samples} done\")\n\n mc_stack = np.stack(mc_probs, axis=0) # [MC, N, 4]\n mean_probs = mc_stack.mean(axis=0) # [N, 4]\n variance = mc_stack.var(axis=0).mean(axis=1) # [N] — mean variance across 4 thresholds\n eps = 1e-8\n entropy = -np.mean(\n mc_stack * np.log(np.clip(mc_stack, eps, 1)) +\n (1 - mc_stack) * np.log(np.clip(1 - mc_stack, eps, 1)),\n axis=(0, 2)) # [N]\n\n preds = np.array([ordinal_to_grade(p) for p in mean_probs])\n correct = (preds == labels)\n\n out = {\n \"n_samples\": int(len(labels)),\n \"mean_variance_correct\": round(float(variance[correct].mean()), 5) if correct.any() else None,\n \"mean_variance_incorrect\": round(float(variance[~correct].mean()), 5) if (~correct).any() else None,\n \"mean_entropy_correct\": round(float(entropy[correct].mean()), 5) if correct.any() else None,\n \"mean_entropy_incorrect\": round(float(entropy[~correct].mean()), 5) if (~correct).any() else None,\n \"accuracy\": round(float(correct.mean()), 4),\n }\n with open(out_file, \"w\") as f: json.dump(out, f, indent=2)\n print(f\"[UNCERTAINTY] Variance correct={out['mean_variance_correct']} \"\n f\"incorrect={out['mean_variance_incorrect']}\")\n print(f\"[UNCERTAINTY] Entropy correct={out['mean_entropy_correct']} \"\n f\"incorrect={out['mean_entropy_incorrect']}\")\n return out\n\nprint(\"[OK] run_uncertainty_analysis defined.\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell F — Error Analysis & Statistical Significance","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL F — Error Analysis & Statistical Significance\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef adjacent_vs_severe_errors(y_true, y_pred, save_tag=\"error_analysis\"):\n \"\"\"Ordinal grading naturally tolerates near-miss errors (0->1) better than\n far misses (0->4). Breaks down error severity rather than only accuracy.\"\"\"\n y_true = np.asarray(y_true); y_pred = np.asarray(y_pred)\n diff = np.abs(y_true - y_pred)\n n_correct = int((diff == 0).sum())\n n_adjacent = int((diff == 1).sum())\n n_severe = int((diff > 1).sum())\n n_total = len(y_true)\n out = {\n \"n_total\": n_total,\n \"correct\": n_correct,\n \"adjacent_errors\": n_adjacent,\n \"severe_errors\": n_severe,\n \"adjacent_pct_of_errors\": round(n_adjacent / (n_adjacent + n_severe + 1e-8), 4),\n \"severe_pct_of_errors\": round(n_severe / (n_adjacent + n_severe + 1e-8), 4),\n }\n with open(f\"{OUT_DIR}/{save_tag}.json\", \"w\") as f: json.dump(out, f, indent=2)\n print(f\"[ERROR ANALYSIS] Correct={n_correct} Adjacent-grade errors={n_adjacent} \"\n f\"Severe (>1 grade) errors={n_severe}\")\n return out\n\n\ndef compare_variants_significance(qwk_list_a, qwk_list_b, name_a=\"A\", name_b=\"B\"):\n \"\"\"Paired Wilcoxon signed-rank test between two sets of per-seed QWK scores\n (e.g. from run_statistical_validation-style repeated runs of two variants).\"\"\"\n from scipy.stats import wilcoxon\n if len(qwk_list_a) != len(qwk_list_b) or len(qwk_list_a) < 2:\n print(\"[SIGNIFICANCE] Need matched-length lists (>=2 paired seeds) — skipping.\")\n return None\n stat, p = wilcoxon(qwk_list_a, qwk_list_b)\n out = {\"comparison\": f\"{name_a}_vs_{name_b}\", \"wilcoxon_stat\": float(stat), \"p_value\": float(p)}\n print(f\"[SIGNIFICANCE] {name_a} vs {name_b}: Wilcoxon p={p:.4f}\")\n return out\n\nprint(\"[OK] Error analysis & significance testing functions defined.\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell G — Preprocessing Ablation (None / CLAHE / Ben Graham / Both)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL G — Preprocessing Ablation (None / CLAHE / Ben Graham / Both)\n# Adds a mode-aware preprocessing function WITHOUT modifying the original\n# preprocess_image (which stays untouched and keeps powering the main\n# pipeline, TFRecords, and all existing ablations).\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef preprocess_image_v2(path: str, img_size: int, augment: bool = False,\n mode: str = \"clahe\") -> np.ndarray:\n \"\"\"mode: 'none' | 'clahe' (default, matches preprocess_image) |\n 'ben_graham' | 'both'. Black-border crop + resize always applied.\"\"\"\n img = cv2.imread(path)\n if img is None:\n return np.zeros((img_size, img_size, 3), dtype=np.float32)\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n _, thr = cv2.threshold(gray, 7, 255, cv2.THRESH_BINARY)\n coords = cv2.findNonZero(thr)\n if coords is not None:\n x, y, w, h = cv2.boundingRect(coords)\n img = img[y:y+h, x:x+w]\n img = cv2.resize(img, (img_size, img_size), interpolation=cv2.INTER_CUBIC)\n\n if mode in (\"ben_graham\", \"both\"):\n # Ben Graham preprocessing: subtract local average color, boosts contrast\n # of lesions without CLAHE's tile artifacts.\n blurred = cv2.GaussianBlur(img, (0, 0), img_size / 30)\n img = cv2.addWeighted(img, 4, blurred, -4, 128)\n img = np.clip(img, 0, 255).astype(np.uint8)\n\n if mode in (\"clahe\", \"both\"):\n lab = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)\n clahe = cv2.createCLAHE(clipLimit=CFG[\"CLAHE_CLIP\"],\n tileGridSize=(CFG[\"CLAHE_TILE\"], CFG[\"CLAHE_TILE\"]))\n lab[:, :, 0] = clahe.apply(lab[:, :, 0])\n img = cv2.cvtColor(lab, cv2.COLOR_LAB2RGB)\n # mode == \"none\": black-border crop + resize only, no contrast enhancement\n\n if augment:\n if random.random() > 0.5: img = cv2.flip(img, 1)\n if random.random() > 0.5: img = cv2.flip(img, 0)\n alpha = random.uniform(0.8, 1.2)\n beta = random.uniform(-20, 20)\n img = np.clip(alpha * img.astype(np.float32) + beta, 0, 255).astype(np.uint8)\n angle = random.uniform(-15, 15)\n M = cv2.getRotationMatrix2D((img_size//2, img_size//2), angle, 1.0)\n img = cv2.warpAffine(img, M, (img_size, img_size))\n return img.astype(np.float32)\n\n\ndef make_tf_dataset_v2(df, batch_size, img_size, augment=False, shuffle=True,\n class_weights=None, preprocess_mode=\"clahe\"):\n \"\"\"Same structure as make_tf_dataset, using preprocess_image_v2 with a mode flag.\"\"\"\n paths = df[\"path\"].values\n labels = df[\"label\"].values\n ord_labels = np.array([grade_to_ordinal(l) for l in labels], dtype=np.float32)\n\n if class_weights is not None:\n weights = np.array([class_weights[l] for l in labels], dtype=np.float32)\n ds = tf.data.Dataset.from_tensor_slices((paths, ord_labels, weights))\n else:\n ds = tf.data.Dataset.from_tensor_slices((paths, ord_labels))\n\n if shuffle:\n ds = ds.shuffle(buffer_size=len(paths), seed=SEED, reshuffle_each_iteration=True)\n\n sz_, aug_, mode_ = img_size, augment, preprocess_mode\n\n def _process(*args):\n path_py = args[0].numpy().decode(\"utf-8\")\n img = preprocess_image_v2(path_py, sz_, aug_, mode_)\n return (img,) + args[1:]\n\n def _tf_wrap(*args):\n dtypes = [tf.float32] + [a.dtype for a in args[1:]]\n out = tf.py_function(_process, args, dtypes)\n out[0].set_shape([sz_, sz_, 3])\n for i, a in enumerate(args[1:], 1):\n out[i].set_shape(a.shape)\n return tuple(out)\n\n ds = (ds.with_options(OPTIONS)\n .map(_tf_wrap, num_parallel_calls=NUM_WORKERS)\n .batch(batch_size, drop_remainder=shuffle)\n .prefetch(2))\n return ds\n\n\ndef run_preprocessing_ablation(train_df, val_df, epochs=25, patience=6):\n out_file = f\"{OUT_DIR}/preprocessing_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n modes = [\"none\", \"clahe\", \"ben_graham\", \"both\"]\n IMG, BS = CFG[\"IMG_SIZE\"], CFG[\"BATCH_SIZE\"]\n cw = get_class_weights(train_df)\n\n for mode in modes:\n if mode in results:\n print(f\"[SKIP] preprocessing={mode}: qwk={results[mode]['qwk']:.4f}\")\n continue\n print(f\"\\n{'='*55}\\nPREPROCESSING ABLATION: {mode}\\n{'='*55}\")\n ckpt_dir = f\"{CKPT_DIR}/preprocess_{mode}\"; os.makedirs(ckpt_dir, exist_ok=True)\n best_ckpt = f\"{ckpt_dir}/best.weights.h5\"\n tds = make_tf_dataset_v2(train_df, BS, IMG, augment=True, class_weights=cw, preprocess_mode=mode)\n vds = make_tf_dataset_v2(val_df, BS, IMG, augment=False, shuffle=False, preprocess_mode=mode)\n with STRATEGY.scope():\n m = build_model(unfreeze_last=-1)\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n m.compile(optimizer=keras.optimizers.AdamW(1e-5, weight_decay=CFG[\"WEIGHT_DECAY\"], clipnorm=1.0),\n loss=get_loss(), metrics=[ordinal_accuracy])\n m.fit(tds, validation_data=vds, epochs=epochs, verbose=1, callbacks=[\n keras.callbacks.ModelCheckpoint(best_ckpt, save_best_only=True, save_weights_only=True,\n monitor=\"val_loss\", mode=\"min\"),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=patience, restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=3, min_lr=1e-8),\n ])\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n raw = m.predict(vds, verbose=0)\n true = val_df[\"label\"].values\n opt_thr, opt_qwk = find_optimal_threshold(raw, true)\n preds = np.array([ordinal_logits_to_grade(l, thr=opt_thr) for l in raw])\n results[mode] = {\n \"accuracy\": round(float(accuracy_score(true, preds)), 4),\n \"qwk\": round(float(opt_qwk), 4),\n }\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n del m; keras.backend.clear_session(); gc.collect()\n auto_zip(tag=f\"preprocess_{mode}_done\")\n\n print(f\"\\n{'='*55}\\nPREPROCESSING ABLATION SUMMARY\\n{'='*55}\")\n for k, v in results.items():\n print(f\" {k:<12} acc={v['accuracy']:.4f} qwk={v['qwk']:.4f}\")\n return results\n\nprint(\"[OK] Preprocessing ablation defined (does not modify preprocess_image).\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell H — Dual-Resolution Fusion Ablation (single CFP source, two scales)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL H — Dual-Resolution Fusion Ablation\n# Feeds the SAME source image at two resolutions (global context + fine detail)\n# through two backbone branches, gated-fusion of pooled features. Uses only\n# existing CFP data — no new datasets required.\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef build_dual_resolution_model(backbone_name=None, sz_low=380, sz_high=512):\n bb_name = backbone_name or CFG[\"ACTIVE_BB\"]\n reg = BACKBONE_REGISTRY[bb_name]\n preprocess_fn = reg.get(\"preprocess\", lambda t: t)\n\n inp_low = keras.Input(shape=(sz_low, sz_low, 3), name=\"input_low\")\n inp_high = keras.Input(shape=(sz_high, sz_high, 3), name=\"input_high\")\n\n base_low = reg[\"loader\"](sz_low); base_low.trainable = True\n base_high = reg[\"loader\"](sz_high); base_high.trainable = True\n for base in (base_low, base_high):\n for layer in base.layers:\n if isinstance(layer, layers.BatchNormalization):\n layer.trainable = False\n\n x_low = layers.Lambda(preprocess_fn, name=\"preprocess_low\")(inp_low)\n x_high = layers.Lambda(preprocess_fn, name=\"preprocess_high\")(inp_high)\n x_low = layers.GlobalAveragePooling2D(name=\"gap_low\")(base_low(x_low))\n x_high = layers.GlobalAveragePooling2D(name=\"gap_high\")(base_high(x_high))\n\n fused = layers.Concatenate(name=\"concat_low_high\")([x_low, x_high])\n gate = layers.Dense(fused.shape[-1], activation=\"sigmoid\", name=\"fusion_gate\")(fused)\n fused = layers.Multiply(name=\"gated_fusion\")([fused, gate])\n fused = layers.Dense(512, activation=\"relu\", name=\"fusion_dense\")(fused)\n fused = layers.BatchNormalization(name=\"fusion_bn\")(fused)\n fused = layers.Dropout(CFG[\"DROPOUT\"], name=\"fusion_dropout\")(fused)\n fused = layers.Activation(\"linear\", dtype=\"float32\", name=\"fp32_cast\")(fused)\n out = layers.Dense(4, name=\"ordinal_head\", dtype=\"float32\")(fused)\n\n return keras.Model([inp_low, inp_high], out, name=f\"DR_{bb_name}_dualres\")\n\n\ndef make_dual_res_dataset(df, batch_size, sz_low, sz_high, augment=False,\n shuffle=True, class_weights=None):\n paths = df[\"path\"].values\n labels = df[\"label\"].values\n ord_labels = np.array([grade_to_ordinal(l) for l in labels], dtype=np.float32)\n\n if class_weights is not None:\n weights = np.array([class_weights[l] for l in labels], dtype=np.float32)\n ds = tf.data.Dataset.from_tensor_slices((paths, ord_labels, weights))\n else:\n ds = tf.data.Dataset.from_tensor_slices((paths, ord_labels))\n\n if shuffle:\n ds = ds.shuffle(buffer_size=len(paths), seed=SEED, reshuffle_each_iteration=True)\n\n aug_ = augment\n\n def _process(*args):\n path_py = args[0].numpy().decode(\"utf-8\")\n img_low = preprocess_image(path_py, sz_low, aug_)\n img_high = preprocess_image(path_py, sz_high, aug_)\n return (img_low, img_high) + args[1:]\n\n def _tf_wrap(*args):\n dtypes = [tf.float32, tf.float32] + [a.dtype for a in args[1:]]\n out = tf.py_function(_process, args, dtypes)\n out[0].set_shape([sz_low, sz_low, 3])\n out[1].set_shape([sz_high, sz_high, 3])\n for i, a in enumerate(args[1:], 2):\n out[i].set_shape(a.shape)\n rest = tuple(out[2:])\n target = rest[0] if len(rest) == 1 else rest\n return (out[0], out[1]), target\n\n ds = (ds.with_options(OPTIONS)\n .map(_tf_wrap, num_parallel_calls=NUM_WORKERS)\n .batch(batch_size, drop_remainder=shuffle)\n .prefetch(2))\n return ds\n\n\ndef run_dual_resolution_ablation(train_df, val_df, sz_low=380, sz_high=512,\n epochs=25, patience=6, batch_size=8):\n out_file = f\"{OUT_DIR}/fusion_dualres_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n if \"DualResolutionFusion\" in results:\n print(f\"[SKIP] DualResolutionFusion: qwk={results['DualResolutionFusion']['qwk']:.4f}\")\n return results\n\n print(f\"\\n{'='*55}\\nFUSION ABLATION: Dual-Resolution ({sz_low}px + {sz_high}px)\\n{'='*55}\")\n bs = batch_size * N_GPUS\n cw = get_class_weights(train_df)\n tds = make_dual_res_dataset(train_df, bs, sz_low, sz_high, augment=True, class_weights=cw)\n vds = make_dual_res_dataset(val_df, bs, sz_low, sz_high, augment=False, shuffle=False)\n\n ckpt_dir = f\"{CKPT_DIR}/fusion_dualres\"; os.makedirs(ckpt_dir, exist_ok=True)\n best_ckpt = f\"{ckpt_dir}/best.weights.h5\"\n with STRATEGY.scope():\n m = build_dual_resolution_model(sz_low=sz_low, sz_high=sz_high)\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n m.compile(optimizer=keras.optimizers.AdamW(1e-5, weight_decay=CFG[\"WEIGHT_DECAY\"], clipnorm=1.0),\n loss=get_loss(), metrics=[ordinal_accuracy])\n m.fit(tds, validation_data=vds, epochs=epochs, verbose=1, callbacks=[\n keras.callbacks.ModelCheckpoint(best_ckpt, save_best_only=True, save_weights_only=True,\n monitor=\"val_loss\", mode=\"min\"),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=patience, restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=3, min_lr=1e-8),\n ])\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n\n raw = m.predict(vds, verbose=0)\n true = val_df[\"label\"].values\n opt_thr, opt_qwk = find_optimal_threshold(raw, true)\n preds = np.array([ordinal_logits_to_grade(l, thr=opt_thr) for l in raw])\n\n sample_low = preprocess_image(val_df.iloc[0][\"path\"], sz_low)[np.newaxis]\n sample_high = preprocess_image(val_df.iloc[0][\"path\"], sz_high)[np.newaxis]\n t0 = time.time()\n for _ in range(20): m([sample_low, sample_high], training=False)\n lat_ms = (time.time() - t0) / 20 * 1000\n\n results[\"DualResolutionFusion\"] = {\n \"accuracy\": round(float(accuracy_score(true, preds)), 4),\n \"qwk\": round(float(opt_qwk), 4),\n \"latency_ms\": round(lat_ms, 1),\n \"sizes\": f\"{sz_low}px+{sz_high}px\",\n }\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n print(f\" DualResolutionFusion: acc={results['DualResolutionFusion']['accuracy']:.4f} \"\n f\"qwk={opt_qwk:.4f} lat={lat_ms:.1f}ms\")\n del m; keras.backend.clear_session(); gc.collect()\n auto_zip(tag=\"fusion_dualres_done\")\n return results\n\nprint(\"[OK] Dual-resolution fusion ablation defined.\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell I — Multi-Backbone (CNN + Transformer/Hybrid) Late-Fusion Ablation","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL I — Multi-Backbone (CNN + Transformer/Hybrid) Late-Fusion Ablation\n# Reuses already-trained backbone checkpoints from run_backbone_ablation\n# (frozen), trains only a small fusion head on top — cheap, no full retraining\n# of either backbone.\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef build_multi_backbone_fusion_model(backbone_a=\"EfficientNetB4\", backbone_b=\"MaxViTTiny\",\n ckpt_a=None, ckpt_b=None):\n if backbone_a not in BACKBONE_REGISTRY or backbone_b not in BACKBONE_REGISTRY:\n raise ValueError(f\"Backbones must be in BACKBONE_REGISTRY: {list(BACKBONE_REGISTRY.keys())}\")\n\n m_a = build_model(unfreeze_last=0, backbone_name=backbone_a)\n m_b = build_model(unfreeze_last=0, backbone_name=backbone_b)\n if ckpt_a and os.path.exists(ckpt_a): m_a.load_weights(ckpt_a)\n if ckpt_b and os.path.exists(ckpt_b): m_b.load_weights(ckpt_b)\n m_a.trainable = False\n m_b.trainable = False\n\n feat_a = m_a.get_layer(\"head_bn\").output\n feat_b = m_b.get_layer(\"head_bn\").output\n fused = layers.Concatenate(name=\"concat_backbones\")([feat_a, feat_b])\n fused = layers.Dense(256, activation=\"relu\", name=\"fusion_dense\")(fused)\n fused = layers.Dropout(CFG[\"DROPOUT\"], name=\"fusion_dropout\")(fused)\n fused = layers.Activation(\"linear\", dtype=\"float32\", name=\"fp32_cast\")(fused)\n out = layers.Dense(4, name=\"ordinal_head\", dtype=\"float32\")(fused)\n\n return keras.Model([m_a.input, m_b.input], out, name=f\"DR_fusion_{backbone_a}_{backbone_b}\")\n\n\ndef run_multi_backbone_fusion_ablation(train_df, val_df, backbone_a=\"EfficientNetB4\",\n backbone_b=\"MaxViTTiny\", epochs=20, patience=5):\n out_file = f\"{OUT_DIR}/fusion_multibackbone_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n key = f\"{backbone_a}+{backbone_b}\"\n if key in results:\n print(f\"[SKIP] {key}: qwk={results[key]['qwk']:.4f}\")\n return results\n if backbone_a not in BACKBONE_REGISTRY or backbone_b not in BACKBONE_REGISTRY:\n print(f\"[SKIP] {key}: one or both backbones unavailable on this kernel.\")\n return results\n\n ckpt_a = f\"{CKPT_DIR}/bb_{backbone_a}/best.weights.h5\"\n ckpt_b = f\"{CKPT_DIR}/bb_{backbone_b}/best.weights.h5\"\n if not (os.path.exists(ckpt_a) and os.path.exists(ckpt_b)):\n missing = ckpt_a if not os.path.exists(ckpt_a) else ckpt_b\n print(f\"[SKIP] {key}: run_backbone_ablation must complete for both backbones first \"\n f\"(missing {missing}).\")\n return results\n\n print(f\"\\n{'='*55}\\nFUSION ABLATION: {key} (late feature fusion, frozen backbones)\\n{'='*55}\")\n reg_a = BACKBONE_REGISTRY[backbone_a]\n sz_a, sz_b = reg_a[\"img_size\"], BACKBONE_REGISTRY[backbone_b][\"img_size\"]\n bs = 8 * N_GPUS\n\n def _fusion_ds(df, augment, shuffle):\n paths = df[\"path\"].values\n ord_labels = np.array([grade_to_ordinal(l) for l in df[\"label\"].values], dtype=np.float32)\n ds = tf.data.Dataset.from_tensor_slices((paths, ord_labels))\n if shuffle: ds = ds.shuffle(len(paths), seed=SEED, reshuffle_each_iteration=True)\n def _process(path_t, y):\n path_py = path_t.numpy().decode(\"utf-8\")\n return (preprocess_image(path_py, sz_a, augment),\n preprocess_image(path_py, sz_b, augment), y)\n def _wrap(path_t, y):\n a, b, yy = tf.py_function(_process, [path_t, y], [tf.float32, tf.float32, tf.float32])\n a.set_shape([sz_a, sz_a, 3]); b.set_shape([sz_b, sz_b, 3]); yy.set_shape([4])\n return (a, b), yy\n return ds.map(_wrap, num_parallel_calls=NUM_WORKERS).batch(bs, drop_remainder=shuffle).prefetch(2)\n\n tds = _fusion_ds(train_df, augment=True, shuffle=True)\n vds = _fusion_ds(val_df, augment=False, shuffle=False)\n\n ckpt_dir = f\"{CKPT_DIR}/fusion_multibb_{backbone_a}_{backbone_b}\"\n os.makedirs(ckpt_dir, exist_ok=True)\n best_ckpt = f\"{ckpt_dir}/best.weights.h5\"\n with STRATEGY.scope():\n m = build_multi_backbone_fusion_model(backbone_a, backbone_b, ckpt_a, ckpt_b)\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n m.compile(optimizer=keras.optimizers.AdamW(1e-4, weight_decay=CFG[\"WEIGHT_DECAY\"], clipnorm=1.0),\n loss=ordinal_bce_loss, metrics=[ordinal_accuracy])\n m.fit(tds, validation_data=vds, epochs=epochs, verbose=1, callbacks=[\n keras.callbacks.ModelCheckpoint(best_ckpt, save_best_only=True, save_weights_only=True,\n monitor=\"val_loss\", mode=\"min\"),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=patience, restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=2, min_lr=1e-8),\n ])\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n\n raw = m.predict(vds, verbose=0)\n true = val_df[\"label\"].values\n opt_thr, opt_qwk = find_optimal_threshold(raw, true)\n preds = np.array([ordinal_logits_to_grade(l, thr=opt_thr) for l in raw])\n\n results[key] = {\n \"accuracy\": round(float(accuracy_score(true, preds)), 4),\n \"qwk\": round(float(opt_qwk), 4),\n }\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n print(f\" {key}: acc={results[key]['accuracy']:.4f} qwk={opt_qwk:.4f}\")\n del m; keras.backend.clear_session(); gc.collect()\n auto_zip(tag=f\"fusion_multibb_{backbone_a}_{backbone_b}_done\")\n return results\n\nprint(\"[OK] Multi-backbone fusion ablation defined.\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell J — Swin Transformer Backbone (isolated foundation-style baseline)\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL J — Swin Transformer Backbone (isolated foundation-style baseline)\n# Mirrors the existing ViT-Small comparison (Cell 3, run_vit_comparison): kept\n# OUTSIDE BACKBONE_REGISTRY because Swin's HF interface (NCHW pixel_values,\n# reshaped_hidden_states) differs from the keras.applications loaders used\n# everywhere else. Nothing above this cell is modified.\n#\n# NOTE ON WHAT IS *NOT* ADDED HERE, AND WHY:\n# RETFound, DINOv2, EVA-02 are the retinal/vision *foundation* models the\n# reference material recommends first. They are intentionally NOT wired in:\n# - RETFound ships only as a PyTorch/timm checkpoint (MAE-pretrained ViT).\n# There is no TensorFlow port, and no realistic way to load its weights\n# into this Keras pipeline without a full PyTorch bridge or a from-scratch\n# ONNX conversion — that's a separate project, not a pluggable cell.\n# - DINOv2 (`transformers.Dinov2Model`) is PyTorch/Flax-only in HuggingFace\n# Transformers — there is no `TFDinov2Model`. Same blocker as RETFound.\n# - EVA-02 checkpoints are likewise PyTorch/timm-only.\n# Swin IS added below because `transformers.TFSwinModel` is a real, maintained\n# TensorFlow implementation (confirmed: HF docs + accompanying GitHub issues),\n# so it's the closest thing to \"foundation-style hierarchical transformer\"\n# that actually fits this TF/Keras stack without inventing an integration.\n# If true retinal-foundation-model results are required, the honest path is\n# a separate PyTorch fine-tuning script using RETFound/DINOv2 checkpoints,\n# not a forced fit into this notebook.\n# ═══════════════════════════════════════════════════════════════════════════\n\ntry:\n from transformers import TFSwinModel\n _HAS_SWIN = True\nexcept ImportError:\n _HAS_SWIN = False\n\n\nclass SwinOrdinalModel(keras.Model):\n \"\"\"\n Swin-Tiny-patch4-window7-224 wrapped as an ordinal grading head.\n Unlike ViTOrdinalModel (CLS-token only), Swin's pooler_output already\n reflects a spatially-hierarchical average, which tends to transfer better\n to lesion-scale detail than a single CLS token.\n NOTE: GradCAM is not wired up for Swin here — its windowed-attention maps\n don't correspond to a single late conv-like tensor the way EfficientNet's\n do, and a faithful implementation deserves its own cell rather than a\n hurried reuse of get_grad_model(). Left out rather than added half-right.\n \"\"\"\n def __init__(self, swin_backbone, **kwargs):\n super().__init__(**kwargs)\n self.swin = swin_backbone\n self.norm = layers.LayerNormalization()\n self.drop = layers.Dropout(0.3)\n # FIX: same sigmoid/from_logits double-squash issue as ViTOrdinalModel\n # (see Cell 3 / run_vit_comparison for the full explanation) — raw\n # logits here to match ordinal_bce_loss's from_logits=True.\n self.head = layers.Dense(4, activation=None,\n name=\"ordinal_head\", dtype=\"float32\")\n\n def call(self, inputs, training=False):\n # inputs: (B, H, W, 3) float32 pixels [0-255] — match ViT wrapper's convention\n x = inputs / 127.5 - 1.0\n x = tf.transpose(x, perm=[0, 3, 1, 2]) # NHWC → NCHW (HF format)\n out = self.swin(pixel_values=x, training=training)\n pooled = out.pooler_output\n pooled = self.norm(pooled, training=training)\n pooled = self.drop(pooled, training=training)\n return self.head(pooled)\n\n\ndef run_swin_comparison(train_df, val_df):\n \"\"\"\n Swin-Tiny as an additional transformer baseline, saved into the same\n backbone_ablation.json used by run_vit_comparison, under key 'Swin-Tiny'.\n Fully resumable, same OOM-guarded structure as run_vit_comparison.\n \"\"\"\n # Defensive re-check at call time — same rationale as run_vit_comparison.\n global _HAS_SWIN, TFSwinModel\n try:\n from transformers import TFSwinModel as _TFSwinModel_fresh\n TFSwinModel = _TFSwinModel_fresh\n _HAS_SWIN = True\n except ImportError:\n _HAS_SWIN = False\n if not _HAS_SWIN:\n print(\"[SKIP] Swin: 'transformers' package / TFSwinModel not importable on this kernel\")\n return {}\n\n out_file = f\"{OUT_DIR}/backbone_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n if \"Swin-Tiny\" in results:\n r = results[\"Swin-Tiny\"]\n print(f\"[SKIP] Swin-Tiny already done: acc={r['accuracy']:.4f} qwk={r['qwk']:.4f}\")\n return results\n\n keras.backend.clear_session()\n gc.collect()\n tf.random.set_seed(SEED)\n\n SWIN_ID = \"microsoft/swin-tiny-patch4-window7-224\"\n IMG_SZ = 224\n BS = 8 * N_GPUS\n ckd = f\"{CKPT_DIR}/bb_Swin_Tiny\"\n best_p = f\"{ckd}/best.weights.h5\"\n os.makedirs(ckd, exist_ok=True)\n\n print(f\"\\n{'='*60}\")\n print(f\"SWIN BASELINE: {SWIN_ID} @ {IMG_SZ}px | batch={BS} (OOM-safe)\")\n print(f\"{'='*60}\")\n\n cw = get_class_weights(train_df)\n tds = make_tf_dataset(train_df, BS, IMG_SZ, augment=True, class_weights=cw)\n vds = make_tf_dataset(val_df, BS, IMG_SZ, augment=False, shuffle=False)\n\n best_acc = 0.0\n class _SwinCB(keras.callbacks.Callback):\n def on_epoch_end(self, epoch, logs=None):\n nonlocal best_acc\n va = float((logs or {}).get(\"val_ordinal_accuracy\", 0.0))\n if va > best_acc:\n best_acc = va\n self.model.save_weights(best_p)\n print(f\" [BEST] Swin-Tiny ep={epoch} acc={va:.4f}\")\n\n common_cbs = [\n _SwinCB(),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=4,\n restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5,\n patience=2, min_lr=1e-8),\n ]\n\n try:\n with STRATEGY.scope():\n swin_bb = TFSwinModel.from_pretrained(SWIN_ID, from_pt=True)\n m = SwinOrdinalModel(swin_bb, name=\"Swin_Tiny_Ordinal\")\n _ = m(tf.zeros((1, IMG_SZ, IMG_SZ, 3)), training=False)\n\n # Phase 1: frozen backbone — train head only\n swin_bb.trainable = False\n m.compile(optimizer=keras.optimizers.Adam(1e-3),\n loss=ordinal_bce_loss, metrics=[ordinal_accuracy])\n\n print(\"\\n Phase 1/2: Head-only (frozen Swin, 10 epochs)\")\n m.fit(tds, validation_data=vds, epochs=10, verbose=1, callbacks=common_cbs)\n\n # Phase 2: partial unfreeze — last stage of the Swin encoder\n print(\"\\n Phase 2/2: Partial unfreeze (last encoder stage, 15 epochs)\")\n with STRATEGY.scope():\n try:\n for enc_layer in swin_bb.swin.encoder.layers[-1].blocks:\n enc_layer.trainable = True\n except Exception:\n # Fallback: unfreeze the whole encoder rather than fail silently\n swin_bb.trainable = True\n m.compile(optimizer=keras.optimizers.Adam(5e-6),\n loss=ordinal_bce_loss, metrics=[ordinal_accuracy])\n m.fit(tds, validation_data=vds, epochs=15, verbose=1, callbacks=common_cbs)\n\n if os.path.exists(best_p):\n m.load_weights(best_p)\n\n vds_e = make_tf_dataset(val_df, BS, IMG_SZ, augment=False, shuffle=False)\n raw = m.predict(vds_e, verbose=0)\n true = val_df[\"label\"].values\n # FIX: logit-space optimal-threshold decode (see run_vit_comparison).\n opt_thr, _ = find_optimal_threshold(raw, true)\n preds = np.array([ordinal_logits_to_grade(p, thr=opt_thr) for p in raw])\n acc = float(accuracy_score(true, preds))\n qwk = float(cohen_kappa_score(true, preds, weights=\"quadratic\"))\n f1_m = float(f1_score(true, preds, average=\"macro\", zero_division=0))\n\n samp = preprocess_image(val_df.iloc[0][\"path\"], IMG_SZ)[np.newaxis]\n t0 = time.time()\n for _ in range(20): m(samp, training=False)\n lat = (time.time()-t0)/20*1000\n\n n_p = sum(np.prod(v.shape) for v in m.trainable_variables)\n\n results[\"Swin-Tiny\"] = {\n \"accuracy\": round(acc, 4),\n \"qwk\": round(qwk, 4),\n \"f1_macro\": round(f1_m, 4),\n \"auc_macro\": 0.0, # GradCAM/AUC parity with ViT row — not computed here\n \"latency_ms\": round(lat, 1),\n \"params_M\": round(n_p / 1e6, 1),\n \"img_size\": IMG_SZ,\n \"deploy\": \"ONNX ⚠️ (untested) TFLite-FP32 ⚠️ TFLite-INT8 ❌ BentoML ✅\",\n \"notes\": \"Hierarchical transformer baseline. No GradCAM wired up (see cell docstring).\",\n }\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n print(f\"\\n Swin-Tiny: acc={acc:.4f} qwk={qwk:.4f} lat={lat:.1f}ms\")\n auto_zip(tag=\"swin_comparison_done\")\n\n except tf.errors.ResourceExhaustedError:\n print(\"[OOM] Swin-Tiny ran out of GPU memory — skipping (reduce BS or try FP16)\")\n except Exception as e:\n print(f\"[ERROR] Swin comparison failed: {e}\")\n finally:\n try: del m, swin_bb\n except Exception: pass\n keras.backend.clear_session()\n gc.collect()\n print(\"[OK] Swin GPU memory released\")\n\n return results\n\nprint(\"[OK] Swin-Tiny comparison defined. RETFound/DINOv2/EVA-02 deliberately skipped — see cell docstring.\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell K — Attention Modules (CBAM & Coordinate Attention) Ablation\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL K — Attention Modules (CBAM & Coordinate Attention) Ablation\n# Standard, faithful implementations (Woo et al. 2018 CBAM; Hou et al. 2021\n# Coordinate Attention) inserted between the backbone's spatial feature map\n# and GlobalAveragePooling2D. Additive: duplicates the relevant slice of\n# build_model() rather than editing it, exactly like Extended Cell H/I do.\n#\n# Only applies to backbones that emit a spatial 4D (B,H,W,C) feature map\n# (EfficientNet*, ResNet50V2, DenseNet121, MobileNetV3Large, ConvNeXtTiny,\n# EfficientNetV2*, MaxViTTiny-via-keras_cv). ViT-Small and Swin-Tiny (Cells\n# 3/J) are CLS/pooler-token models with no spatial map to attend over, so\n# they are intentionally excluded rather than forced through a reshape hack.\n# ═══════════════════════════════════════════════════════════════════════════\n\nclass ChannelAttention(layers.Layer):\n \"\"\"CBAM channel-attention sub-module (Woo et al. 2018).\"\"\"\n def __init__(self, reduction_ratio=8, **kwargs):\n super().__init__(**kwargs)\n self.reduction_ratio = reduction_ratio\n\n def build(self, input_shape):\n channels = int(input_shape[-1])\n hidden = max(channels // self.reduction_ratio, 8)\n self.shared_mlp = keras.Sequential([\n layers.Dense(hidden, activation=\"relu\"),\n layers.Dense(channels),\n ])\n self.avg_pool = layers.GlobalAveragePooling2D()\n self.max_pool = layers.GlobalMaxPooling2D()\n super().build(input_shape)\n\n def call(self, x):\n avg_out = self.shared_mlp(self.avg_pool(x))\n max_out = self.shared_mlp(self.max_pool(x))\n scale = tf.nn.sigmoid(avg_out + max_out)\n scale = tf.reshape(scale, (-1, 1, 1, tf.shape(scale)[-1]))\n return x * scale\n\n\nclass SpatialAttention(layers.Layer):\n \"\"\"CBAM spatial-attention sub-module (Woo et al. 2018).\"\"\"\n def __init__(self, kernel_size=7, **kwargs):\n super().__init__(**kwargs)\n self.conv = layers.Conv2D(1, kernel_size, padding=\"same\", activation=\"sigmoid\")\n\n def call(self, x):\n avg_out = tf.reduce_mean(x, axis=-1, keepdims=True)\n max_out = tf.reduce_max(x, axis=-1, keepdims=True)\n concat = tf.concat([avg_out, max_out], axis=-1)\n scale = self.conv(concat)\n return x * scale\n\n\nclass CBAM(layers.Layer):\n \"\"\"Channel attention → spatial attention, applied sequentially (as in the\n original paper, which found channel-then-spatial to outperform parallel\n or spatial-then-channel orderings on their benchmarks).\"\"\"\n def __init__(self, reduction_ratio=8, kernel_size=7, **kwargs):\n super().__init__(**kwargs)\n self.channel_attn = ChannelAttention(reduction_ratio, name=\"cbam_channel\")\n self.spatial_attn = SpatialAttention(kernel_size, name=\"cbam_spatial\")\n\n def call(self, x):\n x = self.channel_attn(x)\n x = self.spatial_attn(x)\n return x\n\n\nclass CoordinateAttention(layers.Layer):\n \"\"\"Coordinate Attention (Hou, Zhou & Feng, CVPR 2021). Factorises channel\n attention into two 1D pooling directions (H and W) so positional\n information is preserved — the paper reports this matters specifically\n for small, spatially-localised targets, which is the regime microaneurysms\n and small hemorrhages sit in.\"\"\"\n def __init__(self, reduction_ratio=32, **kwargs):\n super().__init__(**kwargs)\n self.reduction_ratio = reduction_ratio\n\n def build(self, input_shape):\n channels = int(input_shape[-1])\n mip = max(8, channels // self.reduction_ratio)\n self.conv1 = layers.Conv2D(mip, 1, use_bias=False, name=\"coord_reduce\")\n self.bn1 = layers.BatchNormalization(name=\"coord_bn\")\n self.act = layers.Activation(tf.nn.swish)\n self.conv_h = layers.Conv2D(channels, 1, name=\"coord_h\")\n self.conv_w = layers.Conv2D(channels, 1, name=\"coord_w\")\n super().build(input_shape)\n\n def call(self, x):\n shape = tf.shape(x)\n h, w = shape[1], shape[2]\n x_h = tf.reduce_mean(x, axis=2, keepdims=True) # (B,H,1,C)\n x_w = tf.reduce_mean(x, axis=1, keepdims=True) # (B,1,W,C)\n x_w = tf.transpose(x_w, perm=[0, 2, 1, 3]) # (B,W,1,C)\n y = tf.concat([x_h, x_w], axis=1) # (B,H+W,1,C)\n y = self.conv1(y); y = self.bn1(y); y = self.act(y)\n x_h, x_w = tf.split(y, [h, w], axis=1)\n x_w = tf.transpose(x_w, perm=[0, 2, 1, 3]) # (B,1,W,mip)\n a_h = tf.nn.sigmoid(self.conv_h(x_h)) # (B,H,1,C)\n a_w = tf.nn.sigmoid(self.conv_w(x_w)) # (B,1,W,C)\n return x * a_h * a_w\n\n\ndef build_model_with_attention(attention_type=\"cbam\", unfreeze_last=-1, backbone_name=None):\n \"\"\"\n Same architecture as build_model() but inserts an attention block right\n after the backbone's spatial output, before GlobalAveragePooling2D.\n attention_type: 'none' | 'cbam' | 'coord'\n \"\"\"\n bb_name = backbone_name or CFG[\"ACTIVE_BB\"]\n reg = BACKBONE_REGISTRY[bb_name]\n sz = reg[\"img_size\"]\n base = reg[\"loader\"](sz)\n base.trainable = True\n\n for layer in base.layers:\n if isinstance(layer, layers.BatchNormalization):\n layer.trainable = False\n if unfreeze_last > 0:\n for layer in base.layers[:-unfreeze_last]:\n layer.trainable = False\n elif unfreeze_last == 0:\n base.trainable = False\n\n inp = keras.Input(shape=(sz, sz, 3), name=\"fundus_input\")\n preprocess_fn = reg.get(\"preprocess\", lambda t: t)\n x = layers.Lambda(preprocess_fn, name=\"backbone_preprocess\")(inp)\n x = base(x)\n\n if x.shape.rank != 4:\n raise ValueError(\n f\"build_model_with_attention: backbone '{bb_name}' does not emit a \"\n f\"4D spatial map (got shape {x.shape}); attention modules need \"\n f\"(B,H,W,C). Use a CNN/CNN-hybrid backbone from BACKBONE_REGISTRY.\"\n )\n\n if attention_type == \"cbam\":\n x = CBAM(name=\"cbam\")(x)\n elif attention_type == \"coord\":\n x = CoordinateAttention(name=\"coord_attn\")(x)\n elif attention_type != \"none\":\n raise ValueError(f\"Unknown attention_type: {attention_type}\")\n\n x = layers.GlobalAveragePooling2D(name=\"gap\")(x)\n x = layers.BatchNormalization(name=\"head_bn\")(x)\n x = layers.Dropout(CFG[\"DROPOUT\"], name=\"mc_dropout\")(x)\n x = layers.Activation(\"linear\", dtype=\"float32\", name=\"fp32_cast\")(x)\n out = layers.Dense(4, name=\"ordinal_head\", dtype=\"float32\")(x)\n\n return keras.Model(inp, out, name=f\"DR_{bb_name}_{attention_type}\")\n\n\ndef run_attention_ablation(train_df, val_df, backbone_name=None, epochs=25, patience=6):\n \"\"\"\n Baseline (no attention, i.e. build_model's own architecture) vs CBAM vs\n Coordinate Attention, on CFG['ACTIVE_BB'] (or backbone_name). Resumable,\n same fit/eval/checkpoint pattern as the other Extended ablation cells.\n \"\"\"\n out_file = f\"{OUT_DIR}/attention_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n bb_name = backbone_name or CFG[\"ACTIVE_BB\"]\n IMG, BS = BACKBONE_REGISTRY[bb_name][\"img_size\"], CFG[\"BATCH_SIZE\"]\n cw = get_class_weights(train_df)\n tds = make_tf_dataset(train_df, BS, IMG, augment=True, class_weights=cw)\n vds = make_tf_dataset(val_df, BS, IMG, augment=False, shuffle=False)\n\n for variant in (\"none\", \"cbam\", \"coord\"):\n key = f\"{bb_name}+{variant}\"\n if key in results:\n print(f\"[SKIP] {key}: qwk={results[key]['qwk']:.4f}\")\n continue\n print(f\"\\n{'='*55}\\nATTENTION ABLATION: {key}\\n{'='*55}\")\n ckpt_dir = f\"{CKPT_DIR}/attn_{bb_name}_{variant}\"; os.makedirs(ckpt_dir, exist_ok=True)\n best_ckpt = f\"{ckpt_dir}/best.weights.h5\"\n try:\n with STRATEGY.scope():\n m = build_model_with_attention(attention_type=variant, backbone_name=bb_name)\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n m.compile(optimizer=keras.optimizers.AdamW(1e-4, weight_decay=CFG[\"WEIGHT_DECAY\"], clipnorm=1.0),\n loss=ordinal_bce_loss, metrics=[ordinal_accuracy])\n m.fit(tds, validation_data=vds, epochs=epochs, verbose=1, callbacks=[\n keras.callbacks.ModelCheckpoint(best_ckpt, save_best_only=True, save_weights_only=True,\n monitor=\"val_loss\", mode=\"min\"),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=patience, restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=3, min_lr=1e-8),\n ])\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n raw = m.predict(vds, verbose=0)\n true = val_df[\"label\"].values\n opt_thr, opt_qwk = find_optimal_threshold(raw, true)\n preds = np.array([ordinal_logits_to_grade(l, thr=opt_thr) for l in raw])\n results[key] = {\n \"accuracy\": round(float(accuracy_score(true, preds)), 4),\n \"qwk\": round(float(opt_qwk), 4),\n }\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n del m; keras.backend.clear_session(); gc.collect()\n auto_zip(tag=f\"attn_{bb_name}_{variant}_done\")\n except Exception as e:\n print(f\"[ERROR] Attention ablation '{key}' failed: {e}\")\n\n print(f\"\\n{'='*55}\\nATTENTION ABLATION SUMMARY\\n{'='*55}\")\n for k, v in results.items():\n print(f\" {k:<28} acc={v['accuracy']:.4f} qwk={v['qwk']:.4f}\")\n return results\n\nprint(\"[OK] CBAM / Coordinate Attention modules and run_attention_ablation defined.\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell L — Earth Mover's Distance (EMD) Ordinal Loss Ablation\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL L — Earth Mover's Distance (EMD) Ordinal Loss Ablation\n# The one item from the ordinal-learning list (CORAL/CORN/EMD) not already\n# covered by Extended Cell A/B. EMD operates on full 5-class distributions\n# (softmax), not the 4-way cumulative encoding used by ordinal_bce_loss/\n# focal_ordinal_loss elsewhere — so it reuses build_softmax_model() from\n# Extended Cell A rather than the Dense(4) ordinal head.\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef emd_ordinal_loss(y_true, y_pred):\n \"\"\"Squared Earth Mover's Distance between the true one-hot distribution\n and predicted softmax distribution over the 5 ordered DR grades.\n EMD^2 = sum_k (CDF_true_k - CDF_pred_k)^2, summed over classes.\n Returns PER-EXAMPLE loss (not batch-reduced) so Keras can still apply\n sample weights / batch averaging itself — same convention as\n ordinal_bce_loss and focal_ordinal_loss above.\n \"\"\"\n y_true = tf.cast(y_true, tf.float32)\n y_pred = tf.cast(y_pred, tf.float32)\n cdf_true = tf.cumsum(y_true, axis=-1)\n cdf_pred = tf.cumsum(y_pred, axis=-1)\n return tf.reduce_sum(tf.square(cdf_true - cdf_pred), axis=-1)\n\n\ndef _make_onehot_softmax_dataset(df, batch_size, img_size, augment, shuffle, class_weights=None):\n \"\"\"Wraps make_tf_dataset_integer_labels (Extended Cell B) and one-hots the\n label so it can be paired with emd_ordinal_loss. No changes to that\n function — this only maps its output.\"\"\"\n ds = make_tf_dataset_integer_labels(df, batch_size, img_size, augment=augment,\n shuffle=shuffle, class_weights=class_weights)\n if class_weights is not None:\n return ds.map(lambda x, y, w: (x, tf.one_hot(tf.cast(y, tf.int32), 5), w))\n return ds.map(lambda x, y: (x, tf.one_hot(tf.cast(y, tf.int32), 5)))\n\n\ndef run_emd_ablation(train_df, val_df, epochs=25, patience=6):\n \"\"\"\n Trains the Softmax5class architecture with emd_ordinal_loss instead of\n sparse_categorical_crossentropy, and appends the result to the SAME\n encoding_ablation.json produced by Extended Cell B, under key\n 'EMD_Softmax' — so it shows up alongside Softmax5class / CORAL / CORN\n in one comparison table.\n \"\"\"\n out_file = f\"{OUT_DIR}/encoding_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n if \"EMD_Softmax\" in results:\n print(f\"[SKIP] EMD_Softmax: qwk={results['EMD_Softmax']['qwk']:.4f}\")\n return results\n\n IMG, BS = CFG[\"IMG_SIZE\"], CFG[\"BATCH_SIZE\"]\n cw = get_class_weights(train_df)\n\n print(f\"\\n{'='*55}\\nENCODING ABLATION: EMD_Softmax\\n{'='*55}\")\n tds = _make_onehot_softmax_dataset(train_df, BS, IMG, augment=True, shuffle=True, class_weights=cw)\n vds = _make_onehot_softmax_dataset(val_df, BS, IMG, augment=False, shuffle=False)\n # Un-weighted eval-time dataset (no class_weights) for clean prediction decoding\n vds_eval = _make_onehot_softmax_dataset(val_df, BS, IMG, augment=False, shuffle=False)\n\n ckpt_dir = f\"{CKPT_DIR}/encoding_emd\"; os.makedirs(ckpt_dir, exist_ok=True)\n best_ckpt = f\"{ckpt_dir}/best.weights.h5\"\n try:\n with STRATEGY.scope():\n m = build_softmax_model()\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n m.compile(optimizer=keras.optimizers.AdamW(1e-5, weight_decay=CFG[\"WEIGHT_DECAY\"], clipnorm=1.0),\n loss=emd_ordinal_loss, metrics=[\"accuracy\"])\n m.fit(tds, validation_data=vds, epochs=epochs, verbose=1, callbacks=[\n keras.callbacks.ModelCheckpoint(best_ckpt, save_best_only=True, save_weights_only=True,\n monitor=\"val_loss\", mode=\"min\"),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=patience, restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=3, min_lr=1e-8),\n ])\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n probs = m.predict(vds_eval, verbose=0)\n preds = np.argmax(probs, axis=1)\n true = val_df[\"label\"].values[:len(preds)]\n results[\"EMD_Softmax\"] = {\n \"accuracy\": round(float(accuracy_score(true, preds)), 4),\n \"qwk\": round(float(cohen_kappa_score(true, preds, weights=\"quadratic\")), 4),\n }\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n print(f\" EMD_Softmax: acc={results['EMD_Softmax']['accuracy']:.4f} \"\n f\"qwk={results['EMD_Softmax']['qwk']:.4f}\")\n del m; keras.backend.clear_session(); gc.collect()\n auto_zip(tag=\"encoding_emd_done\")\n except Exception as e:\n print(f\"[ERROR] EMD ablation failed: {e}\")\n\n return results\n\nprint(\"[OK] EMD ordinal loss and run_emd_ablation defined.\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell M — RandAugment & CutMix Ablation\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL M — RandAugment & CutMix Ablation\n# RandAugment: uses keras_cv (already pip-installed in Cell 1, flagged via\n# _HAS_KERAS_CV but never wired into the augmentation pipeline until now) —\n# its documented API (value_range, augmentations_per_image, magnitude) is\n# used as-is, no guessed arguments.\n# CutMix: implemented directly in plain TF ops rather than via\n# keras_cv.layers.CutMix, because that layer expects a {\"images\",\"labels\"}\n# dict with one-hot labels, which doesn't match this pipeline's ordinal\n# 4-vector targets or its (image, label[, weight]) tuple convention. A small\n# custom function is safer here than bending the data format to fit an\n# unverified label-schema assumption.\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef get_randaugment_layer(magnitude=0.3, augmentations_per_image=2):\n if not _HAS_KERAS_CV:\n return None\n return keras_cv.layers.RandAugment(\n value_range=(0, 255),\n augmentations_per_image=augmentations_per_image,\n magnitude=magnitude,\n )\n\n\ndef cutmix_batch(images, ord_labels, alpha=1.0, img_size=None):\n \"\"\"\n Batch-level CutMix. images: (B,H,W,3) float32 [0,255].\n ord_labels: (B,4) float32 ordinal targets (grade_to_ordinal encoding) —\n mixing them linearly by area-fraction is valid because BCE against a\n soft [0,1] target is well-defined (same reasoning MixUp already relies\n on elsewhere in this notebook for the ordinal head).\n \"\"\"\n img_size = img_size or CFG[\"IMG_SIZE\"]\n batch_size = tf.shape(images)[0]\n\n beta = tf.random.gamma([1], alpha) / (tf.random.gamma([1], alpha) + tf.random.gamma([1], alpha) + 1e-8)\n lam = tf.squeeze(beta)\n\n perm = tf.random.shuffle(tf.range(batch_size))\n images2 = tf.gather(images, perm)\n labels2 = tf.gather(ord_labels, perm)\n\n cut_rat = tf.sqrt(1.0 - lam)\n cut_sz = tf.cast(tf.cast(img_size, tf.float32) * cut_rat, tf.int32)\n cx = tf.random.uniform([], 0, img_size, dtype=tf.int32)\n cy = tf.random.uniform([], 0, img_size, dtype=tf.int32)\n x1 = tf.clip_by_value(cx - cut_sz // 2, 0, img_size)\n x2 = tf.clip_by_value(cx + cut_sz // 2, 0, img_size)\n y1 = tf.clip_by_value(cy - cut_sz // 2, 0, img_size)\n y2 = tf.clip_by_value(cy + cut_sz // 2, 0, img_size)\n\n row_idx = tf.range(img_size)[:, None]\n col_idx = tf.range(img_size)[None, :]\n mask = tf.cast((row_idx >= y1) & (row_idx < y2) & (col_idx >= x1) & (col_idx < x2), tf.float32)\n mask = mask[None, :, :, None] # (1,H,W,1) — broadcasts over batch & channels\n\n mixed_images = images * (1.0 - mask) + images2 * mask\n\n box_area = tf.cast((x2 - x1) * (y2 - y1), tf.float32)\n lam_adj = 1.0 - box_area / tf.cast(img_size * img_size, tf.float32)\n mixed_labels = lam_adj * ord_labels + (1.0 - lam_adj) * labels2\n\n return mixed_images, mixed_labels\n\n\ndef make_augmented_dataset(df, batch_size, img_size, mode=\"baseline\", class_weights=None):\n \"\"\"\n mode: 'baseline' — existing preprocess_image augmentation only (flips/\n rotation/brightness), i.e. make_tf_dataset unchanged.\n 'randaugment' — baseline + keras_cv RandAugment on top of each batch.\n 'cutmix' — baseline + batch-level CutMix.\n 'both' — baseline + RandAugment + CutMix.\n Falls back to 'baseline' with a warning if keras_cv is unavailable and\n randaugment/both was requested — never silently trains something other\n than what was asked for without saying so.\n \"\"\"\n base_ds = make_tf_dataset(df, batch_size, img_size, augment=True,\n shuffle=True, class_weights=class_weights)\n\n ra_layer = get_randaugment_layer() if mode in (\"randaugment\", \"both\") else None\n if mode in (\"randaugment\", \"both\") and ra_layer is None:\n print(f\"[WARN] keras_cv unavailable — '{mode}' will run WITHOUT RandAugment.\")\n\n def _apply(*args):\n images, rest = args[0], args[1:]\n if ra_layer is not None:\n images = ra_layer(images, training=True)\n if mode in (\"cutmix\", \"both\") and len(rest) >= 1:\n ord_labels = rest[0]\n images, ord_labels = cutmix_batch(images, ord_labels, img_size=img_size)\n rest = (ord_labels,) + rest[1:]\n return (images,) + rest\n\n return base_ds.map(_apply, num_parallel_calls=tf.data.AUTOTUNE)\n\n\ndef run_augmentation_family_ablation(train_df, val_df, epochs=25, patience=6):\n \"\"\"\n Baseline vs +RandAugment vs +CutMix vs +Both, on CFG['ACTIVE_BB'], using\n the existing ordinal-BCE head (build_model) so results are directly\n comparable to the main pipeline's own numbers. Resumable.\n \"\"\"\n out_file = f\"{OUT_DIR}/augmentation_family_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n IMG, BS = CFG[\"IMG_SIZE\"], CFG[\"BATCH_SIZE\"]\n cw = get_class_weights(train_df)\n vds = make_tf_dataset(val_df, BS, IMG, augment=False, shuffle=False)\n\n for mode in (\"baseline\", \"randaugment\", \"cutmix\", \"both\"):\n if mode in results:\n print(f\"[SKIP] {mode}: qwk={results[mode]['qwk']:.4f}\")\n continue\n print(f\"\\n{'='*55}\\nAUGMENTATION ABLATION: {mode}\\n{'='*55}\")\n tds = make_augmented_dataset(train_df, BS, IMG, mode=mode, class_weights=cw)\n ckpt_dir = f\"{CKPT_DIR}/aug_{mode}\"; os.makedirs(ckpt_dir, exist_ok=True)\n best_ckpt = f\"{ckpt_dir}/best.weights.h5\"\n try:\n with STRATEGY.scope():\n m = build_model(unfreeze_last=-1)\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n m.compile(optimizer=keras.optimizers.AdamW(1e-5, weight_decay=CFG[\"WEIGHT_DECAY\"], clipnorm=1.0),\n loss=ordinal_bce_loss, metrics=[ordinal_accuracy])\n m.fit(tds, validation_data=vds, epochs=epochs, verbose=1, callbacks=[\n keras.callbacks.ModelCheckpoint(best_ckpt, save_best_only=True, save_weights_only=True,\n monitor=\"val_loss\", mode=\"min\"),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=patience, restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=3, min_lr=1e-8),\n ])\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n raw = m.predict(vds, verbose=0)\n true = val_df[\"label\"].values\n opt_thr, opt_qwk = find_optimal_threshold(raw, true)\n preds = np.array([ordinal_logits_to_grade(l, thr=opt_thr) for l in raw])\n results[mode] = {\n \"accuracy\": round(float(accuracy_score(true, preds)), 4),\n \"qwk\": round(float(opt_qwk), 4),\n }\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n del m; keras.backend.clear_session(); gc.collect()\n auto_zip(tag=f\"aug_{mode}_done\")\n except Exception as e:\n print(f\"[ERROR] Augmentation ablation '{mode}' failed: {e}\")\n\n print(f\"\\n{'='*55}\\nAUGMENTATION FAMILY ABLATION SUMMARY\\n{'='*55}\")\n for k, v in results.items():\n print(f\" {k:<14} acc={v['accuracy']:.4f} qwk={v['qwk']:.4f}\")\n return results\n\nprint(\"[OK] RandAugment + CutMix augmentation family ablation defined.\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell N — Foundation Model Linear Probe (2025-2026 benchmark: RETFound / FLAIR / DINOv2 / EVA-02)\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL N — Foundation Model Linear Probe (2025-2026 benchmark)\n# Consumes embeddings produced by the SEPARATE script\n# `foundation_model_feature_extraction.py` (PyTorch, run outside this\n# notebook — see that script's docstring). This cell only does the Keras\n# side: load precomputed embeddings, train the SAME ordinal head style used\n# everywhere else in this notebook on top of them, and log results into the\n# same backbone_ablation.json used by run_vit_comparison / run_swin_comparison\n# so RETFound / FLAIR / DINOv2 / EVA-02 land in the same table as your CNNs.\n#\n# Models covered: RETFound-MAE, FLAIR, DINOv2-base, EVA02-base.\n# NOT covered here (see extraction script docstring for why):\n# RETFound-DINOv2 variant, VisionFM, KeepFIT, EyeFM.\n# ═══════════════════════════════════════════════════════════════════════════\n\nFOUNDATION_MODELS_TO_PROBE = [\"RETFound-MAE\", \"FLAIR\", \"DINOv2-base\", \"EVA02-base\"]\nEMBED_DIR = \"./foundation_embeddings\" # must match EMBED_OUT_DIR in the extraction script\n\n\ndef build_linear_probe_head(embedding_dim, hidden_dim=256, dropout=0.3):\n \"\"\"\n Small MLP ordinal head on top of a frozen foundation-model embedding.\n Mirrors the Dense(4)-sigmoid ordinal head used by build_model() so\n results are directly comparable — same loss (ordinal_bce_loss), same\n decoding (ordinal_to_grade / ordinal_logits_to_grade).\n \"\"\"\n inp = keras.Input(shape=(embedding_dim,), name=\"embedding_input\")\n x = layers.BatchNormalization(name=\"probe_bn\")(inp)\n x = layers.Dense(hidden_dim, activation=\"relu\", name=\"probe_dense\")(x)\n x = layers.Dropout(dropout, name=\"probe_dropout\")(x)\n # FIX: same sigmoid/from_logits mismatch as ViT/Swin heads — this cell's\n # decode already correctly uses find_optimal_threshold + ordinal_logits_to_grade\n # (logit-space), so it was being fed an already-sigmoided value. Emitting\n # raw logits here makes the whole chain (loss + decode) consistent.\n out = layers.Dense(4, activation=None, name=\"ordinal_head\", dtype=\"float32\")(x)\n return keras.Model(inp, out, name=\"LinearProbe\")\n\n\ndef run_foundation_model_linear_probe(epochs=40, patience=8, batch_size=64):\n \"\"\"\n For each model in FOUNDATION_MODELS_TO_PROBE:\n 1. Load {model}_train.npz / {model}_val.npz / {model}_test.npz\n 2. Convert integer labels -> ordinal 4-vector encoding (grade_to_ordinal,\n already defined earlier in this notebook)\n 3. Train a linear-probe head, evaluate the same way as every other\n backbone (accuracy, QWK, macro-F1)\n 4. Append to backbone_ablation.json under key '{model}-LinearProbe'\n Skips cleanly (with a clear message) if a model's .npz files aren't\n found — this is expected until you've run the separate extraction\n script on a GPU runtime.\n \"\"\"\n out_file = f\"{OUT_DIR}/backbone_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n\n for model_name in FOUNDATION_MODELS_TO_PROBE:\n key = f\"{model_name}-LinearProbe\"\n if key in results:\n print(f\"[SKIP] {key}: qwk={results[key]['qwk']:.4f}\")\n continue\n\n train_p = f\"{EMBED_DIR}/{model_name}_train.npz\"\n val_p = f\"{EMBED_DIR}/{model_name}_val.npz\"\n test_p = f\"{EMBED_DIR}/{model_name}_test.npz\"\n if not (os.path.exists(train_p) and os.path.exists(val_p)):\n print(f\"[SKIP] {model_name}: embeddings not found at {EMBED_DIR}/. \"\n f\"Run foundation_model_feature_extraction.py on a GPU runtime first.\")\n continue\n\n print(f\"\\n{'='*55}\\nFOUNDATION MODEL LINEAR PROBE: {model_name}\\n{'='*55}\")\n tr = np.load(train_p); va = np.load(val_p)\n X_train, y_train = tr[\"embeddings\"], tr[\"labels\"]\n X_val, y_val = va[\"embeddings\"], va[\"labels\"]\n\n Y_train = np.stack([grade_to_ordinal(g) for g in y_train])\n Y_val = np.stack([grade_to_ordinal(g) for g in y_val])\n\n emb_dim = X_train.shape[-1]\n ckpt_dir = f\"{CKPT_DIR}/fm_probe_{model_name}\"; os.makedirs(ckpt_dir, exist_ok=True)\n best_ckpt = f\"{ckpt_dir}/best.weights.h5\"\n\n try:\n with STRATEGY.scope():\n m = build_linear_probe_head(emb_dim)\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n m.compile(optimizer=keras.optimizers.Adam(1e-3),\n loss=ordinal_bce_loss, metrics=[ordinal_accuracy])\n m.fit(X_train, Y_train, validation_data=(X_val, Y_val),\n batch_size=batch_size, epochs=epochs, verbose=1, callbacks=[\n keras.callbacks.ModelCheckpoint(best_ckpt, save_best_only=True,\n save_weights_only=True,\n monitor=\"val_loss\", mode=\"min\"),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=patience,\n restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5,\n patience=3, min_lr=1e-8),\n ])\n if os.path.exists(best_ckpt): m.load_weights(best_ckpt)\n raw = m.predict(X_val, verbose=0)\n opt_thr, opt_qwk = find_optimal_threshold(raw, y_val)\n preds = np.array([ordinal_logits_to_grade(l, thr=opt_thr) for l in raw])\n\n results[key] = {\n \"accuracy\": round(float(accuracy_score(y_val, preds)), 4),\n \"qwk\": round(float(opt_qwk), 4),\n \"f1_macro\": round(float(f1_score(y_val, preds, average=\"macro\", zero_division=0)), 4),\n \"embed_dim\": int(emb_dim),\n \"notes\": \"Linear probe (frozen backbone) — see foundation_model_feature_extraction.py\",\n }\n with open(out_file, \"w\") as f: json.dump(results, f, indent=2)\n print(f\" {key}: acc={results[key]['accuracy']:.4f} qwk={results[key]['qwk']:.4f} \"\n f\"(embed_dim={emb_dim})\")\n del m; keras.backend.clear_session(); gc.collect()\n auto_zip(tag=f\"fm_probe_{model_name}_done\")\n except Exception as e:\n print(f\"[ERROR] Linear probe for {model_name} failed: {e}\")\n\n print(f\"\\n{'='*55}\\nFOUNDATION MODEL BENCHMARK SUMMARY\\n{'='*55}\")\n for m in FOUNDATION_MODELS_TO_PROBE:\n k = f\"{m}-LinearProbe\"\n if k in results:\n print(f\" {k:<28} acc={results[k]['accuracy']:.4f} qwk={results[k]['qwk']:.4f}\")\n else:\n print(f\" {k:<28} [not run — embeddings missing]\")\n return results\n\nprint(\"[OK] Foundation model linear-probe head and run_foundation_model_linear_probe defined.\")\nprint(\"[NOTE] Requires foundation_model_feature_extraction.py to be run separately on a GPU/PyTorch runtime first.\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell O — Ensemble (Top-K Trained Backbones + Foundation Probes, no retraining)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL O — Ensemble of Trained Backbones (+ Foundation-Model Probes)\n# Zero retraining: reuses whatever checkpoints already exist on disk from\n# run_backbone_ablation (CNNs), run_swin_comparison / run_vit_comparison\n# (Swin/ViT, logged into the SAME backbone_ablation.json), and\n# run_foundation_model_linear_probe (RETFound/FLAIR/DINOv2/EVA-02, also in\n# backbone_ablation.json under the \"-LinearProbe\" keys).\n#\n# Strategy: pick the top-K entries in backbone_ablation.json BY QWK whose\n# checkpoint actually exists, run a single forward pass (+ H-flip TTA) per\n# member on val/test, average logits (plain mean AND QWK-weighted mean),\n# and report whichever variant wins on val. This is a probability-level\n# (\"late\") ensemble — cheap, and does not touch any individual model's\n# training.\n#\n# Gracefully skips if fewer than 2 trained+checkpointed members are found,\n# and skips re-computation if the same member-set was already ensembled.\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef _get_backbone_ensemble_candidates(top_k=5, min_qwk=0.0):\n \"\"\"Reads backbone_ablation.json, keeps only entries whose checkpoint is\n actually present on disk, sorted by qwk desc, capped at top_k.\"\"\"\n ab_file = f\"{OUT_DIR}/backbone_ablation.json\"\n if not os.path.exists(ab_file):\n return []\n all_results = json.load(open(ab_file))\n candidates = []\n for name, res in all_results.items():\n if not isinstance(res, dict) or \"qwk\" not in res:\n continue\n if res[\"qwk\"] < min_qwk:\n continue\n if name.endswith(\"-LinearProbe\"):\n model_key = name.replace(\"-LinearProbe\", \"\")\n ckpt = f\"{CKPT_DIR}/fm_probe_{model_key}/best.weights.h5\"\n kind = \"foundation_probe\"\n backbone_name = model_key\n else:\n ckpt = f\"{CKPT_DIR}/bb_{name}/best.weights.h5\"\n kind = \"image_backbone\"\n backbone_name = name\n if os.path.exists(ckpt):\n candidates.append({\n \"name\": name, \"qwk\": res[\"qwk\"], \"ckpt\": ckpt,\n \"kind\": kind, \"backbone_name\": backbone_name,\n })\n candidates.sort(key=lambda d: d[\"qwk\"], reverse=True)\n return candidates[:top_k]\n\n\ndef _predict_image_backbone_logits(backbone_name, ckpt_path, df, batch_size=8):\n \"\"\"Single forward pass + H-flip TTA (no MC-dropout — kept cheap since this\n runs once per ensemble member, on top of each member's own full eval).\"\"\"\n sz = BACKBONE_REGISTRY[backbone_name][\"img_size\"]\n with STRATEGY.scope():\n m = build_model(unfreeze_last=0, backbone_name=backbone_name)\n m.load_weights(ckpt_path)\n paths = df[\"path\"].tolist()\n logits_chunks = []\n for i in range(0, len(paths), batch_size):\n imgs = np.array([preprocess_image(p, sz) for p in paths[i:i + batch_size]])\n p1 = m(imgs, training=False).numpy()\n p2 = m(imgs[:, :, ::-1, :], training=False).numpy()\n logits_chunks.append((p1 + p2) / 2.0)\n logits = np.concatenate(logits_chunks, axis=0)\n del m; keras.backend.clear_session(); gc.collect()\n return logits\n\n\ndef _predict_foundation_probe_logits(model_name, ckpt_path, df, split_name):\n \"\"\"Loads precomputed embeddings for `split_name` and runs the trained\n linear-probe head. Returns None if embeddings for this split/model don't\n exist (e.g. FLAIR/RETFound not yet extracted) — caller skips that member\n for that split rather than failing the whole ensemble.\"\"\"\n npz_path = f\"{EMBED_DIR}/{model_name}_{split_name}.npz\"\n if not os.path.exists(npz_path):\n return None\n data = np.load(npz_path, allow_pickle=True)\n X, emb_paths = data[\"embeddings\"], data[\"paths\"]\n path_to_idx = {p: i for i, p in enumerate(emb_paths)}\n order = [path_to_idx.get(p) for p in df[\"path\"].tolist()]\n if any(o is None for o in order):\n print(f\" [WARN] {model_name}: some rows of this split aren't in {npz_path}; skipping member.\")\n return None\n X_ord = X[order]\n m = build_linear_probe_head(X.shape[-1])\n m.load_weights(ckpt_path)\n logits = m.predict(X_ord, verbose=0)\n del m; keras.backend.clear_session(); gc.collect()\n return logits\n\n\ndef run_ensemble(train_df, val_df, test_df=None, top_k=5, min_qwk=0.0):\n out_file = f\"{OUT_DIR}/ensemble_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n\n candidates = _get_backbone_ensemble_candidates(top_k=top_k, min_qwk=min_qwk)\n if len(candidates) < 2:\n print(f\"[SKIP] Ensemble needs >=2 trained+checkpointed models; found {len(candidates)}. \"\n f\"Run the backbone/Swin/attention/foundation-probe ablations first.\")\n return results\n\n ens_key = \"+\".join(c[\"name\"] for c in candidates)\n if ens_key in results:\n print(f\"[SKIP] Ensemble '{ens_key}' already exists -> qwk={results[ens_key]['qwk']:.4f}\")\n return results\n\n print(f\"\\n{'='*55}\\nENSEMBLE: top {len(candidates)} models by QWK\\n{'='*55}\")\n for c in candidates:\n print(f\" - {c['name']} (qwk={c['qwk']:.4f}, kind={c['kind']})\")\n\n def _collect_split_logits(df, split_name):\n per_model_logits, weights, used_names = [], [], []\n for c in candidates:\n try:\n if c[\"kind\"] == \"image_backbone\":\n logits = _predict_image_backbone_logits(c[\"backbone_name\"], c[\"ckpt\"], df)\n else:\n logits = _predict_foundation_probe_logits(c[\"backbone_name\"], c[\"ckpt\"], df, split_name)\n if logits is None:\n print(f\" [SKIP-MEMBER] {c['name']}: unavailable for split '{split_name}'.\")\n continue\n per_model_logits.append(logits)\n weights.append(c[\"qwk\"])\n used_names.append(c[\"name\"])\n print(f\" [OK] {c['name']} on '{split_name}': logits {logits.shape}\")\n except Exception as e:\n print(f\" [SKIP-MEMBER] {c['name']} failed on split '{split_name}': {e}\")\n return per_model_logits, weights, used_names\n\n val_logits_list, val_weights, val_used = _collect_split_logits(val_df, \"val\")\n if len(val_logits_list) < 2:\n print(\"[SKIP] Fewer than 2 members produced usable val predictions — aborting ensemble.\")\n return results\n\n val_stack = np.stack(val_logits_list, axis=0) # (M, N, 4)\n w = np.array(val_weights, dtype=np.float64); w = w / w.sum()\n\n def _weighted_avg(stack, weights):\n return np.tensordot(weights, stack, axes=([0], [0]))\n\n y_val = val_df[\"label\"].values\n variants = {}\n for tag, lg in [(\"simple_avg\", val_stack.mean(axis=0)),\n (\"qwk_weighted_avg\", _weighted_avg(val_stack, w))]:\n opt_thr, opt_qwk = find_optimal_threshold(lg, y_val)\n preds = np.array([ordinal_logits_to_grade(l, thr=opt_thr) for l in lg])\n variants[tag] = {\n \"accuracy\": round(float(accuracy_score(y_val, preds)), 4),\n \"qwk\": round(float(opt_qwk), 4),\n \"f1_macro\": round(float(f1_score(y_val, preds, average=\"macro\", zero_division=0)), 4),\n \"threshold\": round(float(opt_thr), 4),\n }\n print(f\" [{tag}] val acc={variants[tag]['accuracy']:.4f} qwk={variants[tag]['qwk']:.4f}\")\n\n best_tag = max(variants, key=lambda k: variants[k][\"qwk\"])\n best = variants[best_tag]\n\n test_result = None\n if test_df is not None and len(test_df) > 0:\n test_logits_list, _, test_used = _collect_split_logits(test_df, \"test\")\n if len(test_logits_list) >= 2:\n test_stack = np.stack(test_logits_list, axis=0)\n lg = test_stack.mean(axis=0) if best_tag == \"simple_avg\" else _weighted_avg(test_stack, w)\n y_test = test_df[\"label\"].values\n preds = np.array([ordinal_logits_to_grade(l, thr=best[\"threshold\"]) for l in lg])\n test_result = {\n \"accuracy\": round(float(accuracy_score(y_test, preds)), 4),\n \"qwk\": round(float(cohen_kappa_score(y_test, preds, weights=\"quadratic\")), 4),\n \"f1_macro\": round(float(f1_score(y_test, preds, average=\"macro\", zero_division=0)), 4),\n \"members_used\": test_used,\n }\n print(f\" [TEST/{best_tag}] acc={test_result['accuracy']:.4f} qwk={test_result['qwk']:.4f}\")\n else:\n print(\" [SKIP] Fewer than 2 members produced usable test predictions.\")\n\n results[ens_key] = {\n \"members_requested\": [c[\"name\"] for c in candidates],\n \"members_used_val\": val_used,\n \"member_qwks\": {c[\"name\"]: c[\"qwk\"] for c in candidates},\n \"variants\": variants,\n \"best_variant\": best_tag,\n \"accuracy\": best[\"accuracy\"],\n \"qwk\": best[\"qwk\"],\n \"f1_macro\": best[\"f1_macro\"],\n \"test\": test_result,\n \"notes\": (\"Late/probability-level ensemble of independently trained backbones \"\n \"(+ foundation-model linear probes where embeddings exist). No \"\n \"retraining — reuses existing checkpoints/embeddings as-is.\"),\n }\n with open(out_file, \"w\") as f:\n json.dump(results, f, indent=2)\n print(f\"[SAVED] {out_file}\")\n auto_zip(tag=\"ensemble_done\")\n return results\n\nprint(\"[OK] Ensemble (top-K checkpoint reuse, late fusion) defined.\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell P — Patient-Level vs. Naive Image-Level Split (Leakage Inflation Proof)\n","metadata":{}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL P — Patient-Level vs. Naive Image-Level Split\n# (Leakage Inflation Proof) — EYEPACS-ONLY, ISOLATED COMPARISON\n#\n# This notebook's own splits (Cell 7, make_splits) are ALWAYS patient-level —\n# correctly so. This cell exists ONLY to empirically demonstrate WHY that\n# matters, by holding EVERYTHING except the split methodology fixed:\n#\n# REWRITE NOTE: the previous version of this cell built its naive split\n# from `df_all` (all 4 pooled datasets) and compared it against the MAIN\n# PIPELINE's already-trained result (outputs/eval_val.json). That result\n# is trained on all 4 sources pooled together and evaluated on an\n# EyePACS-only val set (see make_splits: only EyePACS patients are ever\n# split into val/test; APTOS/Messidor/IDRiD-train go entirely into train).\n# Comparing against it would have confounded THREE things at once: (a)\n# training-population composition, (b) evaluation population, and (c)\n# split methodology — leakage would not have been isolated as the only\n# variable.\n#\n# This version fixes that: BOTH variants below are trained AND evaluated\n# on splits built from the SAME EyePACS-only subset of df_all, with\n# identical proportions (val_frac/test_frac) and the same SEED, using the\n# same architecture (build_model(unfreeze_last=-1)) and the same ordinal\n# BCE loss. The ONLY thing that differs between the two runs is whether\n# the split respects patient_id or ignores it. APTOS/Messidor/IDRiD are\n# excluded from this specific comparison entirely — they're untouched by\n# this cell and by the rest of the notebook's data pipeline.\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef make_eyepacs_patient_level_split(df_eyepacs, val_frac=0.10, test_frac=0.10, seed=None):\n \"\"\"\n Mirrors make_splits()'s EyePACS branch exactly: per-patient label taken\n as the mode across that patient's images, a stratified train_test_split\n on the (train) vs (val+test) boundary, then an unstratified split of\n the remainder into val/test — same two-stage convention as Cell 7, just\n standalone here so this comparison trains its own patient-level model\n from scratch rather than reusing the main pipeline's pooled-training run.\n No patient's images can appear in more than one of train/val/test.\n \"\"\"\n seed = seed if seed is not None else SEED\n pat = (df_eyepacs.groupby(\"patient_id\")\n .agg(label=(\"label\", lambda x: x.mode()[0]))\n .reset_index())\n\n tr_p, tmp_p = train_test_split(\n pat[\"patient_id\"].values, test_size=val_frac + test_frac,\n stratify=pat[\"label\"].values, random_state=seed)\n va_p, te_p = train_test_split(\n tmp_p, test_size=test_frac / (val_frac + test_frac), random_state=seed)\n\n tr = df_eyepacs[df_eyepacs[\"patient_id\"].isin(tr_p)].reset_index(drop=True)\n va = df_eyepacs[df_eyepacs[\"patient_id\"].isin(va_p)].reset_index(drop=True)\n te = df_eyepacs[df_eyepacs[\"patient_id\"].isin(te_p)].reset_index(drop=True)\n\n overlap_check = (set(tr[\"patient_id\"]) & set(va[\"patient_id\"])) | \\\n (set(tr[\"patient_id\"]) & set(te[\"patient_id\"])) | \\\n (set(va[\"patient_id\"]) & set(te[\"patient_id\"]))\n print(f\"[EYEPACS PATIENT-LEVEL SPLIT] train={len(tr):,} val={len(va):,} test={len(te):,} \"\n f\"({pat.shape[0]:,} unique patients, {len(overlap_check)} overlapping — should be 0)\")\n return tr, va, te\n\n\ndef make_naive_image_level_split(df, val_frac=0.10, test_frac=0.10, seed=None):\n \"\"\"\n Naive split that ignores patient_id entirely — the same patient's\n images (e.g. left/right eye) can end up split across train/val/test.\n This is exactly the methodological flaw this thesis's literature review\n repeatedly flags in other papers. Built ONLY to quantify that flaw's\n effect on a like-for-like EyePACS-only subset — not used elsewhere.\n \"\"\"\n seed = seed if seed is not None else SEED\n idx = df.index.values\n tr_i, tmp_i = train_test_split(idx, test_size=val_frac + test_frac,\n stratify=df[\"label\"].values, random_state=seed)\n va_i, te_i = train_test_split(tmp_i, test_size=test_frac / (val_frac + test_frac),\n random_state=seed)\n tr = df.loc[tr_i].reset_index(drop=True)\n va = df.loc[va_i].reset_index(drop=True)\n te = df.loc[te_i].reset_index(drop=True)\n\n overlap_tv = set(tr[\"patient_id\"]) & set(va[\"patient_id\"])\n overlap_tt = set(tr[\"patient_id\"]) & set(te[\"patient_id\"])\n overlap_vt = set(va[\"patient_id\"]) & set(te[\"patient_id\"])\n print(f\"[NAIVE SPLIT, EyePACS-only] train={len(tr):,} val={len(va):,} test={len(te):,}\")\n print(f\" Patients leaking across train/val : {len(overlap_tv):,}\")\n print(f\" Patients leaking across train/test : {len(overlap_tt):,}\")\n print(f\" Patients leaking across val/test : {len(overlap_vt):,}\")\n return tr, va, te\n\n\ndef run_leakage_ablation(df_all, epochs=25, patience=6, val_frac=0.10, test_frac=0.10):\n \"\"\"\n Isolates data-leakage inflation as the ONLY variable. Both models are\n trained/evaluated on splits built from the SAME EyePACS-only subset of\n df_all, same proportions, same SEED, same architecture\n (build_model(unfreeze_last=-1)), same ordinal_bce_loss. APTOS/Messidor/\n IDRiD are excluded from this specific comparison entirely.\n Resumable: each variant (patient_level / naive_image_level) is skipped\n independently if already computed, so an interrupted run only redoes\n whichever variant didn't finish. Saves to outputs/leakage_ablation.json.\n \"\"\"\n out_file = f\"{OUT_DIR}/leakage_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n\n if \"patient_level\" in results and \"naive_image_level\" in results:\n print(f\"[SKIP] Both leakage-ablation variants already done.\")\n _print_leakage_summary(results)\n return results\n\n # ── Scope to EyePACS ONLY for this specific comparison ──────────────────\n df_eyepacs = df_all[df_all[\"source\"] == \"eyepacs\"].reset_index(drop=True)\n print(f\"[LEAKAGE ABLATION] Scoped to EyePACS only: {len(df_eyepacs):,} images, \"\n f\"{df_eyepacs['patient_id'].nunique():,} unique patients \"\n f\"(APTOS/Messidor/IDRiD excluded from this comparison)\")\n\n IMG, BS = CFG[\"IMG_SIZE\"], CFG[\"BATCH_SIZE\"]\n\n def _train_and_eval(tr, va, tag):\n cw = get_class_weights(tr)\n tds = make_tf_dataset(tr, BS, IMG, augment=True, class_weights=cw)\n vds = make_tf_dataset(va, BS, IMG, augment=False, shuffle=False)\n\n ckpt_dir = f\"{CKPT_DIR}/leakage_{tag}\"\n os.makedirs(ckpt_dir, exist_ok=True)\n best_ckpt = f\"{ckpt_dir}/best.weights.h5\"\n\n with STRATEGY.scope():\n m = build_model(unfreeze_last=-1)\n if os.path.exists(best_ckpt):\n m.load_weights(best_ckpt)\n m.compile(optimizer=keras.optimizers.AdamW(\n 1e-4, weight_decay=CFG[\"WEIGHT_DECAY\"], clipnorm=1.0),\n loss=ordinal_bce_loss, metrics=[ordinal_accuracy])\n\n m.fit(tds, validation_data=vds, epochs=epochs, verbose=1, callbacks=[\n keras.callbacks.ModelCheckpoint(best_ckpt, save_best_only=True,\n save_weights_only=True,\n monitor=\"val_loss\", mode=\"min\"),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=patience,\n restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5,\n patience=3, min_lr=1e-8),\n ])\n\n if os.path.exists(best_ckpt):\n m.load_weights(best_ckpt)\n raw = m.predict(vds, verbose=0)\n true = va[\"label\"].values\n opt_thr, opt_qwk = find_optimal_threshold(raw, true)\n preds = np.array([ordinal_logits_to_grade(l, thr=opt_thr) for l in raw])\n\n res = {\n \"accuracy\": round(float(accuracy_score(true, preds)), 4),\n \"qwk\": round(float(opt_qwk), 4),\n \"f1_macro\": round(float(f1_score(true, preds, average=\"macro\", zero_division=0)), 4),\n \"n_train\": int(len(tr)),\n \"n_val\": int(len(va)),\n }\n del m\n keras.backend.clear_session()\n gc.collect()\n return res\n\n # ── Variant 1/2: patient-level (correct) split, EyePACS-only ────────────\n if \"patient_level\" not in results:\n print(f\"\\n{'='*60}\\nLEAKAGE ABLATION — Variant 1/2: \"\n f\"Patient-Level Split (EyePACS-only)\\n{'='*60}\")\n tr_p, va_p, te_p = make_eyepacs_patient_level_split(\n df_eyepacs, val_frac=val_frac, test_frac=test_frac)\n try:\n results[\"patient_level\"] = _train_and_eval(tr_p, va_p, \"eyepacs_patient\")\n with open(out_file, \"w\") as f:\n json.dump(results, f, indent=2)\n auto_zip(tag=\"leakage_ablation_patient_level_done\")\n r = results[\"patient_level\"]\n print(f\" patient_level: acc={r['accuracy']:.4f} qwk={r['qwk']:.4f} \"\n f\"(n_train={r['n_train']:,} n_val={r['n_val']:,})\")\n except Exception as e:\n print(f\"[ERROR] Patient-level leakage variant failed: {e}\")\n else:\n print(f\"[SKIP] patient_level already done: qwk={results['patient_level']['qwk']:.4f}\")\n\n # ── Variant 2/2: naive image-level split, SAME EyePACS-only subset ─────\n if \"naive_image_level\" not in results:\n print(f\"\\n{'='*60}\\nLEAKAGE ABLATION — Variant 2/2: \"\n f\"Naive Image-Level Split (EyePACS-only)\\n{'='*60}\")\n tr_n, va_n, te_n = make_naive_image_level_split(\n df_eyepacs, val_frac=val_frac, test_frac=test_frac)\n try:\n results[\"naive_image_level\"] = _train_and_eval(tr_n, va_n, \"eyepacs_naive\")\n with open(out_file, \"w\") as f:\n json.dump(results, f, indent=2)\n auto_zip(tag=\"leakage_ablation_naive_done\")\n r = results[\"naive_image_level\"]\n print(f\" naive_image_level: acc={r['accuracy']:.4f} qwk={r['qwk']:.4f} \"\n f\"(n_train={r['n_train']:,} n_val={r['n_val']:,})\")\n except Exception as e:\n print(f\"[ERROR] Naive image-level leakage variant failed: {e}\")\n else:\n print(f\"[SKIP] naive_image_level already done: qwk={results['naive_image_level']['qwk']:.4f}\")\n\n _print_leakage_summary(results)\n return results\n\n\ndef _print_leakage_summary(results):\n print(f\"\\n{'='*60}\\nLEAKAGE ABLATION: Patient-Level vs. Naive Image-Level Split \"\n f\"(EyePACS-only, isolated comparison)\\n{'='*60}\")\n if \"patient_level\" in results and \"naive_image_level\" in results:\n pl, nv = results[\"patient_level\"], results[\"naive_image_level\"]\n print(f\" Patient-level split (EyePACS-only, this thesis's methodology):\")\n print(f\" accuracy={pl['accuracy']:.4f} qwk={pl['qwk']:.4f} \"\n f\"f1_macro={pl['f1_macro']:.4f} (n_train={pl['n_train']:,} n_val={pl['n_val']:,})\")\n print(f\" Naive image-level split (EyePACS-only, same proportions/seed/architecture —\")\n print(f\" the flaw this thesis critiques):\")\n print(f\" accuracy={nv['accuracy']:.4f} qwk={nv['qwk']:.4f} \"\n f\"f1_macro={nv['f1_macro']:.4f} (n_train={nv['n_train']:,} n_val={nv['n_val']:,})\")\n d_qwk = nv[\"qwk\"] - pl[\"qwk\"]\n d_acc = nv[\"accuracy\"] - pl[\"accuracy\"]\n print(f\" Inflation from leakage (naive minus patient-level; split methodology\")\n print(f\" is the ONLY variable that differs between these two runs):\")\n print(f\" delta_QWK={d_qwk:+.4f} delta_accuracy={d_acc:+.4f}\")\n if d_qwk > 0 or d_acc > 0:\n print(\" -> Naive split scores HIGHER, consistent with data leakage inflating results.\")\n else:\n print(\" -> No inflation observed in this run -- report the numbers as-is, \"\n \"don't force the expected direction.\")\n else:\n missing = [k for k in (\"patient_level\", \"naive_image_level\") if k not in results]\n print(f\" [INCOMPLETE] Missing: {missing}. Run this cell fully \"\n f\"(both variants train from scratch, EyePACS-only) to compare.\")\n\n\nprint(\"[OK] Leakage ablation (patient-level vs. naive image-level split, EyePACS-only) defined.\")\nprint(\"[NOTE] Both variants train from scratch on EyePACS-only data — this no longer reuses\")\nprint(\" the main pipeline's pooled-training eval_val.json, so leakage is the only\")\nprint(\" variable being tested (no confound from training-population or eval-set differences).\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell Q — Feature-Space SMOTE Ablation (real implementation)","metadata":{}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL Q — Feature-Space SMOTE Ablation (real implementation)\n# Replaces the plain \"oversample\" flag used elsewhere in this notebook with\n# an actual SMOTE run, addressing the thesis's stated \"evaluating the exact\n# impact of SMOTE on Grade 3/4 imbalance\" claim for real.\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef extract_frozen_features(model, df, batch_size=32):\n \"\"\"\n Extract the model's penultimate feature vector (the \"head_bn\" layer —\n post-GAP, post-BatchNorm, the same representation build_model()'s head\n classifies from) for every image in df, using the model's CURRENT\n trained weights. Returns (features [N, D], labels [N]).\n \"\"\"\n try:\n feat_layer = model.get_layer(\"head_bn\")\n except ValueError:\n # Defensive fallback if a differently-built model doesn't have this\n # exact layer name — use the second-to-last layer instead.\n feat_layer = model.layers[-2]\n print(f\" [WARN] 'head_bn' layer not found — falling back to \"\n f\"'{feat_layer.name}' for feature extraction.\")\n feat_model = keras.Model(inputs=model.input, outputs=feat_layer.output)\n\n feats = []\n paths = df[\"path\"].tolist()\n sz = CFG[\"IMG_SIZE\"]\n for i in range(0, len(paths), batch_size):\n batch_paths = paths[i:i + batch_size]\n imgs = np.array([preprocess_image(p, sz, augment=False) for p in batch_paths])\n feats.append(feat_model(imgs, training=False).numpy())\n X = np.concatenate(feats, axis=0)\n y = df[\"label\"].values\n return X, y\n\n\ndef run_smote_ablation(model, train_df, val_df, k_neighbors=5):\n \"\"\"\n SMOTE for image classification only makes sense in a continuous feature\n space — linearly interpolating two raw fundus photos in pixel space\n produces a blurred, semantically meaningless ghost image, not a\n plausible synthetic fundus. Standard practice (and what's done here) is:\n 1. Extract frozen backbone features (post-GAP, post-BN) using the\n current trained model.\n 2. Apply imbalanced-learn's SMOTE in that feature space.\n 3. Train a lightweight linear-probe head (same architecture family as\n Extended Cell N's build_linear_probe_head) on real+synthetic\n features, and separately on real-only features with class weights,\n so SMOTE's effect is isolated against a fair, same-architecture\n baseline instead of the full end-to-end fine-tuned CNN (a\n different regime — that comparison would confound \"SMOTE\" with\n \"much smaller/faster model\").\n Saves to smote_ablation.json. Resumable.\n \"\"\"\n out_file = f\"{OUT_DIR}/smote_ablation.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n if \"LinearProbe_Baseline\" in results and \"LinearProbe_SMOTE\" in results:\n for k in (\"LinearProbe_Baseline\", \"LinearProbe_SMOTE\"):\n r = results[k]\n print(f\"[SKIP] {k}: acc={r['accuracy']:.4f} qwk={r['qwk']:.4f}\")\n return results\n\n from imblearn.over_sampling import SMOTE\n from collections import Counter\n\n print(f\"\\n{'='*60}\\nSMOTE ABLATION (feature-space)\\n{'='*60}\")\n print(\"[1/4] Extracting frozen features for train/val...\")\n X_train, y_train = extract_frozen_features(model, train_df)\n X_val, y_val = extract_frozen_features(model, val_df)\n print(f\" train features: {X_train.shape}, val features: {X_val.shape}\")\n counts = Counter(y_train)\n print(f\" train class counts: {dict(sorted(counts.items()))}\")\n\n def _train_and_eval(X_tr, y_tr, tag, use_class_weight=False):\n Y_tr = np.stack([grade_to_ordinal(g) for g in y_tr])\n Y_va = np.stack([grade_to_ordinal(g) for g in y_val])\n ckpt_dir = f\"{CKPT_DIR}/smote_{tag}\"\n os.makedirs(ckpt_dir, exist_ok=True)\n best_ckpt = f\"{ckpt_dir}/best.weights.h5\"\n\n sample_weight = None\n if use_class_weight:\n cw = get_class_weights(pd.DataFrame({\"label\": y_tr}))\n sample_weight = np.array([cw[l] for l in y_tr], dtype=np.float32)\n\n with STRATEGY.scope():\n m = build_linear_probe_head(X_tr.shape[-1])\n if os.path.exists(best_ckpt):\n m.load_weights(best_ckpt)\n m.compile(optimizer=keras.optimizers.Adam(1e-3),\n loss=ordinal_bce_loss, metrics=[ordinal_accuracy])\n\n cbs = [\n keras.callbacks.ModelCheckpoint(best_ckpt, save_best_only=True,\n save_weights_only=True,\n monitor=\"val_loss\", mode=\"min\"),\n keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=8,\n restore_best_weights=True),\n keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5,\n patience=3, min_lr=1e-8),\n ]\n fit_kwargs = dict(validation_data=(X_val, Y_va), batch_size=64,\n epochs=40, verbose=0, callbacks=cbs)\n if sample_weight is not None:\n m.fit(X_tr, Y_tr, sample_weight=sample_weight, **fit_kwargs)\n else:\n m.fit(X_tr, Y_tr, **fit_kwargs)\n if os.path.exists(best_ckpt):\n m.load_weights(best_ckpt)\n\n raw = m.predict(X_val, verbose=0)\n opt_thr, opt_qwk = find_optimal_threshold(raw, y_val)\n preds = np.array([ordinal_logits_to_grade(l, thr=opt_thr) for l in raw])\n res = {\n \"accuracy\": round(float(accuracy_score(y_val, preds)), 4),\n \"qwk\": round(float(opt_qwk), 4),\n \"f1_macro\": round(float(f1_score(y_val, preds, average=\"macro\", zero_division=0)), 4),\n \"n_train\": int(len(y_tr)),\n \"train_class_counts\": {int(k): int(v) for k, v in Counter(y_tr).items()},\n }\n del m\n keras.backend.clear_session()\n gc.collect()\n return res\n\n print(\"[2/4] Training LinearProbe_Baseline (real features, class-weighted, no SMOTE)...\")\n results[\"LinearProbe_Baseline\"] = _train_and_eval(X_train, y_train, \"baseline\", use_class_weight=True)\n with open(out_file, \"w\") as f:\n json.dump(results, f, indent=2)\n b = results[\"LinearProbe_Baseline\"]\n print(f\" acc={b['accuracy']:.4f} qwk={b['qwk']:.4f}\")\n\n print(\"[3/4] Applying SMOTE in feature space...\")\n min_class_count = min(counts.values())\n k = max(1, min(k_neighbors, min_class_count - 1))\n if k < k_neighbors:\n print(f\" [WARN] smallest class has {min_class_count} samples -> \"\n f\"reducing k_neighbors to {k}\")\n smote = SMOTE(random_state=SEED, k_neighbors=k)\n X_res, y_res = smote.fit_resample(X_train, y_train)\n print(f\" resampled class counts: {dict(sorted(Counter(y_res).items()))}\")\n\n print(\"[4/4] Training LinearProbe_SMOTE (SMOTE-balanced features)...\")\n results[\"LinearProbe_SMOTE\"] = _train_and_eval(X_res, y_res, \"smote\", use_class_weight=False)\n with open(out_file, \"w\") as f:\n json.dump(results, f, indent=2)\n s = results[\"LinearProbe_SMOTE\"]\n print(f\" acc={s['accuracy']:.4f} qwk={s['qwk']:.4f}\")\n\n print(f\"\\n{'='*60}\\nSMOTE ABLATION SUMMARY \"\n f\"(val set, same frozen features, same head architecture)\\n{'='*60}\")\n for k2 in (\"LinearProbe_Baseline\", \"LinearProbe_SMOTE\"):\n r = results[k2]\n print(f\" {k2:<24} acc={r['accuracy']:.4f} qwk={r['qwk']:.4f} n_train={r['n_train']}\")\n print(\" NOTE: both use the SAME frozen features + SAME linear-probe head —\")\n print(\" this isolates SMOTE's effect. It is NOT directly comparable to the\")\n print(\" full end-to-end fine-tuned EfficientNetB4 baseline (different regime,\")\n print(\" see run_loss_ablation's 'Oversampling' variant for that comparison instead).\")\n auto_zip(tag=\"smote_ablation_done\")\n return results\n\n\nprint(\"[OK] Feature-space SMOTE ablation defined (extract_frozen_features, run_smote_ablation).\")\nprint(\"[NOTE] Requires 'imbalanced-learn' (pip-installed in Cell 1) and a trained `model`.\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell R — Real Model Export: ONNX + TFLite (FP32/INT8) + Quantization-Impact QWK","metadata":{}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL R — Real Model Export: ONNX + TFLite (FP32/INT8) +\n# Quantization-Impact QWK + Cloud-GPU vs. Edge-Proxy Latency\n#\n# Replaces the previous per-backbone \"deploy\": \"ONNX ✅ TFLite-INT8 ✅ ...\"\n# text labels in BACKBONE_REGISTRY (descriptive only, never executed) with\n# actual conversions and measured numbers. Addresses ablations A13\n# (inference latency, cloud-GPU vs. edge-device) and A14 (quantization\n# impact on QWK at INT8).\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef _representative_dataset_gen(df, img_size, n_samples=100):\n \"\"\"Yields preprocessed images for TFLite INT8 calibration.\"\"\"\n paths = df[\"path\"].sample(n=min(n_samples, len(df)), random_state=SEED).tolist()\n\n def gen():\n for p in paths:\n img = preprocess_image(p, img_size, augment=False)\n yield [img[np.newaxis].astype(np.float32)]\n return gen\n\n\ndef export_to_tflite(model, out_path, img_size, quantize_int8=False,\n representative_df=None):\n \"\"\"\n Converts a Keras model to TFLite. If quantize_int8=True, applies full\n INT8 post-training quantization using a representative dataset sampled\n from representative_df (required in that case). Returns file size in MB.\n NOTE: models with unusual Lambda-based preprocessing (a few backbones'\n `preprocess_fn` in BACKBONE_REGISTRY) can occasionally trip up the\n INT8 converter — if that happens for a given backbone, the fix is\n usually to bake normalisation into standard Keras ops rather than an\n arbitrary Python function, or to exclude that backbone from this bench.\n \"\"\"\n converter = tf.lite.TFLiteConverter.from_keras_model(model)\n if quantize_int8:\n assert representative_df is not None, \"INT8 quantization needs a representative_df\"\n converter.optimizations = [tf.lite.Optimize.DEFAULT]\n converter.representative_dataset = _representative_dataset_gen(representative_df, img_size)\n converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]\n converter.inference_input_type = tf.float32\n converter.inference_output_type = tf.float32\n tflite_model = converter.convert()\n with open(out_path, \"wb\") as f:\n f.write(tflite_model)\n return os.path.getsize(out_path) / 1e6\n\n\ndef export_to_onnx(model, out_path, img_size):\n \"\"\"Converts a Keras model to ONNX via tf2onnx. Returns file size in MB.\"\"\"\n import tf2onnx\n spec = (tf.TensorSpec((None, img_size, img_size, 3), tf.float32, name=\"fundus_input\"),)\n _, _ = tf2onnx.convert.from_keras(model, input_signature=spec, output_path=out_path)\n return os.path.getsize(out_path) / 1e6\n\n\ndef evaluate_tflite_model(tflite_path, df, img_size):\n \"\"\"\n Runs a TFLite model over df through the standard TFLite Interpreter (the\n same runtime an edge device would actually execute) and returns\n accuracy/QWK using this notebook's normal optimal-threshold logit\n decode. Also times per-image latency through this interpreter as an\n edge-deployment proxy (a true on-device benchmark would need physical\n edge hardware, which this notebook does not have access to — this is\n the standard software-side stand-in).\n \"\"\"\n interpreter = tf.lite.Interpreter(model_path=tflite_path)\n interpreter.allocate_tensors()\n inp_details = interpreter.get_input_details()[0]\n out_details = interpreter.get_output_details()[0]\n\n logits_list, labels = [], []\n paths = df[\"path\"].tolist()\n for p, lbl in zip(paths, df[\"label\"].tolist()):\n img = preprocess_image(p, img_size, augment=False)[np.newaxis].astype(np.float32)\n interpreter.set_tensor(inp_details[\"index\"], img)\n interpreter.invoke()\n logits_list.append(interpreter.get_tensor(out_details[\"index\"])[0])\n labels.append(lbl)\n raw = np.array(logits_list)\n true = np.array(labels)\n\n opt_thr, opt_qwk = find_optimal_threshold(raw, true)\n preds = np.array([ordinal_logits_to_grade(l, thr=opt_thr) for l in raw])\n acc = float(accuracy_score(true, preds))\n\n n_lat = min(50, len(paths))\n t0 = time.time()\n for p in paths[:n_lat]:\n img = preprocess_image(p, img_size, augment=False)[np.newaxis].astype(np.float32)\n interpreter.set_tensor(inp_details[\"index\"], img)\n interpreter.invoke()\n _ = interpreter.get_tensor(out_details[\"index\"])\n lat_ms = (time.time() - t0) / n_lat * 1000\n\n return {\"accuracy\": round(acc, 4), \"qwk\": round(float(opt_qwk), 4),\n \"latency_ms\": round(lat_ms, 1), \"n_eval\": int(len(paths))}\n\n\ndef run_deployment_export_benchmark(model, val_df, backbone_name=\"EfficientNetB4\",\n img_size=None, n_eval_subset=300):\n \"\"\"\n For the given trained model:\n 1. Exports ONNX (A13 prerequisite)\n 2. Exports TFLite FP32 and TFLite INT8 (A13/A14)\n 3. Evaluates FP32-Keras (cloud-GPU proxy) vs. TFLite-FP32 vs.\n TFLite-INT8 (edge proxy) on the SAME val subset, reporting the\n QWK trade-off at INT8 (A14) and the latency gap (A13)\n Saves to deployment_benchmark.json. Resumable per backbone_name.\n n_eval_subset caps how many val images run through the (slow,\n single-image) TFLite interpreter loop.\n \"\"\"\n out_file = f\"{OUT_DIR}/deployment_benchmark.json\"\n results = json.load(open(out_file)) if os.path.exists(out_file) else {}\n if backbone_name in results:\n print(f\"[SKIP] Deployment benchmark for {backbone_name} already done.\")\n return results\n\n img_size = img_size or CFG[\"IMG_SIZE\"]\n export_dir = f\"{OUT_DIR}/exports/{backbone_name}\"\n os.makedirs(export_dir, exist_ok=True)\n eval_subset = val_df.sample(n=min(n_eval_subset, len(val_df)), random_state=SEED)\n\n print(f\"\\n{'='*60}\\nDEPLOYMENT EXPORT BENCHMARK: {backbone_name}\\n{'='*60}\")\n entry = {\"img_size\": img_size, \"n_eval_subset\": int(len(eval_subset))}\n\n try:\n print(\"[1/5] Exporting ONNX...\")\n onnx_path = f\"{export_dir}/model.onnx\"\n entry[\"onnx_mb\"] = round(export_to_onnx(model, onnx_path, img_size), 2)\n print(f\" onnx: {entry['onnx_mb']} MB\")\n except Exception as e:\n print(f\"[ERROR] ONNX export failed: {e}\")\n entry[\"onnx_mb\"] = None\n\n try:\n print(\"[2/5] Exporting TFLite FP32...\")\n fp32_path = f\"{export_dir}/model_fp32.tflite\"\n entry[\"tflite_fp32_mb\"] = round(\n export_to_tflite(model, fp32_path, img_size, quantize_int8=False), 2)\n print(f\" tflite-fp32: {entry['tflite_fp32_mb']} MB\")\n print(\" Evaluating TFLite FP32 on val subset...\")\n entry[\"tflite_fp32_eval\"] = evaluate_tflite_model(fp32_path, eval_subset, img_size)\n e32 = entry[\"tflite_fp32_eval\"]\n print(f\" acc={e32['accuracy']:.4f} qwk={e32['qwk']:.4f} \"\n f\"lat={e32['latency_ms']:.1f}ms (edge-proxy)\")\n except Exception as e:\n print(f\"[ERROR] TFLite FP32 export/eval failed: {e}\")\n entry[\"tflite_fp32_mb\"] = None\n entry[\"tflite_fp32_eval\"] = None\n\n try:\n print(\"[3/5] Exporting TFLite INT8 (quantized)...\")\n int8_path = f\"{export_dir}/model_int8.tflite\"\n entry[\"tflite_int8_mb\"] = round(\n export_to_tflite(model, int8_path, img_size, quantize_int8=True,\n representative_df=val_df), 2)\n print(f\" tflite-int8: {entry['tflite_int8_mb']} MB\")\n print(\" Evaluating TFLite INT8 on val subset (A14: quantization impact)...\")\n entry[\"tflite_int8_eval\"] = evaluate_tflite_model(int8_path, eval_subset, img_size)\n e8 = entry[\"tflite_int8_eval\"]\n print(f\" acc={e8['accuracy']:.4f} qwk={e8['qwk']:.4f} \"\n f\"lat={e8['latency_ms']:.1f}ms (edge-proxy)\")\n except Exception as e:\n print(f\"[ERROR] TFLite INT8 export/eval failed: {e}\")\n entry[\"tflite_int8_mb\"] = None\n entry[\"tflite_int8_eval\"] = None\n\n try:\n print(\"[4/5] Timing native Keras (cloud-GPU proxy) latency...\")\n sample = preprocess_image(eval_subset.iloc[0][\"path\"], img_size)[np.newaxis]\n _ = model(sample, training=False)\n t0 = time.time()\n for _ in range(50):\n _ = model(sample, training=False)\n entry[\"keras_cloud_gpu_latency_ms\"] = round((time.time() - t0) / 50 * 1000, 2)\n print(f\" keras (cloud-GPU proxy): {entry['keras_cloud_gpu_latency_ms']}ms\")\n except Exception as e:\n print(f\"[ERROR] Keras latency timing failed: {e}\")\n entry[\"keras_cloud_gpu_latency_ms\"] = None\n\n print(\"[5/5] Summary\")\n if entry.get(\"tflite_fp32_eval\") and entry.get(\"tflite_int8_eval\"):\n qwk_drop = entry[\"tflite_fp32_eval\"][\"qwk\"] - entry[\"tflite_int8_eval\"][\"qwk\"]\n entry[\"int8_qwk_drop_vs_fp32\"] = round(float(qwk_drop), 4)\n print(f\" QWK drop FP32->INT8: {entry['int8_qwk_drop_vs_fp32']:+.4f}\")\n\n results[backbone_name] = entry\n with open(out_file, \"w\") as f:\n json.dump(results, f, indent=2, default=str)\n print(f\"[SAVED] {out_file}\")\n auto_zip(tag=f\"deployment_export_{backbone_name}\")\n return results\n\n\nprint(\"[OK] Real ONNX/TFLite export + quantization-impact benchmarking defined.\")\nprint(\"[NOTE] Produces buildable export artifacts and measured numbers — not a persistently\")\nprint(\" deployed serving endpoint (see Extended Cell T for BentoML packaging).\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell S — I/O Throughput Benchmark (OpenCV vs. TFRecords)","metadata":{}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL S — I/O Throughput Benchmark: OpenCV vs. Pre-built TFRecords\n# Addresses ablation A12. Previously the notebook used TFRecords unconditionally\n# when available but never actually measured the speedup against the naive path.\n# ═══════════════════════════════════════════════════════════════════════════\n\ndef run_io_throughput_benchmark(df, img_size=None, n_images=500, batch_size=32):\n \"\"\"\n Times (a) the on-the-fly OpenCV/PIL preprocess_image() path vs. (b)\n reading the same images from pre-built TFRecords, over the same\n n_images, reporting images/sec for each and the speedup multiplier.\n Builds a small n_images-sized TFRecord shard for this benchmark if one\n doesn't already exist. Saves to io_throughput_benchmark.json.\n \"\"\"\n out_file = f\"{OUT_DIR}/io_throughput_benchmark.json\"\n if os.path.exists(out_file):\n r = json.load(open(out_file))\n print(f\"[SKIP] I/O throughput benchmark already done: \"\n f\"opencv={r['opencv_images_per_sec']:.1f} img/s, \"\n f\"tfrecord={r['tfrecord_images_per_sec']:.1f} img/s \"\n f\"({r['speedup_x']:.1f}x)\")\n return r\n\n img_size = img_size or CFG[\"IMG_SIZE\"]\n subset = df.sample(n=min(n_images, len(df)), random_state=SEED).reset_index(drop=True)\n n = len(subset)\n print(f\"\\n{'='*60}\\nI/O THROUGHPUT BENCHMARK: OpenCV vs. TFRecords ({n} images)\\n{'='*60}\")\n\n print(\"[1/2] Timing on-the-fly preprocess_image() (OpenCV path)...\")\n t0 = time.time()\n for p in subset[\"path\"].tolist():\n _ = preprocess_image(p, img_size, augment=False)\n t_opencv = time.time() - t0\n opencv_ips = n / t_opencv if t_opencv > 0 else float(\"inf\")\n print(f\" {n} images in {t_opencv:.1f}s = {opencv_ips:.1f} images/sec\")\n\n print(\"[2/2] Timing pre-built TFRecord reads...\")\n bench_tag = \"io_bench_subset\"\n if not tfrecords_exist(bench_tag):\n print(f\" Building a {n}-image TFRecord shard for this benchmark...\")\n build_tfrecords(subset, bench_tag, img_size, images_per_shard=n)\n ds = make_tf_dataset_fast(bench_tag, batch_size, img_size, augment=False,\n shuffle=False, n_images=n)\n t0 = time.time()\n count = 0\n for batch in ds:\n count += int(batch[0].shape[0])\n t_tfr = time.time() - t0\n tfr_ips = count / t_tfr if t_tfr > 0 else float(\"inf\")\n print(f\" {count} images in {t_tfr:.1f}s = {tfr_ips:.1f} images/sec\")\n\n result = {\n \"n_images\": n,\n \"opencv_seconds\": round(t_opencv, 2),\n \"opencv_images_per_sec\": round(opencv_ips, 1),\n \"tfrecord_seconds\": round(t_tfr, 2),\n \"tfrecord_images_per_sec\": round(tfr_ips, 1),\n \"speedup_x\": round(tfr_ips / opencv_ips, 2) if opencv_ips > 0 else None,\n }\n with open(out_file, \"w\") as f:\n json.dump(result, f, indent=2)\n print(f\"\\n OpenCV: {result['opencv_images_per_sec']:.1f} img/s\")\n print(f\" TFRecords: {result['tfrecord_images_per_sec']:.1f} img/s\")\n print(f\" Speedup: {result['speedup_x']:.2f}x\")\n auto_zip(tag=\"io_throughput_benchmark_done\")\n return result\n\n\nprint(\"[OK] I/O throughput benchmark (OpenCV vs. TFRecords) defined.\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extended Cell T — Real MLflow Tracking + BentoML Service Packaging","metadata":{}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED CELL T — Real MLflow Tracking + BentoML Service Packaging\n# Replaces: `pip(\"mlflow\")` with no further use anywhere in the notebook, and\n# the static \"deploy\": \"... BentoML ✅\" text labels in BACKBONE_REGISTRY,\n# with an actually-functional local MLflow tracking store and a buildable\n# BentoML service definition.\n# ═══════════════════════════════════════════════════════════════════════════\n\nimport mlflow\n\n\ndef init_mlflow(experiment_name=\"MMRDR-Phase1-CFP\"):\n \"\"\"\n Local, file-based MLflow tracking store — works standalone on Kaggle\n with no external tracking server needed. To view the dashboard, copy\n the mlruns/ folder out of the Kaggle output and run:\n mlflow ui --backend-store-uri file:///path/to/mlruns\n \"\"\"\n mlflow.set_tracking_uri(f\"file:{WORK_DIR}/mlruns\")\n mlflow.set_experiment(experiment_name)\n print(f\"[MLFLOW] Tracking URI: file:{WORK_DIR}/mlruns | Experiment: {experiment_name}\")\n\n\ndef log_run_to_mlflow(run_name, params=None, metrics=None, tags=None):\n \"\"\"Thin wrapper: one MLflow run = one ablation variant/backbone/etc.\"\"\"\n with mlflow.start_run(run_name=run_name):\n if params:\n mlflow.log_params({k: v for k, v in params.items() if v is not None})\n if metrics:\n mlflow.log_metrics({k: float(v) for k, v in metrics.items()\n if isinstance(v, (int, float)) and not isinstance(v, bool)})\n if tags:\n mlflow.set_tags(tags)\n\n\ndef log_all_existing_results_to_mlflow():\n \"\"\"\n Back-fills MLflow with every ablation JSON already produced by this\n notebook (past AND future runs alike show up in one place), so the\n \"track hyperparameter velocity via MLflow\" goal covers the whole\n ablation history, not just whatever runs after this cell exists.\n Safe to re-run — MLflow just accumulates additional run records.\n \"\"\"\n init_mlflow()\n manifest = [\n (\"backbone_ablation.json\", \"backbone\"),\n (\"loss_ablation.json\", \"loss\"),\n (\"dataset_ablation.json\", \"dataset\"),\n (\"resolution_ablation.json\", \"resolution\"),\n (\"statistical_results.json\", \"stat_val\"),\n (\"eval_binary_triage.json\", \"binary_triage\"),\n (\"encoding_ablation.json\", \"ordinal_encoding\"),\n (\"combo_ablation.json\", \"combo\"),\n (\"preprocessing_ablation.json\", \"preprocessing\"),\n (\"calibration_results.json\", \"calibration\"),\n (\"eval_uncertainty.json\", \"uncertainty\"),\n (\"fusion_dualres_ablation.json\", \"dualres_fusion\"),\n (\"fusion_multibackbone_ablation.json\", \"multibb_fusion\"),\n (\"attention_ablation.json\", \"attention\"),\n (\"emd_ablation.json\", \"emd\"),\n (\"augmentation_family_ablation.json\", \"aug_family\"),\n (\"ensemble_ablation.json\", \"ensemble\"),\n (\"leakage_ablation.json\", \"leakage\"),\n (\"smote_ablation.json\", \"smote\"),\n (\"deployment_benchmark.json\", \"deployment\"),\n (\"io_throughput_benchmark.json\", \"io_throughput\"),\n ]\n n_logged, n_files = 0, 0\n for fname, category in manifest:\n p = f\"{OUT_DIR}/{fname}\"\n if not os.path.exists(p):\n continue\n n_files += 1\n data = json.load(open(p))\n if isinstance(data, dict) and all(isinstance(v, dict) for v in data.values()) and data:\n entries = list(data.items())\n else:\n entries = [(category, data)]\n for variant_name, metrics in entries:\n if not isinstance(metrics, dict):\n continue\n flat_metrics = {k: v for k, v in metrics.items()\n if isinstance(v, (int, float)) and not isinstance(v, bool)}\n other_params = {k: v for k, v in metrics.items()\n if isinstance(v, (str, bool)) and k != \"notes\"}\n log_run_to_mlflow(run_name=f\"{category}__{variant_name}\",\n params=other_params, metrics=flat_metrics,\n tags={\"category\": category, \"source_file\": fname})\n n_logged += 1\n print(f\"[MLFLOW] Logged {n_logged} runs across {n_files} JSON files.\")\n print(f\"[MLFLOW] View later with: mlflow ui --backend-store-uri file:{WORK_DIR}/mlruns\")\n return {\"n_logged\": n_logged, \"n_files\": n_files}\n\n\nBENTOML_SERVICE_TEMPLATE = '''\"\"\"\nAuto-generated BentoML service for the MMRDR Phase 1 CFP ordinal DR grader.\nRun `bentoml build` in this directory, then `bentoml serve service:svc` to\ntest locally, or push the resulting Bento to a deployment target. This\ndefines a real, buildable service — it is not run persistently inside the\ntraining notebook itself (a batch notebook cell is the wrong place to host\na long-lived server); building/serving is a separate step, same as any\nother model-serving deployment.\n\"\"\"\nimport numpy as np\nimport bentoml\nfrom bentoml.io import Image, JSON\n\nBENTO_MODEL_TAG = \"__MODEL_TAG__\"\nIMG_SIZE = __IMG_SIZE__\nDECODE_THRESHOLD = __DECODE_THRESHOLD__\nGRADE_NAMES = [\"No DR\", \"Mild NPDR\", \"Moderate NPDR\", \"Severe NPDR\", \"Proliferative DR\"]\n\nrunner = bentoml.keras.get(BENTO_MODEL_TAG).to_runner()\nsvc = bentoml.Service(\"mmrdr_cfp_grader\", runners=[runner])\n\n\n@svc.api(input=Image(), output=JSON())\ndef predict(img):\n arr = np.asarray(img.resize((IMG_SIZE, IMG_SIZE))).astype(\"float32\")\n logits = runner.run(arr[np.newaxis])[0]\n grade = int(np.sum(logits > DECODE_THRESHOLD))\n return {\"grade\": grade, \"grade_name\": GRADE_NAMES[grade], \"logits\": logits.tolist()}\n'''\n\n\ndef write_bentoml_service(model, model_tag_name=\"mmrdr_cfp_efficientnetb4\",\n img_size=None, decode_threshold=0.0, out_dir=None):\n \"\"\"\n Saves the trained Keras model into BentoML's local model store, writes\n a real service.py + bentofile.yaml next to it, and (if the bentoml CLI\n is available) runs `bentoml build` to produce an actual Bento artifact.\n NOTE: BentoML's exact save/serve API has shifted across 1.0/1.1/1.2 —\n if `bentoml.keras.save_model` doesn't match your installed version,\n check `bentoml.keras.__all__` or the installed docs for the current\n call signature; the surrounding structure here (save -> write service.py\n -> bentoml build) stays the same across versions.\n \"\"\"\n import bentoml\n import subprocess\n img_size = img_size or CFG[\"IMG_SIZE\"]\n out_dir = out_dir or f\"{OUT_DIR}/bentoml_service\"\n os.makedirs(out_dir, exist_ok=True)\n\n print(f\"\\n{'='*60}\\nBENTOML PACKAGING\\n{'='*60}\")\n print(\"[1/3] Saving model into BentoML's local model store...\")\n saved = bentoml.keras.save_model(model_tag_name, model)\n print(f\" saved as: {saved.tag}\")\n\n print(\"[2/3] Writing service.py + bentofile.yaml...\")\n service_code = (BENTOML_SERVICE_TEMPLATE\n .replace(\"__MODEL_TAG__\", str(saved.tag))\n .replace(\"__IMG_SIZE__\", str(img_size))\n .replace(\"__DECODE_THRESHOLD__\", str(decode_threshold)))\n with open(f\"{out_dir}/service.py\", \"w\") as f:\n f.write(service_code)\n\n bentofile = (\n 'service: \"service:svc\"\\n'\n \"labels:\\n\"\n \" project: mmrdr-phase1-cfp\\n\"\n \"include:\\n\"\n ' - \"service.py\"\\n'\n \"python:\\n\"\n \" packages:\\n\"\n \" - numpy\\n\"\n \" - pillow\\n\"\n )\n with open(f\"{out_dir}/bentofile.yaml\", \"w\") as f:\n f.write(bentofile)\n print(f\" wrote {out_dir}/service.py and {out_dir}/bentofile.yaml\")\n\n print(\"[3/3] Attempting `bentoml build`...\")\n result = {\"model_tag\": str(saved.tag), \"service_dir\": out_dir, \"build_tag\": None}\n try:\n proc = subprocess.run([\"bentoml\", \"build\", out_dir], capture_output=True,\n text=True, timeout=120)\n if proc.returncode == 0:\n print(proc.stdout[-500:])\n lines = proc.stdout.strip().splitlines()\n result[\"build_tag\"] = lines[-1] if lines else None\n print(\" bentoml build succeeded.\")\n else:\n print(f\" [WARN] bentoml build exited {proc.returncode}: {proc.stderr[-500:]}\")\n except Exception as e:\n print(f\" [WARN] Could not run `bentoml build` from Python ({e}). \"\n f\"Run it manually: cd {out_dir} && bentoml build\")\n\n with open(f\"{OUT_DIR}/bentoml_packaging.json\", \"w\") as f:\n json.dump(result, f, indent=2)\n auto_zip(tag=\"bentoml_packaging_done\")\n return result\n\n\nprint(\"[OK] Real MLflow tracking (local file store) + BentoML service packaging defined.\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Intentionally NOT added — and why\n\nEverything below was in the \"missing\" list but is deliberately excluded rather\nthan bolted on in a way that wouldn't actually hold up:\n\n- **RETFound / DINOv2 / EVA-02 — UPDATE: now partially added.** As of Extended Cell N, these ARE benchmarked, but via a linear-probe bridge (separate PyTorch extraction script -> precomputed embeddings -> Keras linear-probe head), not a native Keras backbone, since none of them have real TensorFlow ports (confirmed: no `TFDinov2Model`; RETFound/EVA-02 are PyTorch/timm-only). Full end-to-end fine-tuning of these (rather than frozen linear probing) would still require a separate PyTorch training loop, not a Keras cell.\n- **VisionFM, KeepFIT/MM-Retinal, EyeFM — still not added, deliberately.** VisionFM requires its own multi-modality data pipeline and custom repo structure; KeepFIT needs the MM-Retinal repo's specific setup; EyeFM is a LLaVA-style report-generation VLM, not a simple embedding extractor — none of these are a clean drop-in the way FLAIR (pip-installable) or RETFound (git clone + documented weights) are. Flagged as phase-2 stretch goals rather than silently promised.\n- **Multi-task learning (grading + lesion/optic-disc/macula segmentation).**\n This needs pixel-level lesion/OD/macula masks. The notebook's own data\n loaders (`load_eyepacs`, `load_aptos`, `load_messidor`, IDRiD loader) only\n ever extract the grade label — none of them parse segmentation masks, and\n wiring that up means touching dataset loading, which you asked to leave\n alone. Bolting on a second task head with no real second-task labels would\n just be a no-op head trained on nothing, which isn't a genuine addition.\n- **Domain adaptation / test-time adaptation across cameras or hospitals.**\n The current splits are patient-level, not camera- or site-level — the\n notebook doesn't track a \"domain\" field per image at all. Any domain-\n adaptation loss (e.g. adversarial domain classifier, batch-norm\n recalibration at test time) needs that domain label to train against;\n without it, it would be decorative code with nothing real driving it.\n- **Vision-language models.** These need paired image/text (report) data.\n None of EyePACS/APTOS/Messidor/IDRiD, as loaded here, provide free-text\n reports — only a numeric grade. There's nothing to pair a VLM's text\n encoder against, so it's skipped rather than faked with placeholder text.\n\nIf any of these become worth doing for real, the honest next step is sourcing\nthe matching annotations/data first (lesion masks, domain/camera metadata, or\npaired reports) — at which point they'd be additive cells just like the ones\nabove, not a redesign.\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"markdown","source":"### Extended Main Execution — runs all of the above, in order, fully resumable","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]}},{"cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════════\n# EXTENDED MAIN EXECUTION — Run this AFTER the original Cell 21 main execution\n# has completed at least once (needs `model`, `train_df`, `val_df`, `test_df`,\n# and ideally a completed run_backbone_ablation for the fusion step).\n# Every step below is independently resumable — safe to re-run the whole cell.\n# ═══════════════════════════════════════════════════════════════════════════\n\nprint(\"=\"*60)\nprint(\"EXTENDED STEP 1 — Ordinal Encoding Ablation (Softmax / CORAL / CORN)\")\nprint(\"=\"*60)\ntry:\n encoding_abl = run_ordinal_encoding_ablation(train_df, val_df)\nexcept Exception as e:\n print(f\"[ERROR] Ordinal encoding ablation failed: {e}\")\nauto_zip(tag=\"after_encoding_ablation\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 2 — Combinatorial Loss/Imbalance Ablation\")\nprint(\"=\"*60)\ntry:\n combo_abl = run_combo_ablation(train_df, val_df)\nexcept Exception as e:\n print(f\"[ERROR] Combo ablation failed: {e}\")\nauto_zip(tag=\"after_combo_ablation\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 3 — Preprocessing Ablation (None / CLAHE / Ben Graham / Both)\")\nprint(\"=\"*60)\ntry:\n preproc_abl = run_preprocessing_ablation(train_df, val_df)\nexcept Exception as e:\n print(f\"[ERROR] Preprocessing ablation failed: {e}\")\nauto_zip(tag=\"after_preprocessing_ablation\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 4 — Calibration Analysis\")\nprint(\"=\"*60)\ntry:\n _calib_test_df = idrid_test_df if (\"idrid_test_df\" in dir() and idrid_test_df is not None) else test_df\n calib_results = run_calibration_analysis(model, val_df, test_df=_calib_test_df)\nexcept Exception as e:\n print(f\"[ERROR] Calibration analysis failed: {e}\")\nauto_zip(tag=\"after_calibration\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 5 — Uncertainty Quantification\")\nprint(\"=\"*60)\ntry:\n uncertainty_results = run_uncertainty_analysis(model, val_df)\nexcept Exception as e:\n print(f\"[ERROR] Uncertainty analysis failed: {e}\")\nauto_zip(tag=\"after_uncertainty\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 6 — Error Analysis (Adjacent vs Severe Grade Errors)\")\nprint(\"=\"*60)\ntry:\n res_val_final = evaluate(model, val_df, desc=\"Val — error analysis\")\n error_abl = adjacent_vs_severe_errors(res_val_final[\"labels\"], res_val_final[\"preds\"])\nexcept Exception as e:\n print(f\"[ERROR] Error analysis failed: {e}\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 7 — Dual-Resolution Fusion Ablation\")\nprint(\"=\"*60)\ntry:\n dualres_abl = run_dual_resolution_ablation(train_df, val_df)\nexcept Exception as e:\n print(f\"[ERROR] Dual-resolution fusion failed: {e}\")\nauto_zip(tag=\"after_dualres_fusion\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 8 — Multi-Backbone Late-Fusion Ablation\")\nprint(\"=\"*60)\nprint(\" Note: requires run_backbone_ablation() to have completed for both\")\nprint(\" backbones first (checkpoints under CKPT_DIR/bb_/best.weights.h5).\")\ntry:\n fusion_bb_abl = run_multi_backbone_fusion_ablation(\n train_df, val_df, backbone_a=\"EfficientNetB4\", backbone_b=\"MaxViTTiny\")\nexcept Exception as e:\n print(f\"[ERROR] Multi-backbone fusion failed: {e}\")\nauto_zip(tag=\"after_multibb_fusion\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 9 — Swin-Tiny Transformer Baseline\")\nprint(\"=\"*60)\ntry:\n swin_abl = run_swin_comparison(train_df, val_df)\nexcept Exception as e:\n print(f\"[ERROR] Swin comparison failed: {e}\")\nauto_zip(tag=\"after_swin_comparison\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 10 — Attention Modules Ablation (CBAM / Coordinate Attention)\")\nprint(\"=\"*60)\ntry:\n attention_abl = run_attention_ablation(train_df, val_df)\nexcept Exception as e:\n print(f\"[ERROR] Attention ablation failed: {e}\")\nauto_zip(tag=\"after_attention_ablation\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 11 — EMD Ordinal Loss Ablation\")\nprint(\"=\"*60)\ntry:\n emd_abl = run_emd_ablation(train_df, val_df)\nexcept Exception as e:\n print(f\"[ERROR] EMD ablation failed: {e}\")\nauto_zip(tag=\"after_emd_ablation\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 12 — Augmentation Family Ablation (RandAugment / CutMix)\")\nprint(\"=\"*60)\ntry:\n aug_family_abl = run_augmentation_family_ablation(train_df, val_df)\nexcept Exception as e:\n print(f\"[ERROR] Augmentation family ablation failed: {e}\")\nauto_zip(tag=\"after_augmentation_family_ablation\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 13 — Foundation Model Linear Probe (RETFound/FLAIR/DINOv2/EVA-02)\")\nprint(\"=\"*60)\ntry:\n fm_probe_abl = run_foundation_model_linear_probe()\nexcept Exception as e:\n print(f\"[ERROR] Foundation model linear probe failed: {e}\")\nauto_zip(tag=\"after_foundation_model_probe\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 14 — Ensemble (Top-K Trained Backbones + Foundation Probes)\")\nprint(\"=\"*60)\nprint(\" Note: purely reuses existing checkpoints/embeddings — requires at least\")\nprint(\" 2 completed, checkpointed models (any mix of backbone ablation, Swin,\")\nprint(\" attention variants, or foundation-model probes) to do anything.\")\ntry:\n _ens_test_df = idrid_test_df if (\"idrid_test_df\" in dir() and idrid_test_df is not None) else test_df\n ensemble_abl = run_ensemble(train_df, val_df, test_df=_ens_test_df, top_k=5)\nexcept Exception as e:\n print(f\"[ERROR] Ensemble failed: {e}\")\nauto_zip(tag=\"after_ensemble\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 14b — Full Cross-Category Leaderboard\")\nprint(\"=\"*60)\nprint(\" Consolidates everything now available: Category A (fine-tuned CNN/hybrid\")\nprint(\" backbones), Category B (fine-tuned Transformers: ViT-Small, Swin-Tiny),\")\nprint(\" Category C (frozen foundation-model linear probes), and Category D\")\nprint(\" (ensemble). Categories are reported separately because they are NOT an\")\nprint(\" apples-to-apples comparison — Category C is deliberately a weaker regime\")\nprint(\" (frozen backbone, linear head only) than A/B's full fine-tuning, and\")\nprint(\" Category D combines multiple already-trained members.\")\n\n\ndef build_cross_category_leaderboard(metric=\"qwk\"):\n bb_file = f\"{OUT_DIR}/backbone_ablation.json\"\n ens_file = f\"{OUT_DIR}/ensemble_ablation.json\"\n all_results = json.load(open(bb_file)) if os.path.exists(bb_file) else {}\n ens_results = json.load(open(ens_file)) if os.path.exists(ens_file) else {}\n\n registry_keys = set(BACKBONE_REGISTRY.keys())\n\n def _category(name):\n if name.endswith(\"-LinearProbe\"):\n return \"C: frozen foundation-model linear-probe\"\n if name in registry_keys:\n return \"A: fine-tuned CNN/hybrid backbone\"\n if name in (\"ViT-Small\", \"Swin-Tiny\"):\n return \"B: fine-tuned Transformer\"\n return \"? (unrecognised key — check naming)\"\n\n rows = []\n for name, r in all_results.items():\n if isinstance(r, dict) and metric in r:\n rows.append((name, _category(name), r[metric], name in registry_keys))\n for name, r in ens_results.items():\n if isinstance(r, dict) and metric in r:\n rows.append((f\"Ensemble:{name}\", \"D: ensemble of trained members\", r[metric], False))\n\n if not rows:\n print(\" [INFO] Nothing to rank yet — run the search-phase and Extended ablation \"\n \"cells first.\")\n return None\n\n rows.sort(key=lambda x: -x[2])\n print(f\"\\n {'Rank':<5}{'Model':<28}{'Category':<42}{metric.upper()}\")\n for i, (name, cat, val, eligible) in enumerate(rows, 1):\n print(f\" {i:<5}{name:<28}{cat:<42}{val:.4f}\")\n\n overall_winner = rows[0]\n registry_winner = next((r for r in rows if r[3]), None)\n print(f\"\\n Overall highest {metric.upper()}: {overall_winner[0]} ({overall_winner[2]:.4f}, {overall_winner[1]})\")\n if registry_winner and registry_winner[0] != overall_winner[0]:\n print(f\" Best BACKBONE_REGISTRY-eligible (i.e. what STEP 4 of the main pipeline\")\n print(f\" actually trained expensively): {registry_winner[0]} ({registry_winner[2]:.4f})\")\n print(f\" Gap to overall winner: {overall_winner[2] - registry_winner[2]:+.4f} {metric.upper()}\")\n if overall_winner[1].startswith(\"C:\"):\n print(f\" -> {overall_winner[0]} is a FROZEN linear probe. If this margin holds,\")\n print(f\" it's a signal that FULLY fine-tuning that foundation model (in its\")\n print(f\" native PyTorch training code — not this notebook) could do even better,\")\n print(f\" not that the linear probe itself should be deployed as-is.\")\n elif overall_winner[1].startswith(\"B:\") or overall_winner[1].startswith(\"D:\"):\n print(f\" -> Worth a follow-up: extend this pipeline's train()/evaluate() plumbing\")\n print(f\" to support {overall_winner[1].split(': ')[1]} natively if you want it\")\n print(f\" to compete for the 'winner' slot automatically next run.\")\n\n leaderboard = {\n \"metric\": metric,\n \"ranking\": [{\"name\": n, \"category\": c, metric: v, \"registry_eligible\": e}\n for n, c, v, e in rows],\n \"overall_winner\": overall_winner[0],\n \"registry_eligible_winner\": registry_winner[0] if registry_winner else None,\n }\n with open(f\"{OUT_DIR}/cross_category_leaderboard.json\", \"w\") as f:\n json.dump(leaderboard, f, indent=2)\n return leaderboard\n\n\ntry:\n leaderboard = build_cross_category_leaderboard(metric=\"qwk\")\nexcept Exception as e:\n print(f\"[ERROR] Cross-category leaderboard failed: {e}\")\nauto_zip(tag=\"after_cross_category_leaderboard\")\n\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 15 — Leakage Ablation (Patient-Level vs. Naive Image-Level Split)\")\nprint(\"=\"*60)\ntry:\n leakage_abl = run_leakage_ablation(df_all)\nexcept Exception as e:\n print(f\"[ERROR] Leakage ablation failed: {e}\")\nauto_zip(tag=\"after_leakage_ablation\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 16 — Feature-Space SMOTE Ablation\")\nprint(\"=\"*60)\ntry:\n smote_abl = run_smote_ablation(model, train_df, val_df)\nexcept Exception as e:\n print(f\"[ERROR] SMOTE ablation failed: {e}\")\nauto_zip(tag=\"after_smote_ablation\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 17 — Real Deployment Export (ONNX + TFLite FP32/INT8)\")\nprint(\"=\"*60)\ntry:\n deploy_abl = run_deployment_export_benchmark(model, val_df, backbone_name=CFG[\"ACTIVE_BB\"])\nexcept Exception as e:\n print(f\"[ERROR] Deployment export benchmark failed: {e}\")\nauto_zip(tag=\"after_deployment_export\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 18 — I/O Throughput Benchmark (OpenCV vs. TFRecords)\")\nprint(\"=\"*60)\ntry:\n io_bench = run_io_throughput_benchmark(val_df)\nexcept Exception as e:\n print(f\"[ERROR] I/O throughput benchmark failed: {e}\")\nauto_zip(tag=\"after_io_throughput\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED STEP 19 — MLflow Logging (backfill all results) + BentoML Packaging\")\nprint(\"=\"*60)\ntry:\n mlflow_summary = log_all_existing_results_to_mlflow()\nexcept Exception as e:\n print(f\"[ERROR] MLflow logging failed: {e}\")\ntry:\n bento_result = write_bentoml_service(model, img_size=CFG[\"IMG_SIZE\"])\nexcept Exception as e:\n print(f\"[ERROR] BentoML packaging failed: {e}\")\nauto_zip(tag=\"after_mlflow_bentoml\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXTENDED — Final summary of new ablations\")\nprint(\"=\"*60)\nextended_summary = {}\nfor fname, key in [\n (\"encoding_ablation.json\", \"ordinal_encoding\"),\n (\"combo_ablation.json\", \"combo\"),\n (\"preprocessing_ablation.json\", \"preprocessing\"),\n (\"calibration_results.json\", \"calibration\"),\n (\"eval_uncertainty.json\", \"uncertainty\"),\n (\"error_analysis.json\", \"error_analysis\"),\n (\"fusion_dualres_ablation.json\", \"dual_resolution_fusion\"),\n (\"fusion_multibackbone_ablation.json\", \"multi_backbone_fusion\"),\n (\"backbone_ablation.json\", \"backbone_incl_swin_vit\"),\n (\"attention_ablation.json\", \"attention_modules\"),\n (\"augmentation_family_ablation.json\", \"augmentation_family\"),\n (\"backbone_ablation.json\", \"backbone_incl_foundation_models\"),\n (\"ensemble_ablation.json\", \"ensemble\"),\n (\"leakage_ablation.json\", \"leakage_split_comparison\"),\n (\"smote_ablation.json\", \"smote_feature_space\"),\n (\"deployment_benchmark.json\", \"deployment_export\"),\n (\"io_throughput_benchmark.json\", \"io_throughput\"),\n (\"bentoml_packaging.json\", \"bentoml_packaging\"),\n]:\n p = f\"{OUT_DIR}/{fname}\"\n if os.path.exists(p):\n extended_summary[key] = json.load(open(p))\nwith open(f\"{OUT_DIR}/extended_ablations_summary.json\", \"w\") as f:\n json.dump(extended_summary, f, indent=2)\nprint(f\"[SAVED] {OUT_DIR}/extended_ablations_summary.json\")\nauto_zip(tag=\"EXTENDED_ABLATIONS_COMPLETE\")\nprint(\"\\n[OK] Extended ablations complete.\")\n","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"execution_count":null}]}