{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.0"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":14774,"databundleVersionId":875431},{"sourceType":"datasetVersion","sourceId":988278,"datasetId":541202,"databundleVersionId":1016790},{"sourceType":"datasetVersion","sourceId":187731,"datasetId":80814,"databundleVersionId":198687}],"dockerImageVersionId":31287,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🔬 Diabetic Retinopathy Detection — Improved DenseNet-169\n### Base paper: *Mushtaq & Siddiqui (2021)* + Kaggle best practices\n### Dataset: APTOS 2019\n### ✅ Fixed: removed steps_per_epoch bug causing half-epoch training\n\n| Component | Before | Now |\n|---|---|---|\n| Labels | One-hot categorical | Multilabel ordinal |\n| Loss | Categorical crossentropy | Binary crossentropy |\n| Image size | 256×256 | 224×224 (DenseNet native) |\n| Augmentation | Static saved to disk | Live ImageDataGenerator |\n| Metric | Accuracy only | Cohen Kappa per epoch |\n| Threshold | Fixed argmax | Optimized |\n---","metadata":{}},{"cell_type":"markdown","source":"## Step 1: Import Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport cv2\nimport scipy\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import (\n    cohen_kappa_score, accuracy_score,\n    confusion_matrix, classification_report\n)\n\nimport tensorflow as tf\nfrom tensorflow.keras.applications import DenseNet169\nfrom tensorflow.keras.models import Sequential, load_model\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dropout, Dense\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import Callback, ReduceLROnPlateau\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nSEED = 2019\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\n\nprint('TensorFlow version:', tf.__version__)\nprint('GPU available     :', tf.config.list_physical_devices('GPU'))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 2: Configuration","metadata":{}},{"cell_type":"code","source":"# ── Paths ─────────────────────────────────────────────────────────────────────\nTRAIN_CSV  = \"/kaggle/input/competitions/aptos2019-blindness-detection/train.csv\"\nTRAIN_DIR  = \"/kaggle/input/competitions/aptos2019-blindness-detection/train_images\"\nMODEL_PATH = \"/kaggle/working/densenet169_improved.keras\"\n\n# ── Image settings ────────────────────────────────────────────────────────────\nIMG_SIZE    = 224\nNUM_CLASSES = 5\n\n# ── Training settings ─────────────────────────────────────────────────────────\nBATCH_SIZE    = 32\nPHASE1_EPOCHS = 15\nPHASE2_EPOCHS = 100\nPHASE1_LR     = 5e-5\nPHASE2_LR     = 3e-5\nDROPOUT_RATE  = 0.5\nVAL_SPLIT     = 0.15\n\nCLASS_NAMES = [\"No DR\", \"Mild\", \"Moderate\", \"Severe\", \"Proliferative DR\"]\n\nprint('Configuration loaded ✅')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 3: Load & Explore Data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(TRAIN_CSV)\nprint(f'Total images: {len(df)}')\nprint('\\nClass distribution:')\nprint(df['diagnosis'].value_counts().sort_index())\n\nplt.figure(figsize=(8, 4))\nsns.countplot(x='diagnosis', data=df, hue='diagnosis', palette='Blues_d', legend=False)\nplt.xticks(range(5), CLASS_NAMES, rotation=15)\nplt.title('APTOS 2019 — Class Distribution')\nplt.xlabel('Severity'); plt.ylabel('Count')\nplt.tight_layout()\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 4: Preprocessing (exactly as in paper)","metadata":{}},{"cell_type":"code","source":"def crop_black_border(img):\n    \"\"\"Remove black background around fundus image.\"\"\"\n    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    _, thresh = cv2.threshold(gray, 10, 255, cv2.THRESH_BINARY)\n    contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n    if contours:\n        cnt = max(contours, key=cv2.contourArea)\n        x, y, w, h = cv2.boundingRect(cnt)\n        img = img[y:y+h, x:x+w]\n    return img\n\n\ndef apply_gaussian_blur(img, img_size):\n    \"\"\"Gaussian blur as in paper — kernel = img_size/6.\"\"\"\n    kernel_size = int(img_size / 6)\n    if kernel_size % 2 == 0:\n        kernel_size += 1\n    blurred = cv2.GaussianBlur(img, (kernel_size, kernel_size), 0)\n    return cv2.addWeighted(img, 4, blurred, -4, 128)\n\n\ndef preprocess_image(filepath, img_size=IMG_SIZE):\n    \"\"\"Full paper preprocessing pipeline.\"\"\"\n    img = cv2.imread(filepath)\n    if img is None:\n        return np.zeros((img_size, img_size, 3), dtype=np.uint8)\n    img = crop_black_border(img)\n    img = cv2.resize(img, (img_size, img_size))\n    img = apply_gaussian_blur(img, img_size)\n    return img\n\n\n# Visual check\nsample_fp = os.path.join(TRAIN_DIR, df.iloc[0]['id_code'] + '.png')\nraw  = cv2.cvtColor(cv2.imread(sample_fp), cv2.COLOR_BGR2RGB)\nproc = cv2.cvtColor(preprocess_image(sample_fp), cv2.COLOR_BGR2RGB)\n\nfig, axes = plt.subplots(1, 2, figsize=(10, 4))\naxes[0].imshow(raw);  axes[0].set_title('Original');     axes[0].axis('off')\naxes[1].imshow(proc); axes[1].set_title('Preprocessed'); axes[1].axis('off')\nplt.suptitle('Paper preprocessing: crop → resize 224×224 → Gaussian blur')\nplt.tight_layout()\nplt.show()\nprint('Preprocessing defined ✅')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 5: Load All Images into Memory","metadata":{}},{"cell_type":"code","source":"print('Loading and preprocessing all images...')\nN = len(df)\nX = np.empty((N, IMG_SIZE, IMG_SIZE, 3), dtype=np.uint8)\n\nfor i, image_id in enumerate(tqdm(df['id_code'])):\n    fp = os.path.join(TRAIN_DIR, image_id + '.png')\n    X[i] = preprocess_image(fp)\n\nprint(f'\\nLoaded {N} images — shape: {X.shape}')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 6: Multilabel Ordinal Encoding\nCaptures the ordered nature of DR severity:\n```\nClass 0 (No DR)         → [1, 0, 0, 0, 0]\nClass 1 (Mild)          → [1, 1, 0, 0, 0]\nClass 2 (Moderate)      → [1, 1, 1, 0, 0]\nClass 3 (Severe)        → [1, 1, 1, 1, 0]\nClass 4 (Proliferative) → [1, 1, 1, 1, 1]\n```","metadata":{}},{"cell_type":"code","source":"y_onehot = pd.get_dummies(df['diagnosis']).values\n\ny_multi = np.empty(y_onehot.shape, dtype=y_onehot.dtype)\ny_multi[:, 4] = y_onehot[:, 4]\nfor i in range(3, -1, -1):\n    y_multi[:, i] = np.logical_or(y_onehot[:, i], y_multi[:, i+1])\n\nprint('One-hot encoding  :', y_onehot.sum(axis=0))\nprint('Multilabel ordinal:', y_multi.sum(axis=0))\n\nprint('\\nExample encodings:')\nfor cls in range(5):\n    idx = np.where(df['diagnosis'].values == cls)[0][0]\n    print(f'  Class {cls} ({CLASS_NAMES[cls]:<18}) → {y_multi[idx].astype(int)}')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 7: Train / Validation Split","metadata":{}},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(\n    X, y_multi,\n    test_size=VAL_SPLIT,\n    random_state=SEED,\n    stratify=df['diagnosis'].values\n)\n\nprint(f'Train: {X_train.shape[0]} images')\nprint(f'Val  : {X_val.shape[0]} images')\n\n# Free original array to save memory\ndel X","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 8: Live Data Augmentation Generator","metadata":{}},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(\n    rescale=1.0/255.0,\n    horizontal_flip=True,\n    vertical_flip=True,\n    zoom_range=0.15,\n    rotation_range=360,\n    fill_mode='constant',\n    cval=0.\n)\n\n# ── KEY FIX: use repeat=True so generator never runs out of data ──────────────\ntrain_generator = train_datagen.flow(\n    X_train, y_train,\n    batch_size=BATCH_SIZE,\n    seed=SEED\n)\n\n# Validation data — scaled but not augmented\nX_val_scaled = X_val.astype(np.float32) / 255.0\n\nprint(f'Train: {X_train.shape[0]} images | {len(train_generator)} batches/epoch')\nprint(f'Val  : {X_val.shape[0]} images')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 9: Cohen Kappa Callback","metadata":{}},{"cell_type":"code","source":"class KappaCallback(Callback):\n    \"\"\"Computes Cohen Kappa after every epoch and saves best model.\"\"\"\n\n    def on_train_begin(self, logs={}):\n        self.val_kappas = []\n        self.best_kappa = -1\n\n    def on_epoch_end(self, epoch, logs={}):\n        y_pred = self.model.predict(X_val_scaled, verbose=0)\n\n        # Convert multilabel → class label\n        y_pred_class = (y_pred > 0.5).astype(int).sum(axis=1) - 1\n        y_pred_class = np.clip(y_pred_class, 0, 4)\n        y_true_class = y_val.sum(axis=1) - 1\n\n        kappa = cohen_kappa_score(\n            y_true_class, y_pred_class, weights='quadratic'\n        )\n        self.val_kappas.append(kappa)\n        print(f'  val_kappa: {kappa:.4f}')\n\n        if kappa > self.best_kappa:\n            self.best_kappa = kappa\n            self.model.save(MODEL_PATH)\n            print(f'  ✅ Kappa improved → model saved')\n\n\nprint('KappaCallback defined ✅')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 10: Build DenseNet-169 Model","metadata":{}},{"cell_type":"code","source":"def build_model():\n    base_model = DenseNet169(\n        weights='/kaggle/input/datasets/xhlulu/densenet-keras/DenseNet-BC-169-32-no-top.h5',\n        include_top=False,\n        input_shape=(IMG_SIZE, IMG_SIZE, 3)\n    )\n    base_model.trainable = False\n\n    # Paper: GAP2D → Dropout(0.5) → Dense(5)\n    # sigmoid for multilabel, binary_crossentropy as loss\n    model = Sequential([\n        base_model,\n        GlobalAveragePooling2D(),\n        Dropout(DROPOUT_RATE),\n        Dense(NUM_CLASSES, activation='sigmoid')\n    ], name='DenseNet169_Improved')\n\n    model.compile(\n        optimizer=Adam(learning_rate=PHASE1_LR),\n        loss='binary_crossentropy',\n        metrics=['accuracy']\n    )\n    return model\n\n\nmodel = build_model()\nmodel.summary()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 11: Train Phase 1 — Frozen Base\n**Fix applied:** `steps_per_epoch` removed — Keras calculates it automatically from the generator size. This prevents the generator from running out of data mid-epoch.","metadata":{}},{"cell_type":"code","source":"kappa_cb = KappaCallback()\n\ncallbacks = [\n    kappa_cb,\n    ReduceLROnPlateau(\n        monitor='val_loss',\n        factor=0.5,\n        patience=10,\n        min_lr=1e-6,\n        verbose=1\n    )\n]\n\nprint(f'=== Phase 1: Frozen base | LR={PHASE1_LR} | Epochs={PHASE1_EPOCHS} ===')\nhistory1 = model.fit(\n    train_generator,\n    # steps_per_epoch removed — this was causing the bug\n    epochs=PHASE1_EPOCHS,\n    validation_data=(X_val_scaled, y_val),\n    callbacks=callbacks,\n    verbose=1\n)\nprint(f'\\nBest Phase 1 Kappa: {max(kappa_cb.val_kappas):.4f}')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 12: Train Phase 2 — Fine-Tuning (last 100 layers)","metadata":{}},{"cell_type":"code","source":"print(f'=== Phase 2: Fine-tuning | LR={PHASE2_LR} | Epochs={PHASE2_EPOCHS} ===')\n\n# Unfreeze last 100 layers\nfor layer in model.layers[0].layers[:-100]:\n    layer.trainable = False\nfor layer in model.layers[0].layers[-100:]:\n    layer.trainable = True\n\nmodel.compile(\n    optimizer=Adam(learning_rate=PHASE2_LR),\n    loss='binary_crossentropy',\n    metrics=['accuracy']\n)\n\nhistory2 = model.fit(\n    train_generator,\n    # steps_per_epoch removed — this was causing the bug\n    epochs=PHASE2_EPOCHS,\n    validation_data=(X_val_scaled, y_val),\n    callbacks=callbacks,\n    verbose=1\n)\nprint(f'\\nBest Phase 2 Kappa: {max(kappa_cb.val_kappas):.4f}')\nprint('\\nTraining complete ✅')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 13: Training History Plots","metadata":{}},{"cell_type":"code","source":"# Kappa over all epochs\nplt.figure(figsize=(10, 4))\nplt.plot(kappa_cb.val_kappas, 'g-o', label='Val Kappa')\nplt.axhline(\n    y=max(kappa_cb.val_kappas), color='r', linestyle='--',\n    label=f'Best: {max(kappa_cb.val_kappas):.4f}'\n)\nplt.title('Cohen Kappa Score per Epoch')\nplt.xlabel('Epoch'); plt.ylabel('Kappa')\nplt.legend(); plt.grid(True)\nplt.tight_layout()\nplt.show()\n\n\ndef plot_history(history, title_suffix=''):\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))\n    epochs = range(1, len(history.history['accuracy']) + 1)\n    ax1.plot(epochs, history.history['accuracy'],     'b-o', label='Train')\n    ax1.plot(epochs, history.history['val_accuracy'], 'r-o', label='Val')\n    ax1.set_title(f'Accuracy {title_suffix}')\n    ax1.legend(); ax1.grid(True)\n    ax2.plot(epochs, history.history['loss'],     'b-o', label='Train')\n    ax2.plot(epochs, history.history['val_loss'], 'r-o', label='Val')\n    ax2.set_title(f'Loss {title_suffix}')\n    ax2.legend(); ax2.grid(True)\n    plt.tight_layout(); plt.show()\n\n\nplot_history(history1, '(Phase 1 — Frozen Base)')\nplot_history(history2, '(Phase 2 — Fine-tuned)')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 14: Optimise Prediction Threshold","metadata":{}},{"cell_type":"code","source":"# Load best saved model\nmodel = load_model(MODEL_PATH)\n\n# Get val predictions\ny_val_pred   = model.predict(X_val_scaled, verbose=1)\ny_true_class = y_val.sum(axis=1) - 1\n\n\ndef compute_kappa_for_threshold(threshold):\n    y_pred_class = (y_val_pred > threshold).astype(int).sum(axis=1) - 1\n    y_pred_class = np.clip(y_pred_class, 0, 4)\n    return 1 - cohen_kappa_score(y_true_class, y_pred_class, weights='quadratic')\n\n\nresult         = scipy.optimize.minimize(compute_kappa_for_threshold, x0=0.5, method='nelder-mead')\nbest_threshold = result['x'][0]\nprint(f'Optimal threshold: {best_threshold:.4f}')\n\n# Apply optimal threshold\ny_pred_class = (y_val_pred > best_threshold).astype(int).sum(axis=1) - 1\ny_pred_class = np.clip(y_pred_class, 0, 4)\n\nval_acc = accuracy_score(y_true_class, y_pred_class)\nkappa   = cohen_kappa_score(y_true_class, y_pred_class, weights='quadratic')\n\nprint('\\n' + '='*50)\nprint(f'  Validation Accuracy : {val_acc*100:.2f}%')\nprint(f'  Cohen Kappa Score   : {kappa:.4f}')\nprint('='*50)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 15: Evaluation","metadata":{}},{"cell_type":"code","source":"print('Classification Report:')\nprint(classification_report(y_true_class, y_pred_class, target_names=CLASS_NAMES))\n\ncm = confusion_matrix(y_true_class, y_pred_class)\nplt.figure(figsize=(8, 6))\nsns.heatmap(\n    cm, annot=True, fmt='d',\n    xticklabels=CLASS_NAMES,\n    yticklabels=CLASS_NAMES,\n    cmap='Blues'\n)\nplt.title('Confusion Matrix — Improved DenseNet-169')\nplt.ylabel('True Label'); plt.xlabel('Predicted Label')\nplt.xticks(rotation=20)\nplt.tight_layout()\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 16: Sample Predictions","metadata":{}},{"cell_type":"code","source":"indices      = np.random.choice(len(X_val_scaled), 10, replace=False)\nsample_preds = model.predict(X_val_scaled[indices], verbose=0)\n\nsample_pred_class = (sample_preds > best_threshold).astype(int).sum(axis=1) - 1\nsample_pred_class = np.clip(sample_pred_class, 0, 4)\nsample_true_class = y_val[indices].sum(axis=1) - 1\n\nfig, axes = plt.subplots(2, 5, figsize=(18, 7))\naxes = axes.flatten()\nfor i in range(10):\n    img   = cv2.cvtColor((X_val_scaled[indices[i]] * 255).astype(np.uint8), cv2.COLOR_BGR2RGB)\n    color = 'green' if sample_true_class[i] == sample_pred_class[i] else 'red'\n    axes[i].imshow(img)\n    axes[i].set_title(\n        f'True: {CLASS_NAMES[sample_true_class[i]]}\\nPred: {CLASS_NAMES[sample_pred_class[i]]}',\n        color=color, fontsize=9\n    )\n    axes[i].axis('off')\n\nplt.suptitle('Sample Predictions (green = correct, red = wrong)')\nplt.tight_layout()\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 17: Final Summary","metadata":{}},{"cell_type":"code","source":"print('\\n' + '='*50)\nprint(' FINAL RESULTS SUMMARY')\nprint('='*50)\nprint(f' Validation Accuracy  : {val_acc*100:.2f}%')\nprint(f' Cohen Kappa Score    : {kappa:.4f}')\nprint(f' Best threshold       : {best_threshold:.4f}')\nprint('='*50)\nprint(' Paper target         : 90% accuracy, 0.804 kappa')\nprint('\\n' + '='*50)\nprint(' COMPARISON')\nprint('='*50)\nprint(' Previous version accuracy : 79.47%')\nprint(f' This version accuracy     : {val_acc*100:.2f}%')\nprint(' Previous version kappa    : 0.8510')\nprint(f' This version kappa        : {kappa:.4f}')\nprint('='*50)\nprint(' Key improvements:')\nprint('  ✅ Multilabel ordinal encoding')\nprint('  ✅ Binary crossentropy loss')\nprint('  ✅ 224×224 (DenseNet native size)')\nprint('  ✅ Live augmentation')\nprint('  ✅ Kappa monitored per epoch')\nprint('  ✅ Optimized prediction threshold')\nprint('  ✅ Fixed steps_per_epoch bug')","metadata":{},"outputs":[],"execution_count":null}]}