{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"}],"dockerImageVersionId":31240,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"DR detection version 3","metadata":{}},{"cell_type":"code","source":"import os\nimport cv2\nimport glob\nimport time\nimport random\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, confusion_matrix, roc_auc_score, roc_curve, precision_score, recall_score, f1_score\nfrom tensorflow.keras.applications import EfficientNetB3\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout, Conv2D, MaxPooling2D, BatchNormalization, Activation, Multiply, Add, Input, Layer\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\nimport warnings\n\nwarnings.filterwarnings('ignore')\n\n# ------------------------------------------------------------------------------\n# CONFIGURATION\n# ------------------------------------------------------------------------------\nSEED = 42\nIMG_SIZE = 300 \nBATCH_SIZE = 8   \nEPOCHS = 30      \nLEARNING_RATE = 1e-4\n\n# Standard Kaggle Paths - Change these if running locally with different structure\nDATA_DIR = '/kaggle/input/aptos2019-blindness-detection'\nTRAIN_IMG_DIR = os.path.join(DATA_DIR, 'train_images')\nTEST_IMG_DIR = os.path.join(DATA_DIR, 'test_images')\nCSV_PATH = os.path.join(DATA_DIR, 'train.csv')\nOUTPUT_DIR = './'\nPREPROCESS_OUT_DIR = os.path.join(OUTPUT_DIR, 'preprocessing_outputs')\nPLOTS_DIR = os.path.join(OUTPUT_DIR, 'plots')\n\nos.makedirs(PREPROCESS_OUT_DIR, exist_ok=True)\nos.makedirs(PLOTS_DIR, exist_ok=True)\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\nseed_everything(SEED)\n\nprint(f\"TensorFlow Version: {tf.__version__}\")\nprint(f\"Num GPUs Available: {len(tf.config.list_physical_devices('GPU'))}\")\n\n# ------------------------------------------------------------------------------\n# DATA PREPROCESSING\n# ------------------------------------------------------------------------------\n\ndef crop_image_from_gray(img, tol=7):\n    \"\"\"\n    Crops the black borders around the circular fundus image.\n    \"\"\"\n    if img.ndim == 2:\n        mask = img > tol\n        return img[np.ix_(mask.any(1), mask.any(0))]\n    elif img.ndim == 3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img > tol\n        check_shape = img[:,:,0][np.ix_(mask.any(1), mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1 = img[:,:,0][np.ix_(mask.any(1), mask.any(0))]\n            img2 = img[:,:,1][np.ix_(mask.any(1), mask.any(0))]\n            img3 = img[:,:,2][np.ix_(mask.any(1), mask.any(0))]\n            img = np.stack([img1, img2, img3], axis=-1)\n        return img\n\ndef apply_clahe(img):\n    \"\"\"\n    Applies CLAHE (Contrast Limited Adaptive Histogram Equalization)\n    to the green channel (often most informative for DR) or L channel in LAB.\n    \"\"\"\n    lab = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)\n    l, a, b = cv2.split(lab)\n    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\n    cl = clahe.apply(l)\n    limg = cv2.merge((cl, a, b))\n    final = cv2.cvtColor(limg, cv2.COLOR_LAB2RGB)\n    return final\n\ndef ben_graham_preprocessing(img, sigmaX=10):\n    \"\"\"\n    Applies Ben Graham's preprocessing method:\n    image = image * alpha + gaussian_blur * beta + gamma\n    \"\"\"\n    image = cv2.addWeighted(img, 4, cv2.GaussianBlur(img, (0, 0), sigmaX), -4, 128)\n    return image\n\ndef preprocess_pipeline(image_path, sigmaX=10, visualize=False):\n    \"\"\"\n    Reads image and applies the full pipeline:\n    1. Read -> 2. Crop black borders -> 3. Resize -> 4. Ben Graham\n    \"\"\"\n    try:\n        # Read\n        img = cv2.imread(image_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        # 1. Visualization Original\n        orig = img.copy()\n        \n        # 2. Crop\n        img_cropped = crop_image_from_gray(img)\n        img_cropped = cv2.resize(img_cropped, (IMG_SIZE, IMG_SIZE))\n        \n        # 3. CLAHE (Standalone for visualization)\n        img_clahe = apply_clahe(img_cropped)\n        \n        # 4. Ben Graham (Standalone for visualization & Usage)\n        img_ben = ben_graham_preprocessing(img_cropped, sigmaX=sigmaX)\n        \n        # Final Preprocessed Image\n        final_img = img_ben\n        \n        # Normalize to [0, 1]\n        final_img_norm = final_img.astype('float32') / 255.0\n\n        if visualize:\n            return orig, img_ben, img_clahe, final_img\n        else:\n            return final_img_norm\n            \n    except Exception as e:\n        print(f\"Error processing {image_path}: {e}\")\n        return np.zeros((IMG_SIZE, IMG_SIZE, 3))\n\ndef visualize_preprocessing_steps(df, n=3):\n    sample_df = df.sample(n)\n    for idx, row in sample_df.iterrows():\n        img_id = row['id_code']\n        path = os.path.join(TRAIN_IMG_DIR, f\"{img_id}.png\")\n        if not os.path.exists(path): continue\n        orig, ben, clahe, final = preprocess_pipeline(path, visualize=True)\n        \n        fig, axes = plt.subplots(1, 4, figsize=(20, 5))\n        axes[0].imshow(orig); axes[0].set_title(\"Original\"); axes[0].axis('off')\n        axes[1].imshow(ben); axes[1].set_title(\"Ben Graham\"); axes[1].axis('off')\n        axes[2].imshow(clahe); axes[2].set_title(\"CLAHE\"); axes[2].axis('off')\n        axes[3].imshow(final); axes[3].set_title(\"Final Input\"); axes[3].axis('off')\n        \n        save_path = os.path.join(PREPROCESS_OUT_DIR, f\"preprocess_{img_id}.png\")\n        plt.tight_layout(); plt.savefig(save_path); plt.close()\n        print(f\"Saved preprocessing visualization to {save_path}\")\n\n# ------------------------------------------------------------------------------\n# DATA GENERATORS\n# ------------------------------------------------------------------------------\n\ndef find_dataset_path():\n    candidates = [\n        '../input/aptos2019-blindness-detection',\n        '/kaggle/input/aptos2019-blindness-detection',\n        './aptos2019-blindness-detection',\n        'F:/Major Project/Diabetic Retinopathy/aptos2019-blindness-detection'\n    ]\n    for path in candidates:\n        if os.path.exists(path):\n            return path\n    return None\n\ndef get_data_generators():\n    detected_path = find_dataset_path()\n    \n    if detected_path:\n        print(f\"✅ Dataset found at: {detected_path}\")\n        global TRAIN_IMG_DIR, CSV_PATH\n        TRAIN_IMG_DIR = os.path.join(detected_path, 'train_images')\n        CSV_PATH = os.path.join(detected_path, 'train.csv')\n    else:\n        if not os.path.exists(CSV_PATH):\n            print(f\"❌ CRITICAL ERROR: Dataset not found at {CSV_PATH}\")\n            raise FileNotFoundError(f\"Dataset not found at {CSV_PATH}\")\n        else:\n            print(f\"✅ Using configured path: {CSV_PATH}\")\n\n    df = pd.read_csv(CSV_PATH)\n    \n    # Binary Classification\n    df['binary_target'] = df['diagnosis'].apply(lambda x: 1 if x > 0 else 0)\n    df['file_path'] = df['id_code'].apply(lambda x: os.path.join(TRAIN_IMG_DIR, f\"{x}.png\"))\n    \n    # Stratified Split\n    if len(df) < 10:\n        print(\"WARNING: Dataset too small for stratified split. Using simple split.\")\n        train, val = train_test_split(df, test_size=0.2, random_state=SEED)\n    else:\n        try:\n            train, val = train_test_split(df, test_size=0.2, random_state=SEED, stratify=df['binary_target'])\n        except ValueError as e:\n            print(f\"Stratified split failed: {e}. Fallback to random split.\")\n            train, val = train_test_split(df, test_size=0.2, random_state=SEED)\n    \n    print(f\"Training set: {len(train)}\")\n    print(f\"Validation set: {len(val)}\")\n    \n    class DataGenerator(tf.keras.utils.Sequence):\n        def __init__(self, df, batch_size=16, shuffle=True, augment=False):\n            self.df = df\n            self.batch_size = batch_size\n            self.shuffle = shuffle\n            self.augment = augment\n            self.indices = np.arange(len(self.df))\n            self.on_epoch_end()\n            \n            self.aug_layers = tf.keras.Sequential([\n                tf.keras.layers.RandomFlip(\"horizontal\"),\n                tf.keras.layers.RandomRotation(0.1),\n                tf.keras.layers.RandomZoom(0.1),\n                tf.keras.layers.RandomContrast(0.1)\n            ]) if augment else None\n\n        def __len__(self):\n            return int(np.ceil(len(self.df) / self.batch_size))\n\n        def __getitem__(self, index):\n            indices = self.indices[index*self.batch_size:(index+1)*self.batch_size]\n            batch_df = self.df.iloc[indices]\n            \n            current_batch_size = len(batch_df)\n            \n            X = np.empty((current_batch_size, IMG_SIZE, IMG_SIZE, 3), dtype=np.float32)\n            y = np.empty((current_batch_size), dtype=np.float32)\n            \n            for i, (_, row) in enumerate(batch_df.iterrows()):\n                if not os.path.exists(row['file_path']):\n                    if index == 0 and i == 0: \n                         print(f\"Warning: Image not found {row['file_path']}\")\n                    img = np.zeros((IMG_SIZE, IMG_SIZE, 3), dtype=np.float32)\n                else:\n                    img = preprocess_pipeline(row['file_path'], visualize=False)\n                X[i,] = img\n                y[i] = row['binary_target']\n                \n            if self.augment and current_batch_size > 0:\n                X = self.aug_layers(X)\n                \n            return X, y\n\n        def on_epoch_end(self):\n            if self.shuffle:\n                np.random.shuffle(self.indices)\n\n    actual_batch_size = min(BATCH_SIZE, len(train)//2) if len(train) > 0 else BATCH_SIZE\n    actual_batch_size = max(1, actual_batch_size)\n    \n    train_gen = DataGenerator(train, batch_size=actual_batch_size, augment=True)\n    val_gen = DataGenerator(val, batch_size=actual_batch_size, augment=False)\n    \n    neg, pos = np.bincount(df['binary_target'])\n    total = neg + pos\n    weight0 = (1 / neg) * (total / 2.0) if neg > 0 else 1.0\n    weight1 = (1 / pos) * (total / 2.0) if pos > 0 else 1.0\n    class_weights = {0: weight0, 1: weight1}\n    print(f\"Class Weights: {class_weights}\")\n    \n    return train_gen, val_gen, class_weights, df\n\n# ------------------------------------------------------------------------------\n# MODEL ARCHITECTURE: CBAM + EfficientNet\n# ------------------------------------------------------------------------------\n\nclass CBAMBlock(Layer):\n    \"\"\"\n    Convolutional Block Attention Module (CBAM)\n    \"\"\"\n    def __init__(self, reduction_ratio=16, **kwargs):\n        super(CBAMBlock, self).__init__(**kwargs)\n        self.reduction_ratio = reduction_ratio\n\n    def build(self, input_shape):\n        channel_dims = input_shape[-1]\n        \n        # Channel Attention\n        self.global_avg_pool = GlobalAveragePooling2D()\n        self.global_max_pool = tf.keras.layers.GlobalMaxPooling2D()\n        \n        self.shared_dense_one = Dense(channel_dims // self.reduction_ratio, activation='relu', use_bias=True)\n        self.shared_dense_two = Dense(channel_dims, use_bias=True)\n        \n        # Spatial Attention\n        self.conv2d_spatial = Conv2D(1, (7, 7), padding='same', activation='sigmoid', use_bias=False)\n        \n        super(CBAMBlock, self).build(input_shape)\n\n    def call(self, inputs):\n        # Channel Attention\n        avg_pool = self.global_avg_pool(inputs)\n        avg_pool = tf.keras.layers.Reshape((1, 1, avg_pool.shape[1]))(avg_pool)\n        avg_out = self.shared_dense_two(self.shared_dense_one(avg_pool))\n        \n        max_pool = self.global_max_pool(inputs)\n        max_pool = tf.keras.layers.Reshape((1, 1, max_pool.shape[1]))(max_pool)\n        max_out = self.shared_dense_two(self.shared_dense_one(max_pool))\n        \n        channel_attention = Activation('sigmoid')(Add()([avg_out, max_out]))\n        channel_refined = Multiply()([inputs, channel_attention])\n        \n        # Spatial Attention\n        avg_pool_spatial = tf.reduce_mean(channel_refined, axis=-1, keepdims=True)\n        max_pool_spatial = tf.reduce_max(channel_refined, axis=-1, keepdims=True)\n        concat_spatial = tf.concat([avg_pool_spatial, max_pool_spatial], axis=-1)\n        spatial_attention = self.conv2d_spatial(concat_spatial)\n        \n        return Multiply()([channel_refined, spatial_attention])\n\ndef build_dr_model(input_shape=(IMG_SIZE, IMG_SIZE, 3)):\n    \"\"\"\n    Builds the model with Frozen Batch Normalization.\n    \"\"\"\n    base_model = EfficientNetB3(weights='imagenet', include_top=False, input_shape=input_shape)\n    \n    # FREEZE BACKBONE INITIALLY\n    base_model.trainable = False\n    \n    x = base_model.output\n    x = CBAMBlock(reduction_ratio=16, name=\"CBAM_Block\")(x)\n    \n    x = GlobalAveragePooling2D()(x)\n    x = BatchNormalization()(x)\n    x = Dense(256, activation='relu')(x)\n    x = Dropout(0.4)(x)\n    outputs = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inputs=base_model.input, outputs=outputs)\n    return model, base_model\n\ndef unfreeze_model(model, base_model, n_layers=0):\n    \"\"\"\n    Unfreezes the base model for fine-tuning, BUT KEEPS BATCH NORM FROZEN.\n    \"\"\"\n    base_model.trainable = True\n    \n    # CRITICAL: Freeze all BatchNormalization layers in the base model\n    for layer in base_model.layers:\n        if isinstance(layer, BatchNormalization):\n            layer.trainable = False\n            \n    optimizer = Adam(learning_rate=1e-5) # Very low LR for fine-tuning\n    model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy', tf.keras.metrics.AUC(name='auc')])\n    print(\"Model unfreezed (BN kept frozen) and recompiled with LR=1e-5\")\n    return model\n\n# ------------------------------------------------------------------------------\n# TRAINING\n# ------------------------------------------------------------------------------\n\ndef train_network():\n    print(\"\\n--- Starting Pipeline ---\")\n    \n    # 1. Setup Data\n    train_gen, val_gen, class_weights, full_df = get_data_generators()\n    \n    # 2. Visualize Preprocessing\n    # visualize_preprocessing_steps(full_df, n=5) \n    \n    # 3. Build Model (Phase 1: Frozen Backbone)\n    model, base_model = build_dr_model()\n    \n    optimizer = Adam(learning_rate=1e-3)\n    model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy', tf.keras.metrics.AUC(name='auc')])\n    model.summary()\n    \n    # 4. Callbacks\n    checkpoint = ModelCheckpoint(\n        os.path.join(OUTPUT_DIR, 'best_model.keras'), \n        monitor='val_auc', \n        verbose=1, \n        save_best_only=True,\n        mode='max'\n    )\n    early_stop = EarlyStopping(monitor='val_loss', patience=5, verbose=1, restore_best_weights=True)\n    \n    # --- PHASE 1: WARM UP HEAD (5 Epochs) ---\n    print(\"\\n[PHASE 1] Training Custom Head (Backbone Frozen)...\")\n    history_phase1 = model.fit(\n        train_gen,\n        validation_data=val_gen,\n        epochs=5,\n        class_weight=class_weights,\n        callbacks=[checkpoint] \n    )\n    \n    # --- PHASE 2: FINE TUNE ---\n    print(\"\\n[PHASE 2] Fine-tuning Backbone (BN Frozen)...\")\n    model = unfreeze_model(model, base_model)\n    \n    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=2, verbose=1, min_lr=1e-7)\n    \n    history_phase2 = model.fit(\n        train_gen,\n        validation_data=val_gen,\n        epochs=EPOCHS, \n        initial_epoch=5, \n        class_weight=class_weights,\n        callbacks=[checkpoint, reduce_lr, early_stop]\n    )\n    \n    # Merge histories\n    train_acc = history_phase1.history['accuracy'] + history_phase2.history['accuracy']\n    val_acc = history_phase1.history['val_accuracy'] + history_phase2.history['val_accuracy']\n    train_loss = history_phase1.history['loss'] + history_phase2.history['loss']\n    val_loss = history_phase1.history['val_loss'] + history_phase2.history['val_loss']\n    \n    class MergedHistory:\n        def __init__(self):\n            self.history = {\n                'accuracy': train_acc, 'val_accuracy': val_acc,\n                'loss': train_loss, 'val_loss': val_loss\n            }\n            \n    final_history = MergedHistory()\n    return model, final_history, val_gen\n\n# ------------------------------------------------------------------------------\n# EVALUATION & PLOTTING\n# ------------------------------------------------------------------------------\n\ndef plot_history(history):\n    plt.figure(figsize=(10, 5))\n    plt.plot(history.history['accuracy'], label='Train Accuracy')\n    plt.plot(history.history['val_accuracy'], label='Val Accuracy')\n    plt.title('Accuracy over Epochs')\n    plt.xlabel('Epochs')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    plt.savefig(os.path.join(PLOTS_DIR, 'accuracy_plot.png'))\n    plt.close()\n    \n    plt.figure(figsize=(10, 5))\n    plt.plot(history.history['loss'], label='Train Loss')\n    plt.plot(history.history['val_loss'], label='Val Loss')\n    plt.title('Loss over Epochs')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n    plt.savefig(os.path.join(PLOTS_DIR, 'loss_plot.png'))\n    plt.close()\n\ndef evaluate_model(model, val_gen):\n    print(\"\\n--- Evaluating Model ---\")\n    \n    y_true = []\n    y_pred_proba = []\n    \n    val_gen.shuffle = False\n    val_gen.on_epoch_end()\n    \n    for i in range(len(val_gen)):\n        X_batch, y_batch = val_gen[i]\n        preds = model.predict(X_batch, verbose=0)\n        y_true.extend(y_batch)\n        y_pred_proba.extend(preds.flatten())\n    \n    y_true = np.array(y_true)\n    y_pred_proba = np.array(y_pred_proba)\n    y_pred = (y_pred_proba > 0.5).astype(int)\n    \n    acc = np.mean(y_true == y_pred)\n    prec = precision_score(y_true, y_pred, zero_division=0)\n    rec = recall_score(y_true, y_pred, zero_division=0)\n    f1 = f1_score(y_true, y_pred, zero_division=0)\n    try: auc = roc_auc_score(y_true, y_pred_proba)\n    except: auc = 0.5\n        \n    print(f\"\\nFinal Test Accuracy: {acc:.4f}\")\n    print(f\"Precision: {prec:.4f}\")\n    print(f\"Recall: {rec:.4f}\")\n    print(f\"F1 Score: {f1:.4f}\")\n    print(f\"ROC-AUC: {auc:.4f}\")\n    \n    cm = confusion_matrix(y_true, y_pred)\n    plt.figure(figsize=(6, 5))\n    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')\n    plt.title('Confusion Matrix')\n    plt.ylabel('True Label')\n    plt.xlabel('Predicted Label')\n    plt.savefig(os.path.join(PLOTS_DIR, 'confusion_matrix.png'))\n    plt.close()\n    \n    fpr, tpr, _ = roc_curve(y_true, y_pred_proba)\n    plt.figure(figsize=(8, 6))\n    plt.plot(fpr, tpr, label=f\"AUC = {auc:.4f}\")\n    plt.plot([0, 1], [0, 1], 'r--')\n    plt.title('ROC Curve')\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.legend()\n    plt.savefig(os.path.join(PLOTS_DIR, 'roc_curve.png'))\n    plt.close()\n    \n    results_df = pd.DataFrame({\n        'True': y_true,\n        'Pred_Proba': y_pred_proba,\n        'Pred_Label': y_pred\n    })\n    results_df.to_csv(os.path.join(OUTPUT_DIR, 'predictions.csv'), index=False)\n    print(f\"Results saved to {os.path.join(OUTPUT_DIR, 'predictions.csv')}\")\n\nif __name__ == '__main__':\n    try:\n        model, history, val_gen = train_network()\n        plot_history(history)\n        evaluate_model(model, val_gen)\n        print(\"\\nPipeline execution complete.\")\n    except Exception as e:\n        print(f\"\\n[CRITICAL ERROR] Pipeline Failed: {e}\")\n        import traceback\n        traceback.print_exc()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2026-01-04T06:54:40.227980Z","iopub.execute_input":"2026-01-04T06:54:40.228250Z","iopub.status.idle":"2026-01-04T11:55:59.777159Z","shell.execute_reply.started":"2026-01-04T06:54:40.228227Z","shell.execute_reply":"2026-01-04T11:55:59.776570Z"}},"outputs":[],"execution_count":null}]}