{"cells":[{"cell_type":"markdown","metadata":{},"source":"# Diabetic Retinopathy Detection — CNN + Fractal Analysis (Updated)\n\nThis notebook auto-detects which retinal dataset is attached and configures paths accordingly. It supports:\n- APTOS (aptos2019-blindness-detection)\n- IDRiD (idrid_labels.csv + images)\n- DRIVE (inside an uploaded archive folder)\n\nIt uses a classical, robust vessel extraction pipeline (CLAHE + Frangi) so segmentation works without a U-Net. Then it computes fractal features and trains two classifiers (CNN-only and CNN+Fractal) and compares them.\n\n## Instructions\n1. Attach your dataset(s) in the right Data panel. From your screenshot it looks like you have a dataset named `anandita-dhal` with an `archive` folder and `idrid_labels.csv`; this notebook will detect and use that automatically.\n2. Run all cells top-to-bottom. Use GPU for faster training.\n3. Outputs (CSV, models, PNGs) will be saved to `/kaggle/working/` and can be downloaded from Files.\n"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# CELL 1 — Auto-detect attached retinal dataset (APTOS / IDRiD / DRIVE)\nimport os, glob, pprint\nINPUT_ROOT = '/kaggle/input'\nfolders = sorted(os.listdir(INPUT_ROOT)) if os.path.exists(INPUT_ROOT) else []\nprint('Top-level folders under /kaggle/input:')\npprint.pprint(folders)\n\n# detection variables\naptos_path = None\nidrid_path = None\ndrive_path = None\n\n# Find aptos\nfor d in folders:\n    if 'aptos' in d.lower() and 'eda' not in d.lower():\n        cand = os.path.join(INPUT_ROOT, d)\n        if os.path.exists(os.path.join(cand, 'train.csv')):\n            aptos_path = cand\n            break\n\n# Find IDRiD (idrid_labels.csv or idrid folder)\nfor d in folders:\n    cand = os.path.join(INPUT_ROOT, d)\n    # check for idrid_labels.csv\n    if os.path.exists(os.path.join(cand, 'idrid_labels.csv')) or os.path.exists(os.path.join(cand, 'idrid_labels.csv'.lower())):\n        idrid_path = cand\n        break\n    # sometimes files are at root of dataset\n    if os.path.isdir(cand):\n        for f in os.listdir(cand):\n            if 'idrid' in f.lower() and f.lower().endswith('.csv'):\n                idrid_path = cand\n                break\n        if idrid_path:\n            break\n\n# Find DRIVE inside any archive-like folder\nfor d in folders:\n    cand = os.path.join(INPUT_ROOT, d)\n    # search for DRIVE subfolder\n    if os.path.isdir(cand):\n        drive_cand = os.path.join(cand, 'archive', 'DRIVE')\n        if os.path.isdir(drive_cand):\n            drive_path = drive_cand\n            break\n        # sometimes archive is directly the folder\n        drive_cand2 = os.path.join(cand, 'DRIVE')\n        if os.path.isdir(drive_cand2):\n            drive_path = drive_cand2\n            break\n\nprint('\\nDetected dataset paths:')\nprint('APTOS:', aptos_path)\nprint('IDRiD:', idrid_path)\nprint('DRIVE:', drive_path)\n\nif not any([aptos_path, idrid_path, drive_path]):\n    raise RuntimeError('No supported retinal dataset detected under /kaggle/input. Attach APTOS or IDRiD or a dataset containing DRIVE and re-run.')\n"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# CELL 2 — Load labels and detect image folder according to detected dataset\nimport pandas as pd, os, glob\nfrom pathlib import Path\n\nif aptos_path:\n    print('Using APTOS dataset at', aptos_path)\n    train_csv = os.path.join(aptos_path, 'train.csv')\n    df = pd.read_csv(train_csv)\n    img_root = os.path.join(aptos_path, 'train_images')\n    print('Loaded APTOS train.csv rows:', len(df))\n\nelif idrid_path:\n    print('Using IDRiD dataset at', idrid_path)\n    # idrid_labels.csv usually contains columns image, diagnosis (or similar)\n    csv_candidates = [os.path.join(idrid_path, 'idrid_labels.csv'), os.path.join(idrid_path, 'IDRiD_labels.csv')]\n    csv_file = next((c for c in csv_candidates if os.path.exists(c)), None)\n    if csv_file is None:\n        # try any csv with idrid in name\n        for f in glob.glob(os.path.join(idrid_path, '*.csv')):\n            if 'idrid' in os.path.basename(f).lower():\n                csv_file = f; break\n    if csv_file is None:\n        raise RuntimeError('Could not find idrid_labels.csv in the attached dataset folder.')\n    df = pd.read_csv(csv_file)\n    # detect images folder heuristically\n    possible = [os.path.join(idrid_path, 'images'), os.path.join(idrid_path, 'IDRiD_images'), os.path.join(idrid_path, 'train_images'), os.path.join(idrid_path, 'images', 'train')]\n    img_root = next((p for p in possible if os.path.isdir(p)), None)\n    \n    if img_root is None:\n        # fallback: choose parent folder that contains many png/jpg\n        imgs = glob.glob(os.path.join(idrid_path, '**', '*.png'), recursive=True)\n        img_root = str(Path(imgs[0]).parent) if imgs else None\n    print('Loaded IDRiD CSV rows:', len(df))\n\nelse:\n    # Use DRIVE\n    print('Using DRIVE at', drive_path)\n    # We'll use the DRIVE training set as label source for segmentation training if needed\n    # For classification we need a labels CSV — DRIVE only provides vessel masks; so we'll create a simple binary label (no DR/DR unknown)\n    # Prefer IDRiD or APTOS for classification; DRIVE used only for segmentation in original flow\n    df = pd.DataFrame()  # empty; user should attach APTOS or IDRiD for classification\n    img_root = drive_path\n\nprint('Image root detected:', img_root)\n"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# CELL 3 — Compute classical vessel masks + fractal features (works for APTOS and IDRiD)\nimport numpy as np, pandas as pd, cv2, glob\nfrom skimage import filters, morphology\nfrom pathlib import Path\nfrom tqdm import tqdm\n\ndef vessel_mask_classical(img_path):\n    img = cv2.imread(img_path)\n    if img is None:\n        raise FileNotFoundError(img_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    g = img[...,1]\n    clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8,8))\n    g2 = clahe.apply(g)\n    g_blur = cv2.GaussianBlur(g2, (5,5), 0)\n    fr = filters.frangi(g_blur.astype(float)/255.0)\n    fr = (255 * (fr - fr.min()) / (fr.max() - fr.min() + 1e-9)).astype('uint8')\n    mask = cv2.adaptiveThreshold(fr, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 31, 2)\n    mask = morphology.remove_small_objects(mask.astype(bool), min_size=300)\n    return mask.astype('uint8')\n\ndef boxcount_fd(mask):\n    M = mask.astype(bool)\n    if M.sum() == 0:\n        return 0.0\n    h,w = M.shape\n    max_pow = int(np.floor(np.log2(min(h,w))))\n    sizes = 2**np.arange(max_pow, 1, -1)\n    counts = []\n    for s in sizes:\n        c = 0\n        for i in range(0, h, s):\n            for j in range(0, w, s):\n                block = M[i:i+s, j:j+s]\n                if block.any():\n                    c += 1\n        counts.append(c)\n    counts = np.array(counts, dtype=float)\n    valid = counts > 0\n    if valid.sum() < 2:\n        return 0.0\n    logs = np.log(counts[valid])\n    logsizes = np.log(1.0 / sizes[valid])\n    return float(np.polyfit(logsizes, logs, 1)[0])\n\ndef skeleton_len(mask):\n    sk = morphology.skeletonize(mask > 0)\n    return int(np.sum(sk))\n\ndef vessel_area_ratio(mask):\n    return float(np.sum(mask) / mask.size)\n\n# Build image dict for df ids\nimage_dict = {}\nif img_root is None:\n    raise RuntimeError('Image root not detected — cannot compute features. Attach a supported dataset and re-run.')\nfor ext in ('*.png','*.jpg','*.jpeg'):\n    for p in glob.glob(os.path.join(img_root, ext)):\n        image_dict[Path(p).stem.lower()] = p\nprint('Mapped images:', len(image_dict))\n\nrows = []\nif df.empty:\n    # If user provided IDRiD or APTOS, df should be loaded above. If not, stop.\n    raise RuntimeError('Labels dataframe is empty. Ensure APTOS or IDRiD labels are available for classification.')\n\nfor _, r in tqdm(df.iterrows(), total=len(df)):\n    id_code = str(r[df.columns[0]]).lower()\n    if id_code not in image_dict:\n        continue\n    p = image_dict[id_code]\n    try:\n        mask = vessel_mask_classical(p)\n        rows.append({\n            'id_code': id_code,\n            'image_path': p,\n            'label': int(r[df.columns[1]]) if df.shape[1]>1 else -1,\n            'fd': boxcount_fd(mask),\n            'skeleton_length': skeleton_len(mask),\n            'vessel_area_ratio': vessel_area_ratio(mask)\n        })\n    except Exception as e:\n        print('skip', id_code, e)\n\ndfF = pd.DataFrame(rows)\ndfF.to_csv('predicted_fractal_features.csv', index=False)\nprint('Saved predicted_fractal_features.csv — rows:', len(dfF))\ndisplay(dfF.head())\n"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# CELL 4 — Train CNN-only and CNN+Fractal (EfficientNetB0) — same as before\nimport os, numpy as np, pandas as pd, cv2\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models, optimizers, callbacks\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\n\nFEATS = 'predicted_fractal_features.csv'\nif not os.path.exists(FEATS):\n    raise RuntimeError(FEATS + ' not found — run the fractal-generation cell first')\ndfF = pd.read_csv(FEATS)\ndfF = dfF.dropna(subset=['fd']).reset_index(drop=True)\nprint('Features rows after dropna:', len(dfF))\ndfF['label_binary'] = (dfF['label'] > 0).astype(int)\nprint('Label distribution (binary):')\nprint(dfF['label_binary'].value_counts())\n\ntrain_df, test_df = train_test_split(dfF, test_size=0.15, stratify=dfF['label_binary'], random_state=42)\ntrain_df, val_df = train_test_split(train_df, test_size=0.15, stratify=train_df['label_binary'], random_state=42)\n\nscaler = StandardScaler()\nscaler.fit(train_df[['fd','skeleton_length','vessel_area_ratio']])\nfor d in (train_df,val_df,test_df):\n    d[['fd','skeleton_length','vessel_area_ratio']] = scaler.transform(d[['fd','skeleton_length','vessel_area_ratio']])\n\nIMG_SIZE = (224,224)\nBATCH = 16\n\ndef load_img(path):\n    im = cv2.imread(path)\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    im = cv2.resize(im, IMG_SIZE).astype('float32')/255.0\n    return im\n\nclass Gen(tf.keras.utils.Sequence):\n    def __init__(self, df, fractal=False, batch=BATCH, augment=False):\n        self.df = df.reset_index(drop=True); self.fractal = fractal; self.batch = batch; self.augment = augment\n        self.indexes = np.arange(len(self.df)); np.random.shuffle(self.indexes)\n    def __len__(self): return int(np.ceil(len(self.df)/self.batch))\n    def __getitem__(self, idx):\n        batch_idx = self.indexes[idx*self.batch:(idx+1)*self.batch]\n        sub = self.df.iloc[batch_idx]\n        X = np.zeros((len(sub), IMG_SIZE[0], IMG_SIZE[1], 3), dtype='float32')\n        y = np.zeros((len(sub),), dtype='int32')\n        F = np.zeros((len(sub),3), dtype='float32') if self.fractal else None\n        for i, (_, r) in enumerate(sub.iterrows()):\n            X[i] = load_img(r['image_path'])\n            y[i] = int(r['label_binary'])\n            if self.fractal:\n                F[i] = [r['fd'], r['skeleton_length'], r['vessel_area_ratio']]\n        if self.fractal:\n            return [X, F], y\n        return X, y\n    def on_epoch_end(self): np.random.shuffle(self.indexes)\n\ndef build_cnn_only():\n    inp = layers.Input((224,224,3))\n    base = tf.keras.applications.EfficientNetB0(include_top=False, weights='imagenet', input_tensor=inp, pooling='avg')\n    x = layers.Dropout(0.4)(base.output)\n    out = layers.Dense(1, activation='sigmoid')(x)\n    model = models.Model(inputs=inp, outputs=out)\n    model.compile(optimizer=optimizers.Adam(1e-4), loss='binary_crossentropy', metrics=['accuracy'])\n    return model\n\ndef build_cnn_plus_fractal():\n    img_in = layers.Input((224,224,3))\n    fract_in = layers.Input((3,))\n    base = tf.keras.applications.EfficientNetB0(include_top=False, weights='imagenet', input_tensor=img_in, pooling='avg')\n    x = layers.Concatenate()([base.output, fract_in])\n    x = layers.Dense(256, activation='relu')(x)\n    x = layers.Dropout(0.4)(x)\n    out = layers.Dense(1, activation='sigmoid')(x)\n    model = models.Model(inputs=[img_in, fract_in], outputs=out)\n    model.compile(optimizer=optimizers.Adam(1e-4), loss='binary_crossentropy', metrics=['accuracy'])\n    return model\n\ntrain_gen_A = Gen(train_df, fractal=False, batch=BATCH, augment=True)\nval_gen_A = Gen(val_df, fractal=False, batch=BATCH, augment=False)\ntrain_gen_B = Gen(train_df, fractal=True, batch=BATCH, augment=True)\nval_gen_B = Gen(val_df, fractal=True, batch=BATCH, augment=False)\n\nes = callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\nrlr = callbacks.ReduceLROnPlateau(monitor='val_loss', patience=3, factor=0.5)\n\nprint('Training Model A (CNN-only)...')\nmodelA = build_cnn_only()\nhistA = modelA.fit(train_gen_A, validation_data=val_gen_A, epochs=12, callbacks=[es, rlr], verbose=2)\nmodelA.save('modelA_cnn_only.h5')\n\nprint('Training Model B (CNN+Fractal)...')\nmodelB = build_cnn_plus_fractal()\nhistB = modelB.fit(train_gen_B, validation_data=val_gen_B, epochs=12, callbacks=[es, rlr], verbose=2)\nmodelB.save('modelB_cnn_fractal.h5')\n\ntrain_df.to_csv('train_df_split.csv', index=False)\nval_df.to_csv('val_df_split.csv', index=False)\ntest_df.to_csv('test_df_split.csv', index=False)\nprint('Training complete and models saved.')\n"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# CELL 5 — Evaluate models and generate plots\nimport numpy as np, pandas as pd, matplotlib.pyplot as plt\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\nimport seaborn as sns, cv2\nsns.set()\ntest_df = pd.read_csv('test_df_split.csv')\nfrom tensorflow.keras.models import load_model\nmodelA = load_model('modelA_cnn_only.h5', compile=False)\nmodelB = load_model('modelB_cnn_fractal.h5', compile=False)\nclass SimpleEval:\n    def __init__(self, df, fractal=False, batch=16):\n        self.df = df.reset_index(drop=True); self.fractal = fractal; self.batch = batch\n    def __iter__(self):\n        for i in range(0, len(self.df), self.batch):\n            sub = self.df.iloc[i:i+self.batch]\n            X = np.zeros((len(sub),224,224,3), dtype='float32')\n            F = np.zeros((len(sub),3), dtype='float32') if self.fractal else None\n            y = np.zeros((len(sub),), dtype='int32')\n            for j, (_, r) in enumerate(sub.iterrows()):\n                im = cv2.imread(r['image_path'])\n                im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n                im = cv2.resize(im, (224,224)).astype('float32')/255.0\n                X[j] = im\n                y[j] = int(r['label_binary'])\n                if self.fractal:\n                    F[j] = [r['fd'], r['skeleton_length'], r['vessel_area_ratio']]\n            if self.fractal:\n                yield (X, F), y\n            else:\n                yield X, y\ndef eval_model(model, gen, fractal=False):\n    y_true = []; y_pred = []\n    for batch in gen:\n        if fractal:\n            (X,F), y = batch\n            p = model.predict([X,F], verbose=0).ravel()\n        else:\n            X, y = batch\n            p = model.predict(X, verbose=0).ravel()\n        y_true.extend(y.tolist())\n        y_pred.extend((p>0.5).astype(int).tolist())\n    return np.array(y_true), np.array(y_pred)\ntestA = SimpleEval(test_df, fractal=False)\ntestB = SimpleEval(test_df, fractal=True)\nytA, ypA = eval_model(modelA, testA, fractal=False)\nytB, ypB = eval_model(modelB, testB, fractal=True)\nmetricsA = {'accuracy': accuracy_score(ytA, ypA),'precision': precision_score(ytA, ypA, zero_division=0),'recall': recall_score(ytA, ypA, zero_division=0),'f1': f1_score(ytA, ypA, zero_division=0)}\nmetricsB = {'accuracy': accuracy_score(ytB, ypB),'precision': precision_score(ytB, ypB, zero_division=0),'recall': recall_score(ytB, ypB, zero_division=0),'f1': f1_score(ytB, ypB, zero_division=0)}\nprint('Model A (CNN-only):', metricsA)\nprint('Model B (CNN+Fractal):', metricsB)\npd.DataFrame([{'model':'cnn_only', **metricsA}, {'model':'cnn_fractal', **metricsB}]).to_csv('model_comparison_metrics.csv', index=False)\nkeys = ['accuracy','precision','recall','f1']\na_vals = [metricsA[k] for k in keys]\nb_vals = [metricsB[k] for k in keys]\nx = np.arange(len(keys))\nplt.figure(figsize=(8,4))\nplt.bar(x-0.2, a_vals, width=0.4, label='CNN-only')\nplt.bar(x+0.2, b_vals, width=0.4, label='CNN+Fractal')\nplt.xticks(x, [k.upper() for k in keys]); plt.ylim(0,1); plt.legend(); plt.title('Model comparison')\nplt.savefig('model_comparison_bar.png', dpi=150)\nplt.show()\ncmA = confusion_matrix(ytA, ypA)\ncmB = confusion_matrix(ytB, ypB)\nplt.figure(figsize=(10,4))\nplt.subplot(1,2,1); sns.heatmap(cmA, annot=True, fmt='d', cmap='Blues'); plt.title('CNN-only')\nplt.subplot(1,2,2); sns.heatmap(cmB, annot=True, fmt='d', cmap='Blues'); plt.title('CNN+Fractal')\nplt.savefig('confusion_matrices.png', dpi=150)\nplt.show()"},{"cell_type":"markdown","metadata":{},"source":"### Notebook created and saved. Download from the link below once ready.\n"}],"metadata":{"kernelspec":{"display_name":"Python 3 (TensorFlow)","name":"python3"},"language_info":{"name":"python","version":"3.11"}},"nbformat":4,"nbformat_minor":5}