{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"accelerator":"GPU","kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":14774,"databundleVersionId":875431},{"sourceType":"datasetVersion","sourceId":527603,"datasetId":250877,"databundleVersionId":543908}],"dockerImageVersionId":31287,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"3fc31854-c45c-4dce-836c-cb0c5fcfb3df","cell_type":"markdown","source":"# 🔬 Diabetic Retinopathy — Full Pipeline + Fine-Tuned ResNet50\n**Based on:** *Medhi et al. (2023) — IJERR Vol. 31*\n\n| Stage | Details |\n|-------|---------|\n| Datasets | APTOS 2019 + EyePACS 2015 |\n| Preprocessing | Crop → Gaussian Blur Sub → Brightness/Contrast → 256×256 |\n| Augmentation | Flip, Rotate ±30°, Zoom, Brightness jitter |\n| Balancing | 6,967 images per class → ~34,835 total |\n| Split | 80% train / 20% test (stratified) |\n| Model | Fine-tuned ResNet50 (ImageNet) |\n| Optimizer | SGD lr=0.005, momentum=0.9 |\n| Target | Train: 98.69% \\| Val: 89.21% |\n\n> ⚠️ **Disk strategy:** Each intermediate folder is deleted immediately after it is no longer needed to stay within Kaggle's 20 GB limit.","metadata":{}},{"id":"7c7d1999-f95d-4672-97ec-b04fc7553e88","cell_type":"markdown","source":"## ⚙️ Step 0 — Imports","metadata":{}},{"id":"5015f959-7537-4deb-b78c-c136604c993a","cell_type":"code","source":"import os, cv2, random, shutil, glob, warnings, threading\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n\nimport tensorflow as tf\nfrom tensorflow.keras.applications import ResNet50\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import (Dense, Dropout, GlobalAveragePooling2D,\n                                      BatchNormalization)\nfrom tensorflow.keras.optimizers import SGD\nfrom tensorflow.keras.callbacks import (EarlyStopping, ModelCheckpoint,\n                                         ReduceLROnPlateau, CSVLogger)\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nwarnings.filterwarnings('ignore')\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\n\nprint(f'TensorFlow   : {tf.__version__}')\nprint(f'GPU available: {len(tf.config.list_physical_devices(\"GPU\")) > 0}')\nprint('✅ All imports done')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-11T14:33:32.549357Z","iopub.execute_input":"2026-03-11T14:33:32.549649Z","iopub.status.idle":"2026-03-11T14:34:21.067708Z","shell.execute_reply.started":"2026-03-11T14:33:32.549610Z","shell.execute_reply":"2026-03-11T14:34:21.066801Z"}},"outputs":[],"execution_count":null},{"id":"9a712730-f24c-43b9-8f65-01598bac273d","cell_type":"markdown","source":"## 📂 Step 1 — Paths & Config","metadata":{}},{"id":"3c8c9ebf-6b6d-4b36-be2b-2934b780469d","cell_type":"code","source":"# ── Dataset input paths ────────────────────────────────────────────────────\nAPTOS_CSV       = '/kaggle/input/competitions/aptos2019-blindness-detection/train.csv'\nAPTOS_IMG_DIR   = '/kaggle/input/competitions/aptos2019-blindness-detection/train_images'\nEYEPACS_CSV     = '/kaggle/input/datasets/donkeys/retinopathy-train-2015/trainLabels.csv'\nEYEPACS_IMG_DIR = '/kaggle/input/datasets/donkeys/retinopathy-train-2015/rescaled_train_896'\n\n# ── Output paths  (only ONE copy of images exists at any time) ─────────────\nOUT_PRE   = '/kaggle/working/preprocessed'   # deleted after augmentation\nOUT_SPLIT = '/kaggle/working/final_split'    # permanent — used for training\nMODEL_DIR = '/kaggle/working/models'\n\nfor d in [OUT_PRE, OUT_SPLIT, MODEL_DIR]:\n    os.makedirs(d, exist_ok=True)\n\n# ── Constants ──────────────────────────────────────────────────────────────\nCLASS_NAMES      = {0:'No_DR', 1:'Mild', 2:'Moderate', 3:'Severe', 4:'Proliferative_DR'}\nIMG_SIZE         = (256, 256)\nINPUT_SHAPE      = (256, 256, 3)\nTARGET_PER_CLASS = 6967\nTRAIN_RATIO      = 0.80\nNUM_WORKERS      = 4\nCOLORS           = ['#4CAF50','#2196F3','#FF9800','#F44336','#9C27B0']\n\n# ── Model hyperparameters (paper) ──────────────────────────────────────────\nBATCH_SIZE  = 32\nEPOCHS      = 50\nLR          = 0.005\nMOMENTUM    = 0.9\nDROPOUT     = 0.3\nNUM_CLASSES = 5\n\n# ── Helper: print free disk space ──────────────────────────────────────────\ndef disk_free():\n    st = os.statvfs('/kaggle/working')\n    free_gb = st.f_bavail * st.f_frsize / 1024**3\n    print(f'  💾 Free disk space: {free_gb:.2f} GB')\n\n# ── Verify input paths ─────────────────────────────────────────────────────\nall_ok = True\nfor label, path in [('APTOS CSV',    APTOS_CSV),\n                     ('APTOS images', APTOS_IMG_DIR),\n                     ('EyePACS CSV',  EYEPACS_CSV),\n                     ('EyePACS imgs', EYEPACS_IMG_DIR)]:\n    ok = os.path.exists(path)\n    print(f\"{'✅' if ok else '❌'}  {label:15s}: {path}\")\n    if not ok: all_ok = False\n\nif all_ok:\n    print(f'\\n  APTOS images   : {len(glob.glob(APTOS_IMG_DIR + \"/*\"))}')\n    print(f'  EyePACS images : {len(glob.glob(EYEPACS_IMG_DIR + \"/*\"))}')\n    disk_free()\n    print('\\n🚀 All paths OK!')\nelse:\n    print('\\n⚠️  Fix paths before continuing.')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-11T14:34:21.070031Z","iopub.execute_input":"2026-03-11T14:34:21.070619Z","iopub.status.idle":"2026-03-11T14:34:22.228182Z","shell.execute_reply.started":"2026-03-11T14:34:21.070590Z","shell.execute_reply":"2026-03-11T14:34:22.227309Z"}},"outputs":[],"execution_count":null},{"id":"c02da920-07aa-4eb0-93c1-f3ba7950c587","cell_type":"markdown","source":"## 📊 Step 2 — Load & Merge Datasets","metadata":{}},{"id":"ff9179d5-ebf3-4695-96e1-da990848e5b4","cell_type":"code","source":"# ── APTOS 2019 ─────────────────────────────────────────────────────────────\naptos_df = pd.read_csv(APTOS_CSV)\naptos_df = aptos_df.rename(columns={'id_code':'image_id', 'diagnosis':'label'})\naptos_df['image_path'] = APTOS_IMG_DIR + '/' + aptos_df['image_id'] + '.png'\naptos_df['source']     = 'aptos2019'\naptos_df               = aptos_df[['image_id','label','image_path','source']]\n\n# ── EyePACS 2015 ───────────────────────────────────────────────────────────\neyepacs_df = pd.read_csv(EYEPACS_CSV)\neyepacs_df = eyepacs_df.rename(columns={'image':'image_id', 'level':'label'})\n\ndef find_eyepacs_img(img_id):\n    for ext in ('.jpeg', '.jpg', '.png'):\n        p = EYEPACS_IMG_DIR + '/' + img_id + ext\n        if os.path.exists(p): return p\n    return EYEPACS_IMG_DIR + '/' + img_id + '.jpeg'\n\neyepacs_df['image_path'] = eyepacs_df['image_id'].apply(find_eyepacs_img)\neyepacs_df['source']     = 'eyepacs2015'\neyepacs_df               = eyepacs_df[['image_id','label','image_path','source']]\n\n# ── Merge & drop missing ───────────────────────────────────────────────────\ncombined         = pd.concat([aptos_df, eyepacs_df], ignore_index=True)\ncombined['exists'] = combined['image_path'].apply(os.path.exists)\nmissing          = (~combined['exists']).sum()\ncombined         = combined[combined['exists']].drop(columns='exists').reset_index(drop=True)\n\nprint(f'APTOS-2019   : {len(aptos_df):>6} rows')\nprint(f'EyePACS-2015 : {len(eyepacs_df):>6} rows')\nprint(f'Missing files: {missing}')\nprint(f'Combined     : {len(combined):>6} valid images\\n')\nprint('Class distribution:')\nfor lbl, name in CLASS_NAMES.items():\n    print(f'  Stage {lbl}  {name:20s}: {(combined[\"label\"]==lbl).sum()}')\ndisk_free()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-11T14:34:22.229419Z","iopub.execute_input":"2026-03-11T14:34:22.229724Z","iopub.status.idle":"2026-03-11T14:40:08.085334Z","shell.execute_reply.started":"2026-03-11T14:34:22.229699Z","shell.execute_reply":"2026-03-11T14:40:08.084517Z"}},"outputs":[],"execution_count":null},{"id":"f4d9f0ba-7b1f-4d96-be2d-8c159cd69fc9","cell_type":"code","source":"vals = [(combined['label'] == i).sum() for i in range(5)]\nfig, ax = plt.subplots(figsize=(9, 4))\nbars = ax.bar(CLASS_NAMES.values(), vals, color=COLORS, edgecolor='black', linewidth=0.7)\nax.bar_label(bars, padding=4, fontweight='bold')\nax.set_title('Combined Dataset — Before Augmentation', fontsize=13, fontweight='bold')\nax.set_ylabel('Images'); ax.set_ylim(0, max(vals) * 1.15)\nplt.xticks(rotation=15, ha='right'); plt.tight_layout(); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-11T14:40:08.086628Z","iopub.execute_input":"2026-03-11T14:40:08.086909Z","iopub.status.idle":"2026-03-11T14:40:08.383008Z","shell.execute_reply.started":"2026-03-11T14:40:08.086884Z","shell.execute_reply":"2026-03-11T14:40:08.382313Z"}},"outputs":[],"execution_count":null},{"id":"dd3b1a34-e8be-41d2-89a5-e97f438f9365","cell_type":"markdown","source":"## 🛠️ Step 3 — Preprocessing Functions","metadata":{}},{"id":"58c79dac-db9d-49e7-9294-8f8744acad66","cell_type":"code","source":"def crop_black_border(img, tol=7):\n    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) if img.ndim == 3 else img\n    mask = gray > tol\n    if mask.any():\n        rows = np.where(mask.any(axis=1))[0]\n        cols = np.where(mask.any(axis=0))[0]\n        img  = img[rows[0]:rows[-1]+1, cols[0]:cols[-1]+1]\n    return img\n\ndef gaussian_blur_subtraction(img, sigma=10):\n    blur = cv2.GaussianBlur(img, (0, 0), sigma)\n    return cv2.addWeighted(img, 4, blur, -4, 128)\n\ndef enhance_brightness_contrast(img, alpha=1.2, beta=10):\n    return cv2.convertScaleAbs(img, alpha=alpha, beta=beta)\n\ndef preprocess(path):\n    img = cv2.imread(path)\n    if img is None: return None\n    img = crop_black_border(img)\n    img = gaussian_blur_subtraction(img)\n    img = enhance_brightness_contrast(img)\n    img = cv2.resize(img, IMG_SIZE)\n    return img\n\n# ── Visual demo ────────────────────────────────────────────────────────────\nraw = cv2.imread(combined.iloc[0]['image_path'])\ns1  = crop_black_border(raw.copy())\ns2  = gaussian_blur_subtraction(s1.copy())\ns3  = enhance_brightness_contrast(s2.copy())\ns4  = cv2.resize(s3.copy(), IMG_SIZE)\n\nfig, axes = plt.subplots(1, 5, figsize=(20, 4))\nfig.suptitle('Preprocessing Pipeline — Paper Section 4', fontsize=13, fontweight='bold')\nfor ax, (title, img) in zip(axes, [\n        ('1. Original', raw), ('2. Crop Borders', s1),\n        ('3. Gaussian Blur Sub', s2), ('4. Brightness/Contrast', s3),\n        ('5. Resized 256×256', s4)]):\n    ax.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\n    ax.set_title(title, fontsize=9); ax.axis('off')\nplt.tight_layout(); plt.show()\nprint('✅ Preprocessing functions ready')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-11T14:40:08.384070Z","iopub.execute_input":"2026-03-11T14:40:08.384362Z","iopub.status.idle":"2026-03-11T14:40:10.980443Z","shell.execute_reply.started":"2026-03-11T14:40:08.384338Z","shell.execute_reply":"2026-03-11T14:40:10.979540Z"}},"outputs":[],"execution_count":null},{"id":"fd602a8e-d78e-4e3c-9206-8d507581f21f","cell_type":"markdown","source":"## 💾 Step 4 — Preprocess & Save to Temp Folder (Parallel)","metadata":{}},{"id":"7108148c-7605-40c1-afac-4d59aa06d981","cell_type":"code","source":"for name in CLASS_NAMES.values():\n    os.makedirs(OUT_PRE + '/' + name, exist_ok=True)\n\npre_records = []\nlock        = threading.Lock()\n\ndef process_one(row):\n    img = preprocess(row['image_path'])\n    if img is None: return None\n    cls = CLASS_NAMES[row['label']]\n    dst = OUT_PRE + '/' + cls + '/' + row['source'] + '_' + str(row['image_id']) + '.png'\n    cv2.imwrite(dst, img)\n    return {'image_id': row['image_id'], 'label': row['label'],\n            'class_name': cls, 'source': row['source'], 'image_path': dst}\n\nrows = [row for _, row in combined.iterrows()]\n\nwith ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:\n    futures = {executor.submit(process_one, row): row for row in rows}\n    pbar    = tqdm(as_completed(futures), total=len(futures), desc='Preprocessing')\n    for future in pbar:\n        result = future.result()\n        if result:\n            with lock:\n                pre_records.append(result)\n\npre_df = pd.DataFrame(pre_records)\npre_df.to_csv(OUT_PRE + '/manifest.csv', index=False)\nprint(f'\\n✅  {len(pre_df)} images preprocessed → {OUT_PRE}')\ndisk_free()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-11T14:40:10.981576Z","iopub.execute_input":"2026-03-11T14:40:10.981995Z","iopub.status.idle":"2026-03-11T15:04:51.855317Z","shell.execute_reply.started":"2026-03-11T14:40:10.981969Z","shell.execute_reply":"2026-03-11T15:04:51.854403Z"}},"outputs":[],"execution_count":null},{"id":"080cd2cd-891c-4a4a-9277-a4bbd3b1a24a","cell_type":"markdown","source":"## 🔄 Step 5 — Augmentation Functions","metadata":{}},{"id":"5abfc235-640f-4626-8303-44d4007c8a43","cell_type":"code","source":"def augment(img):\n    img = img.copy()\n    if random.random() > 0.5:\n        img = cv2.flip(img, 1)\n    if random.random() > 0.5:\n        img = cv2.flip(img, 0)\n    if random.random() > 0.3:\n        angle = random.uniform(-30, 30)\n        h, w  = img.shape[:2]\n        M     = cv2.getRotationMatrix2D((w//2, h//2), angle, 1.0)\n        img   = cv2.warpAffine(img, M, (w, h),\n                               flags=cv2.INTER_LINEAR,\n                               borderMode=cv2.BORDER_REFLECT)\n    if random.random() > 0.4:\n        h, w   = img.shape[:2]\n        scale  = random.uniform(0.80, 1.00)\n        nh, nw = int(h * scale), int(w * scale)\n        top    = random.randint(0, h - nh)\n        left   = random.randint(0, w - nw)\n        img    = cv2.resize(img[top:top+nh, left:left+nw], (w, h))\n    if random.random() > 0.5:\n        img = cv2.convertScaleAbs(img, alpha=random.uniform(0.8, 1.2), beta=0)\n    return img\n\n# ── Visual demo ────────────────────────────────────────────────────────────\ns        = cv2.imread(pre_df.iloc[0]['image_path'])\naug_list = [augment(s) for _ in range(8)]\nfig = plt.figure(figsize=(20, 3))\ngs  = gridspec.GridSpec(1, 9)\nax0 = fig.add_subplot(gs[0])\nax0.imshow(cv2.cvtColor(s, cv2.COLOR_BGR2RGB))\nax0.set_title('Original', fontsize=9, fontweight='bold'); ax0.axis('off')\nfor i, a in enumerate(aug_list):\n    ax = fig.add_subplot(gs[i+1])\n    ax.imshow(cv2.cvtColor(a, cv2.COLOR_BGR2RGB))\n    ax.set_title(f'Aug {i+1}', fontsize=9); ax.axis('off')\nfig.suptitle('Augmentation — Flip / Rotate / Zoom / Brightness',\n             fontsize=12, fontweight='bold')\nplt.tight_layout(); plt.show()\nprint('✅ Augmentation function ready')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-11T15:04:51.857970Z","iopub.execute_input":"2026-03-11T15:04:51.858252Z","iopub.status.idle":"2026-03-11T15:04:52.600464Z","shell.execute_reply.started":"2026-03-11T15:04:51.858207Z","shell.execute_reply":"2026-03-11T15:04:52.599551Z"}},"outputs":[],"execution_count":null},{"id":"f42f326c-44e8-4989-bcbe-3a3cc1384ed0","cell_type":"markdown","source":"## ⚖️ Step 6 — Balance, Augment & Write Directly to final_split\n> Augmented images are written **straight into** `final_split/train` or `final_split/test`.\n> The preprocessed temp folder is **deleted immediately** after this step.","metadata":{}},{"id":"bda9a767-c36c-44f1-bed9-38d02e0f165f","cell_type":"code","source":"# Create final_split folder structure first\nfor sp in ('train', 'test'):\n    for name in CLASS_NAMES.values():\n        os.makedirs(OUT_SPLIT + '/' + sp + '/' + name, exist_ok=True)\n\nsplit_records = []\nlock          = threading.Lock()\n\ndef write_augmented(args):\n    src_paths, split_dir, cls_name, lbl, idx = args\n    img = cv2.imread(random.choice(src_paths))\n    if img is None: return None\n    dst = split_dir + '/aug_' + f'{idx:06d}_' + os.path.basename(random.choice(src_paths))\n    cv2.imwrite(dst, augment(img))\n    sp  = 'train' if 'train' in split_dir else 'test'\n    return {'label': lbl, 'class_name': cls_name, 'image_path': dst,\n            'augmented': True, 'split': sp}\n\nfor lbl, name in CLASS_NAMES.items():\n    src_paths = pre_df[pre_df['label'] == lbl]['image_path'].tolist()\n\n    # ── Build full balanced list for this class ────────────────────────────\n    all_paths = src_paths.copy()\n    while len(all_paths) < TARGET_PER_CLASS:\n        all_paths.append(random.choice(src_paths))   # mark for augmentation\n    all_paths = all_paths[:TARGET_PER_CLASS]\n\n    # ── Stratified split for this class ───────────────────────────────────\n    n_train = int(len(all_paths) * TRAIN_RATIO)\n    random.shuffle(all_paths)\n    train_paths = all_paths[:n_train]\n    test_paths  = all_paths[n_train:]\n\n    for split_name, paths in [('train', train_paths), ('test', test_paths)]:\n        split_dir  = OUT_SPLIT + '/' + split_name + '/' + name\n        orig_paths = [p for p in paths if p in set(src_paths)]\n        aug_paths  = [p for p in paths if p not in set(src_paths)]\n\n        # Copy original preprocessed images\n        for src in tqdm(orig_paths, desc=f'{split_name} {name} (orig)', leave=False):\n            dst = split_dir + '/' + os.path.basename(src)\n            shutil.copy2(src, dst)\n            split_records.append({'label': lbl, 'class_name': name,\n                                  'image_path': dst, 'augmented': False,\n                                  'split': split_name})\n\n        # Generate augmented images in parallel\n        if aug_paths:\n            args_list = [(src_paths, split_dir, name, lbl, i)\n                         for i, _ in enumerate(aug_paths)]\n            with ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:\n                futures = {executor.submit(write_augmented, a): a for a in args_list}\n                pbar    = tqdm(as_completed(futures), total=len(futures),\n                               desc=f'{split_name} {name} (aug)', leave=False)\n                for future in pbar:\n                    result = future.result()\n                    if result:\n                        with lock:\n                            split_records.append(result)\n\n    print(f'  Stage {lbl}  {name:20s}: {len(train_paths)} train | {len(test_paths)} test')\n\nsplit_df = pd.DataFrame(split_records)\nsplit_df[split_df['split']=='train'].to_csv(OUT_SPLIT + '/train_manifest.csv', index=False)\nsplit_df[split_df['split']=='test'].to_csv( OUT_SPLIT + '/test_manifest.csv',  index=False)\n\ntrain_total = (split_df['split']=='train').sum()\ntest_total  = (split_df['split']=='test').sum()\nprint(f'\\n✅  Train: {train_total}  |  Test: {test_total}')\ndisk_free()\n\n# ── Delete preprocessed temp folder to free disk space ────────────────────\nprint('\\n🗑️  Deleting preprocessed temp folder to free disk...')\nshutil.rmtree(OUT_PRE)\ndisk_free()\nprint('✅  Temp folder deleted')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-11T15:04:52.601957Z","iopub.execute_input":"2026-03-11T15:04:52.602510Z","iopub.status.idle":"2026-03-11T15:05:45.749296Z","shell.execute_reply.started":"2026-03-11T15:04:52.602473Z","shell.execute_reply":"2026-03-11T15:05:45.748452Z"}},"outputs":[],"execution_count":null},{"id":"9582754b-1d53-47c1-8cb2-fba049b3db71","cell_type":"code","source":"# Before vs After chart\nfig, axes = plt.subplots(1, 2, figsize=(16, 4))\nfor ax, (df_, title) in zip(axes, [\n        (combined,  'Before Augmentation'),\n        (split_df,  'After Augmentation (Balanced)')]):\n    v    = [(df_['label'] == i).sum() for i in range(5)]\n    bars = ax.bar(CLASS_NAMES.values(), v, color=COLORS, edgecolor='black', linewidth=0.7)\n    ax.bar_label(bars, padding=4, fontweight='bold', fontsize=9)\n    ax.set_title(title, fontsize=12, fontweight='bold')\n    ax.set_ylim(0, max(v) * 1.15)\n    ax.set_ylabel('Count'); ax.set_xlabel('DR Stage')\n    plt.sca(ax); plt.xticks(rotation=15, ha='right')\nplt.suptitle('Class Distribution: Before vs After Augmentation',\n             fontsize=13, fontweight='bold')\nplt.tight_layout(); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-11T15:05:45.750569Z","iopub.execute_input":"2026-03-11T15:05:45.750947Z","iopub.status.idle":"2026-03-11T15:05:46.074155Z","shell.execute_reply.started":"2026-03-11T15:05:45.750900Z","shell.execute_reply":"2026-03-11T15:05:46.073479Z"}},"outputs":[],"execution_count":null},{"id":"705e02aa-71d7-4941-a48f-30af91f80a8c","cell_type":"markdown","source":"## 🔁 Step 7 — Data Generators","metadata":{}},{"id":"6a3a2c59-2505-4642-8f55-25483b6ed023","cell_type":"code","source":"datagen = ImageDataGenerator(rescale=1.0 / 255)\n\ntrain_gen = datagen.flow_from_directory(\n    OUT_SPLIT + '/train',\n    target_size = IMG_SIZE,\n    batch_size  = BATCH_SIZE,\n    class_mode  = 'categorical',\n    shuffle     = True,\n    seed        = SEED\n)\n\ntest_gen = datagen.flow_from_directory(\n    OUT_SPLIT + '/test',\n    target_size = IMG_SIZE,\n    batch_size  = BATCH_SIZE,\n    class_mode  = 'categorical',\n    shuffle     = False\n)\n\nprint(f'Class indices : {train_gen.class_indices}')\nprint(f'Train batches : {len(train_gen)}')\nprint(f'Test  batches : {len(test_gen)}')\ndisk_free()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-11T15:05:46.075221Z","iopub.execute_input":"2026-03-11T15:05:46.075660Z","iopub.status.idle":"2026-03-11T15:05:49.303207Z","shell.execute_reply.started":"2026-03-11T15:05:46.075623Z","shell.execute_reply":"2026-03-11T15:05:49.302398Z"}},"outputs":[],"execution_count":null},{"id":"a8fa9410-3f55-46aa-8533-16d4da1f438e","cell_type":"markdown","source":"## 🏗️ Step 8 — Build Fine-Tuned ResNet50","metadata":{}},{"id":"05ae683d-fa67-475e-9dd7-989ba3717211","cell_type":"code","source":"base_model = ResNet50(weights='imagenet', include_top=False, input_shape=INPUT_SHAPE)\nfor layer in base_model.layers:\n    layer.trainable = False\n\nx      = base_model.output\nx      = GlobalAveragePooling2D()(x)\nx      = BatchNormalization()(x)\nx      = Dense(1024, activation='relu')(x)\nx      = BatchNormalization()(x)\nx      = Dense(512,  activation='relu')(x)\nx      = Dropout(DROPOUT)(x)\noutput = Dense(NUM_CLASSES, activation='softmax')(x)\n\nmodel  = Model(inputs=base_model.input, outputs=output)\n\nmodel.compile(\n    optimizer = SGD(learning_rate=LR, momentum=MOMENTUM),\n    loss      = 'categorical_crossentropy',\n    metrics   = ['accuracy']\n)\n\ntotal     = model.count_params()\ntrainable = sum([tf.size(w).numpy() for w in model.trainable_weights])\nprint(f'Total params     : {total:,}')\nprint(f'Trainable params : {trainable:,}  (top layers only)')\nprint(f'Frozen params    : {total - trainable:,}  (ResNet50 base)')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-11T15:05:49.304513Z","iopub.execute_input":"2026-03-11T15:05:49.305100Z","iopub.status.idle":"2026-03-11T15:05:55.652872Z","shell.execute_reply.started":"2026-03-11T15:05:49.305072Z","shell.execute_reply":"2026-03-11T15:05:55.652121Z"}},"outputs":[],"execution_count":null},{"id":"28615ab2-5923-469d-9660-1c5c3729cab8","cell_type":"markdown","source":"## 🔥 Step 9 — Phase 1: Warm-Up Top Layers (5 epochs)","metadata":{}},{"id":"ac182250-8d84-4de8-ad03-5252c03b2e3e","cell_type":"code","source":"callbacks_p1 = [\n    EarlyStopping(monitor='val_accuracy', patience=3,\n                  restore_best_weights=True, verbose=1),\n    ModelCheckpoint(MODEL_DIR + '/resnet50_phase1_best.keras',\n                    monitor='val_accuracy', save_best_only=True, verbose=1),\n    CSVLogger(MODEL_DIR + '/phase1_log.csv')\n]\n\nprint('🔥 Phase 1: Training top layers only...')\nhistory_p1 = model.fit(\n    train_gen,\n    epochs          = 5,\n    validation_data = test_gen,\n    callbacks       = callbacks_p1,\n    verbose         = 1\n)\nprint('✅ Phase 1 complete')\ndisk_free()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-11T15:05:55.653912Z","iopub.execute_input":"2026-03-11T15:05:55.654279Z","iopub.status.idle":"2026-03-11T15:12:50.861578Z","shell.execute_reply.started":"2026-03-11T15:05:55.654247Z","shell.execute_reply":"2026-03-11T15:12:50.860721Z"}},"outputs":[],"execution_count":null},{"id":"0879782b-eaca-48a5-80a7-ee0e72b735ae","cell_type":"markdown","source":"## 🧊 Step 10 — Phase 2: Unfreeze Last 30 Layers & Fine-Tune","metadata":{}},{"id":"65222053-062e-4cc7-be92-64365f7d9c4a","cell_type":"code","source":"for layer in base_model.layers[-30:]:\n    layer.trainable = True\n\nmodel.compile(\n    optimizer = SGD(learning_rate=LR / 10, momentum=MOMENTUM),\n    loss      = 'categorical_crossentropy',\n    metrics   = ['accuracy']\n)\n\ntrainable_now = sum([tf.size(w).numpy() for w in model.trainable_weights])\nprint(f'Trainable params after unfreeze: {trainable_now:,}')\n\ncallbacks_p2 = [\n    EarlyStopping(monitor='val_accuracy', patience=7,\n                  restore_best_weights=True, verbose=1),\n    ModelCheckpoint(MODEL_DIR + '/resnet50_finetuned_best.keras',\n                    monitor='val_accuracy', save_best_only=True, verbose=1),\n    ReduceLROnPlateau(monitor='val_loss', factor=0.5,\n                      patience=3, min_lr=1e-7, verbose=1),\n    CSVLogger(MODEL_DIR + '/phase2_log.csv')\n]\n\nprint('\\n🧊 Phase 2: Fine-tuning last 30 layers...')\nhistory_p2 = model.fit(\n    train_gen,\n    epochs          = EPOCHS,\n    validation_data = test_gen,\n    callbacks       = callbacks_p2,\n    verbose         = 1\n)\nprint('✅ Phase 2 complete')\ndisk_free()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-11T15:12:50.862599Z","iopub.execute_input":"2026-03-11T15:12:50.862912Z","execution_failed":"2026-03-11T16:56:04.522Z"}},"outputs":[],"execution_count":null},{"id":"75b0fa1e-855e-4d1e-b2af-e950e85b8c45","cell_type":"markdown","source":"## 📈 Step 11 — Plot Training Curves","metadata":{}},{"id":"93ee8311-80d0-4c1a-b989-bfa880d64674","cell_type":"code","source":"acc      = history_p1.history['accuracy']     + history_p2.history['accuracy']\nval_acc  = history_p1.history['val_accuracy'] + history_p2.history['val_accuracy']\nloss     = history_p1.history['loss']         + history_p2.history['loss']\nval_loss = history_p1.history['val_loss']     + history_p2.history['val_loss']\nep_range = range(1, len(acc) + 1)\np1_end   = len(history_p1.history['accuracy'])\n\nfig, axes = plt.subplots(1, 2, figsize=(16, 5))\n\naxes[0].plot(ep_range, acc,     label='Train Accuracy',      color='#2196F3', linewidth=2)\naxes[0].plot(ep_range, val_acc, label='Validation Accuracy', color='#4CAF50', linewidth=2)\naxes[0].axvline(p1_end, color='gray', linestyle='--', alpha=0.7, label='Fine-tune starts')\naxes[0].set_title('ResNet50 — Accuracy', fontsize=13, fontweight='bold')\naxes[0].set_xlabel('Epoch'); axes[0].set_ylabel('Accuracy')\naxes[0].legend(); axes[0].grid(alpha=0.3)\n\naxes[1].plot(ep_range, loss,     label='Train Loss',      color='#F44336', linewidth=2)\naxes[1].plot(ep_range, val_loss, label='Validation Loss', color='#FF9800', linewidth=2)\naxes[1].axvline(p1_end, color='gray', linestyle='--', alpha=0.7, label='Fine-tune starts')\naxes[1].set_title('ResNet50 — Loss', fontsize=13, fontweight='bold')\naxes[1].set_xlabel('Epoch'); axes[1].set_ylabel('Loss')\naxes[1].legend(); axes[1].grid(alpha=0.3)\n\nplt.suptitle('Fine-Tuned ResNet50 Training History', fontsize=14, fontweight='bold')\nplt.tight_layout()\nplt.savefig(MODEL_DIR + '/training_curves.png', dpi=150, bbox_inches='tight')\nplt.show()","metadata":{"trusted":true,"execution":{"execution_failed":"2026-03-11T16:56:04.531Z"}},"outputs":[],"execution_count":null},{"id":"fa021cf6-4f41-4fc9-8307-2aff5e4263b3","cell_type":"markdown","source":"## 🧪 Step 12 — Evaluate on Test Set","metadata":{}},{"id":"881047e2-b2ce-4203-a29a-c4d907e6cb87","cell_type":"code","source":"test_loss, test_acc = model.evaluate(test_gen, verbose=1)\nprint(f'\\n📊 Test Accuracy : {test_acc * 100:.2f}%')\nprint(f'📊 Test Loss     : {test_loss:.4f}')","metadata":{"trusted":true,"execution":{"execution_failed":"2026-03-11T16:56:04.532Z"}},"outputs":[],"execution_count":null},{"id":"59fad7fd-eb42-46df-a597-fef049b9c705","cell_type":"markdown","source":"## 🔢 Step 13 — Confusion Matrix & Classification Report","metadata":{}},{"id":"0abf0cbd-ed23-4b9c-8ea8-9b3ef62ee80f","cell_type":"code","source":"test_gen.reset()\ny_pred_probs = model.predict(test_gen, verbose=1)\ny_pred       = np.argmax(y_pred_probs, axis=1)\ny_true       = test_gen.classes\n\nprint('\\n' + '='*55)\nprint('  CLASSIFICATION REPORT')\nprint('='*55)\nprint(classification_report(y_true, y_pred, target_names=list(CLASS_NAMES.values())))\n\ncm = confusion_matrix(y_true, y_pred)\nfig, ax = plt.subplots(figsize=(9, 7))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues',\n            xticklabels=CLASS_NAMES.values(),\n            yticklabels=CLASS_NAMES.values(),\n            linewidths=0.5, ax=ax)\nax.set_title('ResNet50 — Confusion Matrix', fontsize=14, fontweight='bold', pad=12)\nax.set_ylabel('Actual Label', fontsize=11)\nax.set_xlabel('Predicted Label', fontsize=11)\nplt.xticks(rotation=20, ha='right'); plt.yticks(rotation=0)\nplt.tight_layout()\nplt.savefig(MODEL_DIR + '/confusion_matrix.png', dpi=150, bbox_inches='tight')\nplt.show()","metadata":{"trusted":true,"execution":{"execution_failed":"2026-03-11T16:56:04.533Z"}},"outputs":[],"execution_count":null},{"id":"397d7c01-0cbb-44cc-b393-2d5e7989066a","cell_type":"markdown","source":"## 💾 Step 14 — Save Model & Final Summary","metadata":{}},{"id":"22a75f35-357d-40de-b83e-f1be0db30153","cell_type":"code","source":"model.save(MODEL_DIR + '/resnet50_finetuned_final.keras')\nprint(f'✅ Model saved → {MODEL_DIR}/resnet50_finetuned_final.keras')\n\nbest_val_acc = max(val_acc)\nbest_epoch   = val_acc.index(best_val_acc) + 1\n\nprint('\\n' + '='*55)\nprint('  FINAL RESULTS — Fine-Tuned ResNet50')\nprint('='*55)\nprint(f'  Best Epoch           : {best_epoch}')\nprint(f'  Train Accuracy       : {acc[best_epoch-1]*100:.2f}%')\nprint(f'  Validation Accuracy  : {best_val_acc*100:.2f}%')\nprint(f'  Validation Loss      : {val_loss[best_epoch-1]:.4f}')\nprint(f'  Test Accuracy        : {test_acc*100:.2f}%')\nprint(f'  Test Loss            : {test_loss:.4f}')\nprint('='*55)\nprint('  Paper target → Train: 98.69%  |  Val: 89.21%')\nprint('='*55)\ndisk_free()","metadata":{"trusted":true,"execution":{"execution_failed":"2026-03-11T16:56:04.534Z"}},"outputs":[],"execution_count":null}]}