{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ============================================================\n# STEP 1 — ENVIRONMENT & GPU VERIFICATION\n# ============================================================\n\nimport os\nimport sys\nimport platform\nimport tensorflow as tf\n\nprint(\"=\" * 70)\nprint(\"DIABETIC RETINOPATHY — ENVIRONMENT CHECK\")\nprint(\"=\" * 70)\n\n# ------------------------------------------------------------\n# 1. Python\n# ------------------------------------------------------------\nprint(\"\\n[1] PYTHON\")\nprint(\"Python version :\", sys.version)\nprint(\"Platform       :\", platform.platform())\n\n# ------------------------------------------------------------\n# 2. TensorFlow\n# ------------------------------------------------------------\nprint(\"\\n[2] TENSORFLOW\")\nprint(\"TensorFlow version :\", tf.__version__)\n\n# ------------------------------------------------------------\n# 3. GPU detection\n# ------------------------------------------------------------\nprint(\"\\n[3] GPU\")\n\ngpus = tf.config.list_physical_devices(\"GPU\")\n\nif gpus:\n    print(\"GPU detected      : YES\")\n    print(\"Number of GPUs    :\", len(gpus))\n\n    for i, gpu in enumerate(gpus):\n        print(f\"GPU {i}             :\", gpu)\n\nelse:\n    print(\"GPU detected      : NO\")\n\n# ------------------------------------------------------------\n# 4. TensorFlow build information\n# ------------------------------------------------------------\nprint(\"\\n[4] TENSORFLOW BUILD\")\nprint(\"Built with CUDA   :\", tf.test.is_built_with_cuda())\n\n# ------------------------------------------------------------\n# 5. Kaggle directories\n# ------------------------------------------------------------\nprint(\"\\n[5] KAGGLE DIRECTORIES\")\n\nfor path in [\"/kaggle/input\", \"/kaggle/working\"]:\n    print(f\"\\n{path}\")\n\n    if os.path.exists(path):\n        items = os.listdir(path)\n        print(\"Exists            : YES\")\n        print(\"Items found       :\", len(items))\n        print(\"First few items   :\", items[:10])\n    else:\n        print(\"Exists            : NO\")\n\n# ------------------------------------------------------------\n# 6. Environment variables related to GPU/CUDA\n# ------------------------------------------------------------\nprint(\"\\n[6] CUDA ENVIRONMENT\")\n\nfor key in [\"CUDA_VISIBLE_DEVICES\", \"NVIDIA_VISIBLE_DEVICES\"]:\n    print(f\"{key} :\", os.environ.get(key, \"Not set\"))\n\n# ------------------------------------------------------------\n# FINAL STATUS\n# ------------------------------------------------------------\nprint(\"\\n\" + \"=\" * 70)\n\nif gpus and tf.test.is_built_with_cuda():\n    print(\"STATUS: ENVIRONMENT CHECK PASSED\")\n    print(\"Kaggle GPU is available to TensorFlow.\")\nelse:\n    print(\"STATUS: CHECK REQUIRED\")\n    print(\"TensorFlow GPU access is not confirmed.\")\n\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T09:52:08.600184Z","iopub.execute_input":"2026-09-03T09:52:08.600892Z","iopub.status.idle":"2026-09-03T09:52:12.907485Z","shell.execute_reply.started":"2026-09-03T09:52:08.600869Z","shell.execute_reply":"2026-09-03T09:52:12.906825Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 2 — DATASET DISCOVERY\n# ============================================================\n\nimport os\nfrom pathlib import Path\n\nINPUT_ROOT = Path(\"/kaggle/input\")\n\nprint(\"=\" * 70)\nprint(\"DATASET DISCOVERY\")\nprint(\"=\" * 70)\n\ndef show_tree(path, max_depth=3, indent=\"\"):\n    path = Path(path)\n\n    if not path.exists():\n        print(f\"{indent}[NOT FOUND] {path}\")\n        return\n\n    items = sorted(path.iterdir(), key=lambda x: (x.is_file(), x.name.lower()))\n\n    for item in items:\n        print(f\"{indent}├── {item.name}\")\n\n        if item.is_dir() and max_depth > 0:\n            show_tree(\n                item,\n                max_depth=max_depth - 1,\n                indent=indent + \"│   \"\n            )\n\nprint(\"\\nINPUT ROOT:\")\nprint(INPUT_ROOT)\n\nprint(\"\\nDIRECTORY TREE:\")\nshow_tree(INPUT_ROOT, max_depth=3)\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TOP-LEVEL DATASETS\")\nprint(\"=\" * 70)\n\nfor item in sorted(INPUT_ROOT.iterdir()):\n    if item.is_dir():\n        print(f\"\\n📁 {item.name}\")\n\n        try:\n            files = list(item.rglob(\"*\"))\n            file_count = sum(x.is_file() for x in files)\n            dir_count = sum(x.is_dir() for x in files)\n\n            print(\"   Files       :\", file_count)\n            print(\"   Directories :\", dir_count)\n\n        except Exception as e:\n            print(\"   Could not inspect:\", e)\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"DISCOVERY COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T09:52:12.908631Z","iopub.execute_input":"2026-09-03T09:52:12.909292Z","iopub.status.idle":"2026-09-03T09:52:19.275168Z","shell.execute_reply.started":"2026-09-03T09:52:12.909267Z","shell.execute_reply":"2026-09-03T09:52:19.274385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 3 — EXACT DATASET PATHS & CSV STRUCTURE\n# ============================================================\n\nfrom pathlib import Path\nimport pandas as pd\nimport os\n\nROOT = Path(\"/kaggle/input\")\n\nprint(\"=\" * 70)\nprint(\"STEP 3 — DATASET IDENTIFICATION\")\nprint(\"=\" * 70)\n\n# ------------------------------------------------------------\n# 1. Find all CSV files\n# ------------------------------------------------------------\nprint(\"\\n[1] CSV FILES\")\nprint(\"-\" * 70)\n\ncsv_files = sorted(ROOT.rglob(\"*.csv\"))\n\nfor i, path in enumerate(csv_files, 1):\n    size_mb = path.stat().st_size / (1024 ** 2)\n    print(f\"{i:02d}. {path}\")\n    print(f\"    Size: {size_mb:.2f} MB\")\n\n# ------------------------------------------------------------\n# 2. Find image directories\n# ------------------------------------------------------------\nprint(\"\\n[2] IMAGE DIRECTORIES\")\nprint(\"-\" * 70)\n\nimage_extensions = {\".png\", \".jpg\", \".jpeg\", \".bmp\", \".tif\", \".tiff\"}\n\nfor directory in sorted(ROOT.rglob(\"*\")):\n    if directory.is_dir():\n\n        try:\n            image_count = sum(\n                1 for f in directory.iterdir()\n                if f.is_file() and f.suffix.lower() in image_extensions\n            )\n\n            if image_count > 0:\n                print(f\"{directory}\")\n                print(f\"    Images directly inside: {image_count}\")\n\n        except PermissionError:\n            pass\n\n# ------------------------------------------------------------\n# 3. Inspect CSV schemas\n# ------------------------------------------------------------\nprint(\"\\n[3] CSV SCHEMA\")\nprint(\"-\" * 70)\n\nfor csv_path in csv_files:\n\n    try:\n        df = pd.read_csv(csv_path)\n\n        print(f\"\\n📄 {csv_path}\")\n        print(\"Shape   :\", df.shape)\n        print(\"Columns :\", list(df.columns))\n\n        print(\"\\nFirst 3 rows:\")\n        print(df.head(3).to_string(index=False))\n\n    except Exception as e:\n        print(f\"\\nCould not read {csv_path}\")\n        print(\"Error:\", e)\n\n# ------------------------------------------------------------\n# 4. Identify likely APTOS files\n# ------------------------------------------------------------\nprint(\"\\n[4] POSSIBLE APTOS DATASET\")\nprint(\"-\" * 70)\n\nfor path in csv_files:\n\n    name = path.name.lower()\n\n    if name in {\n        \"train.csv\",\n        \"test.csv\",\n        \"sample_submission.csv\"\n    }:\n        print(path)\n\n# ------------------------------------------------------------\n# 5. Identify DRIVE-related folders\n# ------------------------------------------------------------\nprint(\"\\n[5] DRIVE / DISEASE GRADING DATASET\")\nprint(\"-\" * 70)\n\nkeywords = [\n    \"segmentation\",\n    \"disease grading\",\n    \"localization\",\n    \"training\",\n    \"test\"\n]\n\nfor path in sorted(ROOT.rglob(\"*\")):\n\n    if path.is_dir():\n\n        name = path.name.lower()\n\n        if any(keyword in name for keyword in keywords):\n            print(path)\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3 COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T09:52:19.276981Z","iopub.execute_input":"2026-09-03T09:52:19.277376Z","iopub.status.idle":"2026-09-03T09:52:19.860129Z","shell.execute_reply.started":"2026-09-03T09:52:19.277351Z","shell.execute_reply":"2026-09-03T09:52:19.859209Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 4 — APTOS DATASET FORENSICS\n# ============================================================\n\nimport os\nimport hashlib\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom collections import Counter\n\n# ------------------------------------------------------------\n# PATHS — VERIFIED FROM STEP 3\n# ------------------------------------------------------------\n\nBASE_DIR = \"/kaggle/input/competitions/aptos2019-blindness-detection\"\n\nTRAIN_CSV = os.path.join(BASE_DIR, \"train.csv\")\nTRAIN_IMG_DIR = os.path.join(BASE_DIR, \"train_images\")\n\n# ------------------------------------------------------------\n# LOAD TRAINING DATA\n# ------------------------------------------------------------\n\ndf = pd.read_csv(TRAIN_CSV)\n\nprint(\"=\" * 70)\nprint(\"APTOS 2019 — DATASET FORENSICS\")\nprint(\"=\" * 70)\n\nprint(\"\\n[1] BASIC DATASET INFORMATION\")\nprint(\"-\" * 70)\n\nprint(\"Shape:\", df.shape)\nprint(\"Columns:\", list(df.columns))\n\nprint(\"\\nFirst 5 rows:\")\ndisplay(df.head())\n\n# ------------------------------------------------------------\n# TARGET ANALYSIS\n# ------------------------------------------------------------\n\nprint(\"\\n[2] TARGET / DIAGNOSIS ANALYSIS\")\nprint(\"-\" * 70)\n\nprint(\"Unique diagnosis values:\")\nprint(sorted(df[\"diagnosis\"].unique()))\n\nprint(\"\\nClass distribution:\")\nclass_counts = df[\"diagnosis\"].value_counts().sort_index()\n\nprint(class_counts)\n\nprint(\"\\nClass percentages:\")\nclass_percent = (class_counts / len(df) * 100).round(2)\n\nfor cls, pct in class_percent.items():\n    print(f\"Class {cls}: {class_counts[cls]:4d} images ({pct:6.2f}%)\")\n\nprint(\"\\nImbalance ratio:\")\nprint(\n    \"Largest class / smallest class =\",\n    round(class_counts.max() / class_counts.min(), 2)\n)\n\n# ------------------------------------------------------------\n# MISSING VALUES\n# ------------------------------------------------------------\n\nprint(\"\\n[3] MISSING VALUES\")\nprint(\"-\" * 70)\n\nprint(df.isnull().sum())\n\n# ------------------------------------------------------------\n# DUPLICATE IDS\n# ------------------------------------------------------------\n\nprint(\"\\n[4] DUPLICATE ID CHECK\")\nprint(\"-\" * 70)\n\nduplicate_ids = df[\"id_code\"].duplicated().sum()\n\nprint(\"Duplicate IDs:\", duplicate_ids)\n\nif duplicate_ids == 0:\n    print(\"✓ No duplicate IDs found\")\nelse:\n    print(\"⚠ Duplicate IDs detected\")\n\n# ------------------------------------------------------------\n# IMAGE EXISTENCE\n# ------------------------------------------------------------\n\nprint(\"\\n[5] IMAGE FILE EXISTENCE\")\nprint(\"-\" * 70)\n\nmissing_images = []\n\nfor image_id in df[\"id_code\"]:\n    image_path = os.path.join(TRAIN_IMG_DIR, image_id + \".png\")\n\n    if not os.path.exists(image_path):\n        missing_images.append(image_id)\n\nprint(\"Expected images:\", len(df))\nprint(\"Missing images :\", len(missing_images))\n\nif len(missing_images) == 0:\n    print(\"✓ Every training image exists\")\nelse:\n    print(\"⚠ Missing images:\")\n    print(missing_images[:20])\n\n# ------------------------------------------------------------\n# IMAGE INTEGRITY + DIMENSIONS\n# ------------------------------------------------------------\n\nprint(\"\\n[6] IMAGE INTEGRITY / DIMENSIONS\")\nprint(\"-\" * 70)\n\nimage_sizes = Counter()\ncorrupt_images = []\n\nsample_count = 0\n\nfor image_id in df[\"id_code\"]:\n\n    image_path = os.path.join(TRAIN_IMG_DIR, image_id + \".png\")\n\n    try:\n        with Image.open(image_path) as img:\n\n            # Verify image\n            img.verify()\n\n        # Reopen after verify\n        with Image.open(image_path) as img:\n\n            image_sizes[img.size] += 1\n            sample_count += 1\n\n    except Exception as e:\n        corrupt_images.append((image_id, str(e)))\n\nprint(\"Images successfully checked:\", sample_count)\nprint(\"Corrupt/unreadable images   :\", len(corrupt_images))\n\nprint(\"\\nMost common image dimensions:\")\n\nfor size, count in image_sizes.most_common(15):\n    print(f\"{size}: {count}\")\n\n# ------------------------------------------------------------\n# CHANNEL / MODE CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[7] IMAGE COLOR MODE\")\nprint(\"-\" * 70)\n\nmodes = Counter()\n\nfor image_id in df[\"id_code\"]:\n\n    image_path = os.path.join(TRAIN_IMG_DIR, image_id + \".png\")\n\n    try:\n        with Image.open(image_path) as img:\n            modes[img.mode] += 1\n\n    except:\n        pass\n\nprint(\"Image modes:\")\nfor mode, count in modes.items():\n    print(f\"{mode}: {count}\")\n\n# ------------------------------------------------------------\n# FILE SIZE FORENSICS\n# ------------------------------------------------------------\n\nprint(\"\\n[8] FILE SIZE ANALYSIS\")\nprint(\"-\" * 70)\n\nfile_sizes = []\n\nfor image_id in df[\"id_code\"]:\n\n    image_path = os.path.join(TRAIN_IMG_DIR, image_id + \".png\")\n\n    if os.path.exists(image_path):\n        file_sizes.append(os.path.getsize(image_path))\n\nfile_sizes = np.array(file_sizes)\n\nprint(f\"Minimum file size : {file_sizes.min() / 1024:.2f} KB\")\nprint(f\"Maximum file size : {file_sizes.max() / 1024:.2f} KB\")\nprint(f\"Mean file size    : {file_sizes.mean() / 1024:.2f} KB\")\nprint(f\"Median file size  : {np.median(file_sizes) / 1024:.2f} KB\")\n\n# ------------------------------------------------------------\n# VERY SMALL IMAGE CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[9] POTENTIALLY SUSPICIOUS FILES\")\nprint(\"-\" * 70)\n\nsmall_files = []\n\nfor image_id in df[\"id_code\"]:\n\n    image_path = os.path.join(TRAIN_IMG_DIR, image_id + \".png\")\n\n    if os.path.exists(image_path):\n\n        size_kb = os.path.getsize(image_path) / 1024\n\n        if size_kb < 10:\n            small_files.append((image_id, round(size_kb, 2)))\n\nprint(\"Images smaller than 10 KB:\", len(small_files))\n\nif small_files:\n    print(\"First 20:\")\n    print(small_files[:20])\n\n# ------------------------------------------------------------\n# SUMMARY\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 4 FORENSICS SUMMARY\")\nprint(\"=\" * 70)\n\nprint(f\"\"\"\nTotal training samples : {len(df)}\nNumber of classes      : {df['diagnosis'].nunique()}\nMissing labels         : {df['diagnosis'].isnull().sum()}\nDuplicate IDs          : {duplicate_ids}\nMissing images         : {len(missing_images)}\nCorrupt images         : {len(corrupt_images)}\nImage modes            : {dict(modes)}\nUnique dimensions      : {len(image_sizes)}\n\"\"\")\n\nprint(\"=\" * 70)\nprint(\"STEP 4 COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T09:52:19.861060Z","iopub.execute_input":"2026-09-03T09:52:19.861363Z","iopub.status.idle":"2026-09-03T09:52:36.508791Z","shell.execute_reply.started":"2026-09-03T09:52:19.861326Z","shell.execute_reply":"2026-09-03T09:52:36.507864Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 5 — VISUAL RETINAL IMAGE INSPECTION\n# ============================================================\n\nimport os\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom PIL import Image\n\n# ------------------------------------------------------------\n# PATHS\n# ------------------------------------------------------------\n\nBASE_DIR = \"/kaggle/input/competitions/aptos2019-blindness-detection\"\n\nTRAIN_CSV = os.path.join(BASE_DIR, \"train.csv\")\nTRAIN_IMG_DIR = os.path.join(BASE_DIR, \"train_images\")\n\n# ------------------------------------------------------------\n# LOAD DATA\n# ------------------------------------------------------------\n\ndf = pd.read_csv(TRAIN_CSV)\n\n# Reproducibility\nrandom.seed(42)\nnp.random.seed(42)\n\n# ------------------------------------------------------------\n# CLASS NAMES\n# ------------------------------------------------------------\n\nCLASS_NAMES = {\n    0: \"No DR\",\n    1: \"Mild\",\n    2: \"Moderate\",\n    3: \"Severe\",\n    4: \"Proliferative DR\"\n}\n\n# ------------------------------------------------------------\n# DISPLAY 5 RANDOM IMAGES PER CLASS\n# ------------------------------------------------------------\n\nprint(\"=\" * 70)\nprint(\"STEP 5 — VISUAL RETINAL IMAGE INSPECTION\")\nprint(\"=\" * 70)\n\nfor class_id in range(5):\n\n    class_df = df[df[\"diagnosis\"] == class_id]\n\n    samples = class_df.sample(\n        n=min(5, len(class_df)),\n        random_state=42\n    )\n\n    fig, axes = plt.subplots(1, len(samples), figsize=(20, 4))\n\n    if len(samples) == 1:\n        axes = [axes]\n\n    for ax, (_, row) in zip(axes, samples.iterrows()):\n\n        image_id = row[\"id_code\"]\n\n        image_path = os.path.join(\n            TRAIN_IMG_DIR,\n            image_id + \".png\"\n        )\n\n        image = Image.open(image_path)\n\n        ax.imshow(image)\n        ax.set_title(\n            f\"Class {class_id}\\n{CLASS_NAMES[class_id]}\\n{image.size}\"\n        )\n        ax.axis(\"off\")\n\n    plt.suptitle(\n        f\"Diagnosis Class {class_id} — {CLASS_NAMES[class_id]}\",\n        fontsize=16\n    )\n\n    plt.tight_layout()\n    plt.show()\n\n# ------------------------------------------------------------\n# MIXED CLASS GRID\n# ------------------------------------------------------------\n\nprint(\"\\nGenerating mixed-class overview...\")\n\nsamples = []\n\nfor class_id in range(5):\n\n    class_df = df[df[\"diagnosis\"] == class_id]\n\n    sample = class_df.sample(\n        n=min(2, len(class_df)),\n        random_state=123\n    )\n\n    samples.extend(\n        [(class_id, row[\"id_code\"]) for _, row in sample.iterrows()]\n    )\n\nfig, axes = plt.subplots(2, 5, figsize=(20, 9))\n\nfor ax, (class_id, image_id) in zip(axes.flat, samples):\n\n    image_path = os.path.join(\n        TRAIN_IMG_DIR,\n        image_id + \".png\"\n    )\n\n    image = Image.open(image_path)\n\n    ax.imshow(image)\n\n    ax.set_title(\n        f\"Class {class_id} — {CLASS_NAMES[class_id]}\"\n    )\n\n    ax.axis(\"off\")\n\nplt.suptitle(\n    \"APTOS 2019 — Mixed Class Visual Overview\",\n    fontsize=18\n)\n\nplt.tight_layout()\nplt.show()\n\nprint(\"=\" * 70)\nprint(\"STEP 5 COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T09:52:36.509928Z","iopub.execute_input":"2026-09-03T09:52:36.510653Z","iopub.status.idle":"2026-09-03T09:52:51.882215Z","shell.execute_reply.started":"2026-09-03T09:52:36.510612Z","shell.execute_reply":"2026-09-03T09:52:51.881051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 6 — RETINAL FIELD-OF-VIEW / CROPPING ANALYSIS\n# ============================================================\n\nimport os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n# ------------------------------------------------------------\n# PATHS\n# ------------------------------------------------------------\n\nBASE_DIR = \"/kaggle/input/competitions/aptos2019-blindness-detection\"\n\nTRAIN_CSV = os.path.join(BASE_DIR, \"train.csv\")\nTRAIN_IMG_DIR = os.path.join(BASE_DIR, \"train_images\")\n\ndf = pd.read_csv(TRAIN_CSV)\n\n# ------------------------------------------------------------\n# RETINAL CROP FUNCTION\n# ------------------------------------------------------------\n\ndef crop_black_background(image, threshold=10, padding=10):\n    \"\"\"\n    Detect the non-black retinal field and crop around it.\n\n    Parameters\n    ----------\n    image : numpy array\n        RGB image\n\n    threshold : int\n        Pixel threshold used to detect retinal region.\n\n    padding : int\n        Extra pixels retained around detected region.\n    \"\"\"\n\n    # Convert RGB -> grayscale\n    gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)\n\n    # Detect non-black pixels\n    mask = gray > threshold\n\n    # Find coordinates\n    coords = np.column_stack(np.where(mask))\n\n    if coords.size == 0:\n        return image, None\n\n    y_min, x_min = coords.min(axis=0)\n    y_max, x_max = coords.max(axis=0)\n\n    # Add padding\n    y_min = max(0, y_min - padding)\n    x_min = max(0, x_min - padding)\n\n    y_max = min(image.shape[0] - 1, y_max + padding)\n    x_max = min(image.shape[1] - 1, x_max + padding)\n\n    cropped = image[y_min:y_max + 1, x_min:x_max + 1]\n\n    bbox = (x_min, y_min, x_max, y_max)\n\n    return cropped, bbox\n\n\n# ------------------------------------------------------------\n# SELECT REPRESENTATIVE IMAGES\n# ------------------------------------------------------------\n\nsamples = []\n\nfor class_id in range(5):\n\n    class_df = df[df[\"diagnosis\"] == class_id]\n\n    row = class_df.sample(\n        n=1,\n        random_state=100 + class_id\n    ).iloc[0]\n\n    samples.append(\n        (class_id, row[\"id_code\"])\n    )\n\n\n# ------------------------------------------------------------\n# DISPLAY RESULTS\n# ------------------------------------------------------------\n\nfig, axes = plt.subplots(\n    5,\n    3,\n    figsize=(15, 22)\n)\n\nfor row_idx, (class_id, image_id) in enumerate(samples):\n\n    image_path = os.path.join(\n        TRAIN_IMG_DIR,\n        image_id + \".png\"\n    )\n\n    # Read image\n    image_bgr = cv2.imread(image_path)\n\n    if image_bgr is None:\n        print(\"Could not read:\", image_path)\n        continue\n\n    image = cv2.cvtColor(\n        image_bgr,\n        cv2.COLOR_BGR2RGB\n    )\n\n    # Crop\n    cropped, bbox = crop_black_background(image)\n\n    # --------------------------------------------------------\n    # ORIGINAL\n    # --------------------------------------------------------\n\n    axes[row_idx, 0].imshow(image)\n\n    axes[row_idx, 0].set_title(\n        f\"Class {class_id} — Original\\n{image.shape[1]}×{image.shape[0]}\"\n    )\n\n    axes[row_idx, 0].axis(\"off\")\n\n    # --------------------------------------------------------\n    # BOUNDING BOX\n    # --------------------------------------------------------\n\n    boxed = image.copy()\n\n    if bbox is not None:\n\n        x_min, y_min, x_max, y_max = bbox\n\n        cv2.rectangle(\n            boxed,\n            (x_min, y_min),\n            (x_max, y_max),\n            (255, 0, 0),\n            max(2, image.shape[0] // 500)\n        )\n\n    axes[row_idx, 1].imshow(boxed)\n\n    axes[row_idx, 1].set_title(\n        \"Detected retinal region\"\n    )\n\n    axes[row_idx, 1].axis(\"off\")\n\n    # --------------------------------------------------------\n    # CROPPED\n    # --------------------------------------------------------\n\n    axes[row_idx, 2].imshow(cropped)\n\n    axes[row_idx, 2].set_title(\n        f\"Cropped\\n{cropped.shape[1]}×{cropped.shape[0]}\"\n    )\n\n    axes[row_idx, 2].axis(\"off\")\n\n\nplt.suptitle(\n    \"STEP 6 — Retinal Field-of-View Cropping Analysis\",\n    fontsize=18\n)\n\nplt.tight_layout()\nplt.show()\n\nprint(\"=\" * 70)\nprint(\"STEP 6 COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T09:52:51.883567Z","iopub.execute_input":"2026-09-03T09:52:51.883881Z","iopub.status.idle":"2026-09-03T09:52:57.365675Z","shell.execute_reply.started":"2026-09-03T09:52:51.883857Z","shell.execute_reply":"2026-09-03T09:52:57.364574Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 7 — PREPROCESSING COMPARISON\n# ============================================================\n\nimport os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n# ------------------------------------------------------------\n# PATHS\n# ------------------------------------------------------------\n\nBASE_DIR = \"/kaggle/input/competitions/aptos2019-blindness-detection\"\n\nTRAIN_CSV = os.path.join(BASE_DIR, \"train.csv\")\nTRAIN_IMG_DIR = os.path.join(BASE_DIR, \"train_images\")\n\ndf = pd.read_csv(TRAIN_CSV)\n\n# ------------------------------------------------------------\n# CROP FUNCTION\n# ------------------------------------------------------------\n\ndef crop_black_background(image, threshold=10, padding=10):\n\n    gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)\n\n    mask = gray > threshold\n\n    coords = np.column_stack(np.where(mask))\n\n    if coords.size == 0:\n        return image\n\n    y_min, x_min = coords.min(axis=0)\n    y_max, x_max = coords.max(axis=0)\n\n    y_min = max(0, y_min - padding)\n    x_min = max(0, x_min - padding)\n\n    y_max = min(image.shape[0] - 1, y_max + padding)\n    x_max = min(image.shape[1] - 1, x_max + padding)\n\n    return image[y_min:y_max + 1, x_min:x_max + 1]\n\n\n# ------------------------------------------------------------\n# MILD CONTRAST ENHANCEMENT\n# ------------------------------------------------------------\n\ndef enhance_retina(image):\n\n    # Convert RGB -> LAB\n    lab = cv2.cvtColor(image, cv2.COLOR_RGB2LAB)\n\n    l_channel, a_channel, b_channel = cv2.split(lab)\n\n    # Mild CLAHE\n    clahe = cv2.createCLAHE(\n        clipLimit=2.0,\n        tileGridSize=(8, 8)\n    )\n\n    l_enhanced = clahe.apply(l_channel)\n\n    enhanced_lab = cv2.merge(\n        [l_enhanced, a_channel, b_channel]\n    )\n\n    enhanced = cv2.cvtColor(\n        enhanced_lab,\n        cv2.COLOR_LAB2RGB\n    )\n\n    return enhanced\n\n\n# ------------------------------------------------------------\n# SELECT REPRESENTATIVE IMAGES\n# ------------------------------------------------------------\n\nsamples = []\n\nfor class_id in range(5):\n\n    class_df = df[df[\"diagnosis\"] == class_id]\n\n    row = class_df.sample(\n        n=1,\n        random_state=200 + class_id\n    ).iloc[0]\n\n    samples.append(\n        (class_id, row[\"id_code\"])\n    )\n\n\n# ------------------------------------------------------------\n# DISPLAY COMPARISON\n# ------------------------------------------------------------\n\nfig, axes = plt.subplots(\n    5,\n    3,\n    figsize=(15, 23)\n)\n\nfor row_idx, (class_id, image_id) in enumerate(samples):\n\n    image_path = os.path.join(\n        TRAIN_IMG_DIR,\n        image_id + \".png\"\n    )\n\n    image_bgr = cv2.imread(image_path)\n\n    image = cv2.cvtColor(\n        image_bgr,\n        cv2.COLOR_BGR2RGB\n    )\n\n    # A — Original\n    original = image\n\n    # B — Crop\n    cropped = crop_black_background(image)\n\n    # C — Crop + enhancement\n    enhanced = enhance_retina(cropped)\n\n    # --------------------------------------------------------\n    # ORIGINAL\n    # --------------------------------------------------------\n\n    axes[row_idx, 0].imshow(original)\n\n    axes[row_idx, 0].set_title(\n        f\"Class {class_id} — Original\"\n    )\n\n    axes[row_idx, 0].axis(\"off\")\n\n    # --------------------------------------------------------\n    # CROPPED\n    # --------------------------------------------------------\n\n    axes[row_idx, 1].imshow(cropped)\n\n    axes[row_idx, 1].set_title(\n        \"Retinal Crop\"\n    )\n\n    axes[row_idx, 1].axis(\"off\")\n\n    # --------------------------------------------------------\n    # ENHANCED\n    # --------------------------------------------------------\n\n    axes[row_idx, 2].imshow(enhanced)\n\n    axes[row_idx, 2].set_title(\n        \"Crop + Mild CLAHE\"\n    )\n\n    axes[row_idx, 2].axis(\"off\")\n\n\nplt.suptitle(\n    \"STEP 7 — Preprocessing Comparison\",\n    fontsize=18\n)\n\nplt.tight_layout()\nplt.show()\n\nprint(\"=\" * 70)\nprint(\"STEP 7 COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T09:52:57.367229Z","iopub.execute_input":"2026-09-03T09:52:57.367567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 8 — STRATIFIED TRAIN / VALIDATION SPLIT\n# ============================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\n\n# ------------------------------------------------------------\n# PATH\n# ------------------------------------------------------------\n\nBASE_DIR = \"/kaggle/input/competitions/aptos2019-blindness-detection\"\n\nTRAIN_CSV = os.path.join(BASE_DIR, \"train.csv\")\n\n# ------------------------------------------------------------\n# LOAD DATA\n# ------------------------------------------------------------\n\ndf = pd.read_csv(TRAIN_CSV)\n\n# ------------------------------------------------------------\n# CLEAN TYPES\n# ------------------------------------------------------------\n\ndf[\"id_code\"] = df[\"id_code\"].astype(str)\ndf[\"diagnosis\"] = df[\"diagnosis\"].astype(int)\n\n# ------------------------------------------------------------\n# STRATIFIED SPLIT\n# ------------------------------------------------------------\n\ntrain_df, val_df = train_test_split(\n    df,\n    test_size=0.20,\n    stratify=df[\"diagnosis\"],\n    random_state=42,\n    shuffle=True\n)\n\n# ------------------------------------------------------------\n# RESET INDEX\n# ------------------------------------------------------------\n\ntrain_df = train_df.reset_index(drop=True)\nval_df = val_df.reset_index(drop=True)\n\n# ------------------------------------------------------------\n# BASIC INFORMATION\n# ------------------------------------------------------------\n\nprint(\"=\" * 70)\nprint(\"STEP 8 — STRATIFIED TRAIN / VALIDATION SPLIT\")\nprint(\"=\" * 70)\n\nprint(\"\\n[1] DATASET SIZES\")\nprint(\"-\" * 70)\n\nprint(\"Original dataset :\", len(df))\nprint(\"Training set     :\", len(train_df))\nprint(\"Validation set   :\", len(val_df))\n\nprint(\"\\nExpected:\")\nprint(\"Training ≈ 80%\")\nprint(\"Validation ≈ 20%\")\n\n# ------------------------------------------------------------\n# CLASS DISTRIBUTION\n# ------------------------------------------------------------\n\nprint(\"\\n[2] TRAINING CLASS DISTRIBUTION\")\nprint(\"-\" * 70)\n\ntrain_counts = train_df[\"diagnosis\"].value_counts().sort_index()\n\nfor cls in range(5):\n    count = train_counts.get(cls, 0)\n    percentage = count / len(train_df) * 100\n\n    print(\n        f\"Class {cls}: {count:4d} \"\n        f\"({percentage:6.2f}%)\"\n    )\n\n# ------------------------------------------------------------\n\nprint(\"\\n[3] VALIDATION CLASS DISTRIBUTION\")\nprint(\"-\" * 70)\n\nval_counts = val_df[\"diagnosis\"].value_counts().sort_index()\n\nfor cls in range(5):\n    count = val_counts.get(cls, 0)\n    percentage = count / len(val_df) * 100\n\n    print(\n        f\"Class {cls}: {count:4d} \"\n        f\"({percentage:6.2f}%)\"\n    )\n\n# ------------------------------------------------------------\n# COMPARE ORIGINAL / TRAIN / VALIDATION\n# ------------------------------------------------------------\n\nprint(\"\\n[4] DISTRIBUTION COMPARISON\")\nprint(\"-\" * 70)\n\ncomparison = pd.DataFrame({\n    \"Original\": df[\"diagnosis\"].value_counts().sort_index(),\n    \"Train\": train_df[\"diagnosis\"].value_counts().sort_index(),\n    \"Validation\": val_df[\"diagnosis\"].value_counts().sort_index()\n})\n\ncomparison[\"Original %\"] = (\n    comparison[\"Original\"] / len(df) * 100\n).round(2)\n\ncomparison[\"Train %\"] = (\n    comparison[\"Train\"] / len(train_df) * 100\n).round(2)\n\ncomparison[\"Validation %\"] = (\n    comparison[\"Validation\"] / len(val_df) * 100\n).round(2)\n\ndisplay(comparison)\n\n# ------------------------------------------------------------\n# LEAKAGE CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[5] TRAIN / VALIDATION OVERLAP CHECK\")\nprint(\"-\" * 70)\n\ntrain_ids = set(train_df[\"id_code\"])\nval_ids = set(val_df[\"id_code\"])\n\noverlap = train_ids.intersection(val_ids)\n\nprint(\"Training IDs     :\", len(train_ids))\nprint(\"Validation IDs   :\", len(val_ids))\nprint(\"Overlapping IDs  :\", len(overlap))\n\nif len(overlap) == 0:\n    print(\"✓ NO ID OVERLAP — split is clean\")\nelse:\n    print(\"⚠ DATA LEAKAGE DETECTED\")\n\n# ------------------------------------------------------------\n# SAVE SPLITS\n# ------------------------------------------------------------\n\nWORKING_DIR = \"/kaggle/working/aptos_splits\"\n\nos.makedirs(WORKING_DIR, exist_ok=True)\n\ntrain_path = os.path.join(\n    WORKING_DIR,\n    \"train_split.csv\"\n)\n\nval_path = os.path.join(\n    WORKING_DIR,\n    \"val_split.csv\"\n)\n\ntrain_df.to_csv(train_path, index=False)\nval_df.to_csv(val_path, index=False)\n\nprint(\"\\n[6] SPLITS SAVED\")\nprint(\"-\" * 70)\n\nprint(\"Training split   :\", train_path)\nprint(\"Validation split :\", val_path)\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 8 COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 9 — EXACT IMAGE DUPLICATE / DATA LEAKAGE CHECK\n# ============================================================\n\nimport os\nimport hashlib\nimport pandas as pd\nfrom collections import defaultdict\n\n# ------------------------------------------------------------\n# PATHS\n# ------------------------------------------------------------\n\nBASE_DIR = \"/kaggle/input/competitions/aptos2019-blindness-detection\"\n\nTRAIN_IMG_DIR = os.path.join(BASE_DIR, \"train_images\")\n\nSPLIT_DIR = \"/kaggle/working/aptos_splits\"\n\nTRAIN_SPLIT = os.path.join(\n    SPLIT_DIR,\n    \"train_split.csv\"\n)\n\nVAL_SPLIT = os.path.join(\n    SPLIT_DIR,\n    \"val_split.csv\"\n)\n\n# ------------------------------------------------------------\n# LOAD SPLITS\n# ------------------------------------------------------------\n\ntrain_df = pd.read_csv(TRAIN_SPLIT)\nval_df = pd.read_csv(VAL_SPLIT)\n\n# ------------------------------------------------------------\n# SHA-256 HASH FUNCTION\n# ------------------------------------------------------------\n\ndef file_hash(filepath, chunk_size=1024 * 1024):\n\n    sha256 = hashlib.sha256()\n\n    with open(filepath, \"rb\") as f:\n\n        while True:\n\n            data = f.read(chunk_size)\n\n            if not data:\n                break\n\n            sha256.update(data)\n\n    return sha256.hexdigest()\n\n\n# ------------------------------------------------------------\n# HASH ALL TRAINING IMAGES\n# ------------------------------------------------------------\n\nprint(\"=\" * 70)\nprint(\"STEP 9 — EXACT IMAGE DUPLICATE / DATA LEAKAGE CHECK\")\nprint(\"=\" * 70)\n\nprint(\"\\n[1] HASHING ALL TRAINING IMAGES\")\nprint(\"-\" * 70)\n\nall_hashes = defaultdict(list)\n\ntotal_images = len(train_df) + len(val_df)\nprocessed = 0\n\nfor _, row in pd.concat(\n    [train_df, val_df],\n    ignore_index=True\n).iterrows():\n\n    image_id = row[\"id_code\"]\n\n    image_path = os.path.join(\n        TRAIN_IMG_DIR,\n        image_id + \".png\"\n    )\n\n    image_hash = file_hash(image_path)\n\n    all_hashes[image_hash].append({\n        \"id_code\": image_id,\n        \"split\": (\n            \"train\"\n            if image_id in set(train_df[\"id_code\"])\n            else \"validation\"\n        ),\n        \"diagnosis\": int(row[\"diagnosis\"])\n    })\n\n    processed += 1\n\n    if processed % 500 == 0:\n        print(f\"Processed: {processed}/{total_images}\")\n\n\n# ------------------------------------------------------------\n# FIND DUPLICATES\n# ------------------------------------------------------------\n\nduplicate_groups = {\n    h: records\n    for h, records in all_hashes.items()\n    if len(records) > 1\n}\n\nprint(\"\\n[2] EXACT DUPLICATE ANALYSIS\")\nprint(\"-\" * 70)\n\nprint(\"Total images hashed :\", total_images)\nprint(\"Unique image hashes :\", len(all_hashes))\nprint(\"Duplicate groups    :\", len(duplicate_groups))\n\nduplicate_image_count = sum(\n    len(records)\n    for records in duplicate_groups.values()\n)\n\nprint(\"Images involved in duplicate groups:\",\n      duplicate_image_count)\n\n\n# ------------------------------------------------------------\n# CHECK CROSS-SPLIT DUPLICATES\n# ------------------------------------------------------------\n\ncross_split_duplicates = {}\n\nfor image_hash, records in duplicate_groups.items():\n\n    splits = set(\n        record[\"split\"]\n        for record in records\n    )\n\n    if len(splits) > 1:\n        cross_split_duplicates[image_hash] = records\n\n\nprint(\"\\n[3] CROSS-SPLIT LEAKAGE CHECK\")\nprint(\"-\" * 70)\n\nprint(\n    \"Duplicate groups crossing train/validation:\",\n    len(cross_split_duplicates)\n)\n\nif len(cross_split_duplicates) == 0:\n\n    print(\n        \"✓ NO EXACT IMAGE DUPLICATE LEAKAGE DETECTED\"\n    )\n\nelse:\n\n    print(\n        \"⚠ EXACT DUPLICATE LEAKAGE DETECTED\"\n    )\n\n    print(\"\\nFirst duplicate groups:\")\n\n    shown = 0\n\n    for image_hash, records in cross_split_duplicates.items():\n\n        print(\"\\nHash:\", image_hash)\n\n        for record in records:\n            print(\n                f\"  {record['id_code']} | \"\n                f\"{record['split']} | \"\n                f\"class={record['diagnosis']}\"\n            )\n\n        shown += 1\n\n        if shown >= 10:\n            break\n\n\n# ------------------------------------------------------------\n# SAME-SPLIT DUPLICATES\n# ------------------------------------------------------------\n\nprint(\"\\n[4] SAME-SPLIT DUPLICATES\")\nprint(\"-\" * 70)\n\nsame_split_groups = {}\n\nfor image_hash, records in duplicate_groups.items():\n\n    splits = set(\n        record[\"split\"]\n        for record in records\n    )\n\n    if len(splits) == 1:\n        same_split_groups[image_hash] = records\n\nprint(\n    \"Duplicate groups contained within one split:\",\n    len(same_split_groups)\n)\n\n\n# ------------------------------------------------------------\n# CLASS CONSISTENCY FOR DUPLICATES\n# ------------------------------------------------------------\n\nprint(\"\\n[5] DUPLICATE LABEL CONSISTENCY\")\nprint(\"-\" * 70)\n\ninconsistent_label_groups = {}\n\nfor image_hash, records in duplicate_groups.items():\n\n    labels = set(\n        record[\"diagnosis\"]\n        for record in records\n    )\n\n    if len(labels) > 1:\n        inconsistent_label_groups[image_hash] = records\n\nprint(\n    \"Duplicate groups with conflicting labels:\",\n    len(inconsistent_label_groups)\n)\n\nif len(inconsistent_label_groups) > 0:\n\n    print(\"\\n⚠ Conflicting duplicate labels found.\")\n\n    shown = 0\n\n    for image_hash, records in inconsistent_label_groups.items():\n\n        print(\"\\nHash:\", image_hash)\n\n        for record in records:\n            print(\n                f\"  {record['id_code']} | \"\n                f\"{record['split']} | \"\n                f\"class={record['diagnosis']}\"\n            )\n\n        shown += 1\n\n        if shown >= 10:\n            break\n\n\n# ------------------------------------------------------------\n# FINAL VERDICT\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 9 FINAL VERDICT\")\nprint(\"=\" * 70)\n\nif len(cross_split_duplicates) == 0:\n\n    print(\n        \"✓ No exact duplicate images cross the train/validation boundary.\"\n    )\n\nelse:\n\n    print(\n        \"⚠ Potential data leakage exists and must be resolved.\"\n    )\n\nprint(\"=\" * 70)\nprint(\"STEP 9 COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 9A — REBUILD LEAKAGE-SAFE GROUPED TRAIN/VALIDATION SPLIT\n# ============================================================\n\nimport os\nimport hashlib\nimport numpy as np\nimport pandas as pd\n\nfrom pathlib import Path\nfrom sklearn.model_selection import StratifiedGroupKFold\n\n\n# ============================================================\n# 1. PATHS\n# ============================================================\n\nBASE_DIR = Path(\n    \"/kaggle/input/competitions/aptos2019-blindness-detection\"\n)\n\nTRAIN_CSV = BASE_DIR / \"train.csv\"\nTRAIN_IMG_DIR = BASE_DIR / \"train_images\"\n\nOUTPUT_DIR = Path(\"/kaggle/working/aptos_splits\")\nOUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n\n\n# ============================================================\n# 2. LOAD DATA\n# ============================================================\n\ndf = pd.read_csv(TRAIN_CSV).copy()\n\nprint(\"=\" * 70)\nprint(\"STEP 9A — REBUILD LEAKAGE-SAFE GROUPED SPLIT\")\nprint(\"=\" * 70)\n\nprint(\"\\n[1] DATASET\")\nprint(\"-\" * 70)\n\nprint(f\"Total samples : {len(df)}\")\nprint(f\"Classes       : {sorted(df['diagnosis'].unique())}\")\n\n\n# ============================================================\n# 3. SHA-256 HASH FUNCTION\n# ============================================================\n\ndef sha256_file(path, chunk_size=1024 * 1024):\n    sha = hashlib.sha256()\n\n    with open(path, \"rb\") as f:\n        while True:\n            chunk = f.read(chunk_size)\n\n            if not chunk:\n                break\n\n            sha.update(chunk)\n\n    return sha.hexdigest()\n\n\n# ============================================================\n# 4. HASH EVERY IMAGE\n# ============================================================\n\nprint(\"\\n[2] HASHING ALL TRAINING IMAGES\")\nprint(\"-\" * 70)\n\nhashes = []\n\nfor i, image_id in enumerate(df[\"id_code\"], start=1):\n\n    image_path = TRAIN_IMG_DIR / f\"{image_id}.png\"\n\n    if not image_path.exists():\n        raise FileNotFoundError(\n            f\"Image not found: {image_path}\"\n        )\n\n    hashes.append(sha256_file(image_path))\n\n    if i % 500 == 0 or i == len(df):\n        print(f\"Processed: {i}/{len(df)}\")\n\n\ndf[\"image_hash\"] = hashes\n\n\n# ============================================================\n# 5. ANALYZE HASH GROUPS\n# ============================================================\n\nprint(\"\\n[3] EXACT-DUPLICATE GROUP ANALYSIS\")\nprint(\"-\" * 70)\n\ngroup_info = (\n    df.groupby(\"image_hash\")\n      .agg(\n          group_size=(\"id_code\", \"size\"),\n          number_of_labels=(\"diagnosis\", \"nunique\")\n      )\n      .reset_index()\n)\n\nduplicate_groups = group_info[\n    group_info[\"group_size\"] > 1\n].copy()\n\nconflicting_groups = duplicate_groups[\n    duplicate_groups[\"number_of_labels\"] > 1\n].copy()\n\nprint(f\"Unique image hashes       : {len(group_info)}\")\nprint(f\"Duplicate groups          : {len(duplicate_groups)}\")\nprint(\n    f\"Conflicting duplicate groups : \"\n    f\"{len(conflicting_groups)}\"\n)\nprint(\n    f\"Images in conflicting groups : \"\n    f\"{conflicting_groups['group_size'].sum()}\"\n)\n\n\n# ============================================================\n# 6. REMOVE CONFLICTING EXACT DUPLICATES\n# ============================================================\n\nconflicting_hashes = set(\n    conflicting_groups[\"image_hash\"]\n)\n\nclean_df = df[\n    ~df[\"image_hash\"].isin(conflicting_hashes)\n].copy()\n\nremoved_count = len(df) - len(clean_df)\n\nprint(\"\\n[4] CONFLICTING GROUP REMOVAL\")\nprint(\"-\" * 70)\n\nprint(f\"Original samples       : {len(df)}\")\nprint(f\"Removed samples        : {removed_count}\")\nprint(f\"Remaining samples      : {len(clean_df)}\")\n\n\n# ============================================================\n# 7. CHECK LABEL DISTRIBUTION AFTER REMOVAL\n# ============================================================\n\nprint(\"\\n[5] CLEAN DATASET CLASS DISTRIBUTION\")\nprint(\"-\" * 70)\n\nclass_counts = (\n    clean_df[\"diagnosis\"]\n    .value_counts()\n    .sort_index()\n)\n\nclass_percent = (\n    clean_df[\"diagnosis\"]\n    .value_counts(normalize=True)\n    .sort_index() * 100\n)\n\ndistribution = pd.DataFrame({\n    \"count\": class_counts,\n    \"percentage\": class_percent.round(2)\n})\n\nprint(distribution)\n\n\n# ============================================================\n# 8. GROUP-AWARE STRATIFIED SPLIT\n# ============================================================\n#\n# Every exact duplicate image has the same hash.\n#\n# The hash is therefore used as the GROUP.\n#\n# This guarantees that identical images cannot be placed\n# in both training and validation.\n#\n# Conflicting groups were already removed above.\n# ============================================================\n\nprint(\"\\n[6] CREATING GROUP-AWARE STRATIFIED SPLIT\")\nprint(\"-\" * 70)\n\nX = clean_df.index.values\ny = clean_df[\"diagnosis\"].values\ngroups = clean_df[\"image_hash\"].values\n\nsgkf = StratifiedGroupKFold(\n    n_splits=5,\n    shuffle=True,\n    random_state=42\n)\n\n\n# Find the fold closest to a 20% validation split.\ncandidate_splits = []\n\nfor fold, (train_idx, val_idx) in enumerate(\n    sgkf.split(X, y, groups),\n    start=1\n):\n\n    val_fraction = len(val_idx) / len(clean_df)\n\n    candidate_splits.append({\n        \"fold\": fold,\n        \"train_idx\": train_idx,\n        \"val_idx\": val_idx,\n        \"val_fraction\": val_fraction\n    })\n\nbest_split = min(\n    candidate_splits,\n    key=lambda x: abs(x[\"val_fraction\"] - 0.20)\n)\n\ntrain_idx = best_split[\"train_idx\"]\nval_idx = best_split[\"val_idx\"]\n\ntrain_df = clean_df.iloc[train_idx].copy()\nval_df = clean_df.iloc[val_idx].copy()\n\n\n# ============================================================\n# 9. REMOVE INTERNAL FORENSIC COLUMN BEFORE SAVING\n# ============================================================\n\ntrain_save = train_df.drop(\n    columns=[\"image_hash\"]\n).copy()\n\nval_save = val_df.drop(\n    columns=[\"image_hash\"]\n).copy()\n\n\n# ============================================================\n# 10. SAVE NEW SPLITS\n# ============================================================\n\nTRAIN_SPLIT_PATH = OUTPUT_DIR / \"train_split.csv\"\nVAL_SPLIT_PATH = OUTPUT_DIR / \"val_split.csv\"\n\ntrain_save.to_csv(\n    TRAIN_SPLIT_PATH,\n    index=False\n)\n\nval_save.to_csv(\n    VAL_SPLIT_PATH,\n    index=False\n)\n\n\n# ============================================================\n# 11. BASIC SPLIT STATISTICS\n# ============================================================\n\nprint(\"\\n[7] NEW SPLIT SIZES\")\nprint(\"-\" * 70)\n\nprint(f\"Selected fold        : {best_split['fold']}\")\nprint(f\"Training samples     : {len(train_save)}\")\nprint(f\"Validation samples   : {len(val_save)}\")\nprint(\n    f\"Validation fraction  : \"\n    f\"{len(val_save) / len(clean_df) * 100:.2f}%\"\n)\n\n\n# ============================================================\n# 12. CLASS DISTRIBUTION\n# ============================================================\n\nprint(\"\\n[8] TRAINING CLASS DISTRIBUTION\")\nprint(\"-\" * 70)\n\ntrain_distribution = (\n    train_save[\"diagnosis\"]\n    .value_counts()\n    .sort_index()\n)\n\nprint(train_distribution)\n\n\nprint(\"\\n[9] VALIDATION CLASS DISTRIBUTION\")\nprint(\"-\" * 70)\n\nval_distribution = (\n    val_save[\"diagnosis\"]\n    .value_counts()\n    .sort_index()\n)\n\nprint(val_distribution)\n\n\n# ============================================================\n# 13. EXACT DUPLICATE CROSS-SPLIT VERIFICATION\n# ============================================================\n\nprint(\"\\n[10] FINAL EXACT-DUPLICATE LEAKAGE VERIFICATION\")\nprint(\"-\" * 70)\n\ntrain_hashes = set(train_df[\"image_hash\"])\nval_hashes = set(val_df[\"image_hash\"])\n\ncross_split_hashes = (\n    train_hashes.intersection(val_hashes)\n)\n\nprint(\n    f\"Unique train image groups : \"\n    f\"{len(train_hashes)}\"\n)\n\nprint(\n    f\"Unique validation groups   : \"\n    f\"{len(val_hashes)}\"\n)\n\nprint(\n    f\"Cross-split duplicate groups : \"\n    f\"{len(cross_split_hashes)}\"\n)\n\n\n# ============================================================\n# 14. ID OVERLAP VERIFICATION\n# ============================================================\n\ntrain_ids = set(train_save[\"id_code\"])\nval_ids = set(val_save[\"id_code\"])\n\nid_overlap = train_ids.intersection(val_ids)\n\nprint(\n    f\"Train/validation ID overlap : \"\n    f\"{len(id_overlap)}\"\n)\n\n\n# ============================================================\n# 15. FINAL VERDICT\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 9A FINAL VERDICT\")\nprint(\"=\" * 70)\n\nif len(cross_split_hashes) == 0 and len(id_overlap) == 0:\n\n    print(\"✓ ZERO EXACT DUPLICATE CROSS-SPLIT LEAKAGE\")\n    print(\"✓ ZERO TRAIN/VALIDATION ID OVERLAP\")\n    print(\"✓ CONFLICTING EXACT-DUPLICATE GROUPS REMOVED\")\n    print(\"✓ DUPLICATE GROUPS KEPT TOGETHER\")\n    print(\"✓ NEW LEAKAGE-SAFE SPLIT CREATED\")\n\nelse:\n\n    print(\"⚠ LEAKAGE STILL EXISTS\")\n    print(\"DO NOT PROCEED TO MODEL TRAINING.\")\n\nprint(\"=\" * 70)\n\nprint(\"\\nSaved files:\")\nprint(TRAIN_SPLIT_PATH)\nprint(VAL_SPLIT_PATH)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 10A — PRODUCTION TENSORFLOW DATA PIPELINE\n# CORRECTED VERSION\n# ============================================================\n\nimport os\nimport random\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\nfrom pathlib import Path\n\n\n# ============================================================\n# 1. REPRODUCIBILITY\n# ============================================================\n\nSEED = 42\n\nos.environ[\"PYTHONHASHSEED\"] = str(SEED)\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\n\n\n# ============================================================\n# 2. PATHS\n# ============================================================\n\nBASE_DIR = Path(\n    \"/kaggle/input/competitions/aptos2019-blindness-detection\"\n)\n\nTRAIN_IMG_DIR = BASE_DIR / \"train_images\"\n\nTRAIN_SPLIT = Path(\n    \"/kaggle/working/aptos_splits/train_split.csv\"\n)\n\nVAL_SPLIT = Path(\n    \"/kaggle/working/aptos_splits/val_split.csv\"\n)\n\n\n# ============================================================\n# 3. CONFIGURATION\n# ============================================================\n\nIMG_SIZE = 224\nBATCH_SIZE = 32\nAUTOTUNE = tf.data.AUTOTUNE\n\n\n# ============================================================\n# 4. LOAD SPLITS\n# ============================================================\n\ntrain_df = pd.read_csv(TRAIN_SPLIT)\nval_df = pd.read_csv(VAL_SPLIT)\n\nprint(\"=\" * 70)\nprint(\"STEP 10A — TENSORFLOW DATA PIPELINE\")\nprint(\"=\" * 70)\n\nprint(\"\\n[1] SPLIT VERIFICATION\")\nprint(\"-\" * 70)\n\nprint(f\"Training samples   : {len(train_df)}\")\nprint(f\"Validation samples : {len(val_df)}\")\n\n\n# ============================================================\n# 5. IMAGE PATH CREATION\n# ============================================================\n\ndef make_image_path(image_id):\n    return str(TRAIN_IMG_DIR / f\"{image_id}.png\")\n\n\ntrain_paths = train_df[\"id_code\"].apply(\n    make_image_path\n).values\n\ntrain_labels = train_df[\"diagnosis\"].astype(\n    np.int32\n).values\n\nval_paths = val_df[\"id_code\"].apply(\n    make_image_path\n).values\n\nval_labels = val_df[\"diagnosis\"].astype(\n    np.int32\n).values\n\n\n# ============================================================\n# 6. VERIFY IMAGE PATHS\n# ============================================================\n\nprint(\"\\n[2] IMAGE PATH VERIFICATION\")\nprint(\"-\" * 70)\n\nmissing_train = [\n    path for path in train_paths\n    if not os.path.exists(path)\n]\n\nmissing_val = [\n    path for path in val_paths\n    if not os.path.exists(path)\n]\n\nprint(f\"Missing training images   : {len(missing_train)}\")\nprint(f\"Missing validation images : {len(missing_val)}\")\n\nif missing_train or missing_val:\n    raise FileNotFoundError(\n        \"One or more split images could not be found.\"\n    )\n\nprint(\"✓ All split image paths exist\")\n\n\n# ============================================================\n# 7. RETINAL FIELD CROP\n# ============================================================\n\ndef crop_black_background(image, threshold=10, padding=10):\n    \"\"\"\n    Conservative retinal-field crop.\n\n    Removes large black/dark borders while retaining\n    peripheral retinal content.\n    \"\"\"\n\n    gray = tf.image.rgb_to_grayscale(image)\n\n    mask = tf.squeeze(gray, axis=-1) > threshold\n\n    coords = tf.where(mask)\n\n    def crop_image():\n\n        # IMPORTANT:\n        # tf.where returns int64 coordinates.\n        # Explicitly cast to int32 so they match\n        # image height/width dtype.\n\n        y_min = tf.cast(\n            tf.reduce_min(coords[:, 0]),\n            tf.int32\n        )\n\n        x_min = tf.cast(\n            tf.reduce_min(coords[:, 1]),\n            tf.int32\n        )\n\n        y_max = tf.cast(\n            tf.reduce_max(coords[:, 0]),\n            tf.int32\n        )\n\n        x_max = tf.cast(\n            tf.reduce_max(coords[:, 1]),\n            tf.int32\n        )\n\n        height = tf.shape(image)[0]\n        width = tf.shape(image)[1]\n\n        # Add conservative padding\n\n        y_min_p = tf.maximum(\n            tf.constant(0, dtype=tf.int32),\n            y_min - tf.constant(\n                padding,\n                dtype=tf.int32\n            )\n        )\n\n        x_min_p = tf.maximum(\n            tf.constant(0, dtype=tf.int32),\n            x_min - tf.constant(\n                padding,\n                dtype=tf.int32\n            )\n        )\n\n        y_max_p = tf.minimum(\n            height - tf.constant(\n                1,\n                dtype=tf.int32\n            ),\n            y_max + tf.constant(\n                padding,\n                dtype=tf.int32\n            )\n        )\n\n        x_max_p = tf.minimum(\n            width - tf.constant(\n                1,\n                dtype=tf.int32\n            ),\n            x_max + tf.constant(\n                padding,\n                dtype=tf.int32\n            )\n        )\n\n        return image[\n            y_min_p:y_max_p + 1,\n            x_min_p:x_max_p + 1\n        ]\n\n    return tf.cond(\n        tf.size(coords) > 0,\n        crop_image,\n        lambda: image\n    )\n\n\n# ============================================================\n# 8. TRAINING AUGMENTATION\n# ============================================================\n\ndef augment_image(image):\n\n    # Horizontal flip\n    image = tf.image.random_flip_left_right(\n        image\n    )\n\n    # Mild brightness variation\n    image = tf.image.random_brightness(\n        image,\n        max_delta=0.08\n    )\n\n    # Mild contrast variation\n    image = tf.image.random_contrast(\n        image,\n        lower=0.90,\n        upper=1.10\n    )\n\n    image = tf.clip_by_value(\n        image,\n        0.0,\n        1.0\n    )\n\n    return image\n\n\n# ============================================================\n# 9. IMAGE LOADING\n# ============================================================\n\ndef load_image(path, label):\n\n    image_bytes = tf.io.read_file(path)\n\n    image = tf.image.decode_png(\n        image_bytes,\n        channels=3\n    )\n\n    image = tf.image.convert_image_dtype(\n        image,\n        tf.float32\n    )\n\n    # Conservative retinal crop\n    image = crop_black_background(image)\n\n    # Resize\n    image = tf.image.resize(\n        image,\n        [IMG_SIZE, IMG_SIZE],\n        method=tf.image.ResizeMethod.BILINEAR\n    )\n\n    image = tf.clip_by_value(\n        image,\n        0.0,\n        1.0\n    )\n\n    return image, label\n\n\n# ============================================================\n# 10. TRAINING PREPROCESSING\n# ============================================================\n\ndef preprocess_train(path, label):\n\n    image, label = load_image(\n        path,\n        label\n    )\n\n    image = augment_image(image)\n\n    return image, label\n\n\n# ============================================================\n# 11. VALIDATION PREPROCESSING\n# ============================================================\n\ndef preprocess_validation(path, label):\n\n    image, label = load_image(\n        path,\n        label\n    )\n\n    # NO RANDOM AUGMENTATION\n\n    return image, label\n\n\n# ============================================================\n# 12. CREATE DATASETS\n# ============================================================\n\ntrain_ds = tf.data.Dataset.from_tensor_slices(\n    (train_paths, train_labels)\n)\n\nval_ds = tf.data.Dataset.from_tensor_slices(\n    (val_paths, val_labels)\n)\n\n\n# ============================================================\n# 13. TRAINING PIPELINE\n# ============================================================\n\ntrain_ds = (\n    train_ds\n    .shuffle(\n        buffer_size=len(train_df),\n        seed=SEED,\n        reshuffle_each_iteration=True\n    )\n    .map(\n        preprocess_train,\n        num_parallel_calls=AUTOTUNE\n    )\n    .batch(\n        BATCH_SIZE,\n        drop_remainder=False\n    )\n    .prefetch(AUTOTUNE)\n)\n\n\n# ============================================================\n# 14. VALIDATION PIPELINE\n# ============================================================\n\nval_ds = (\n    val_ds\n    .map(\n        preprocess_validation,\n        num_parallel_calls=AUTOTUNE\n    )\n    .batch(\n        BATCH_SIZE,\n        drop_remainder=False\n    )\n    .prefetch(AUTOTUNE)\n)\n\n\n# ============================================================\n# 15. TEST TRAINING BATCH\n# ============================================================\n\nprint(\"\\n[3] TRAINING PIPELINE TEST\")\nprint(\"-\" * 70)\n\ntrain_images, train_batch_labels = next(\n    iter(train_ds)\n)\n\nprint(f\"Image batch shape : {train_images.shape}\")\nprint(f\"Label shape       : {train_batch_labels.shape}\")\nprint(f\"Image dtype       : {train_images.dtype}\")\n\nprint(\n    f\"Pixel range       : \"\n    f\"{tf.reduce_min(train_images).numpy():.4f}\"\n    f\" → \"\n    f\"{tf.reduce_max(train_images).numpy():.4f}\"\n)\n\nprint(\n    f\"Unique labels     : \"\n    f\"{np.unique(train_batch_labels.numpy())}\"\n)\n\n\n# ============================================================\n# 16. TEST VALIDATION BATCH\n# ============================================================\n\nprint(\"\\n[4] VALIDATION PIPELINE TEST\")\nprint(\"-\" * 70)\n\nval_images, val_batch_labels = next(\n    iter(val_ds)\n)\n\nprint(f\"Image batch shape : {val_images.shape}\")\nprint(f\"Label shape       : {val_batch_labels.shape}\")\nprint(f\"Image dtype       : {val_images.dtype}\")\n\nprint(\n    f\"Pixel range       : \"\n    f\"{tf.reduce_min(val_images).numpy():.4f}\"\n    f\" → \"\n    f\"{tf.reduce_max(val_images).numpy():.4f}\"\n)\n\nprint(\n    f\"Unique labels     : \"\n    f\"{np.unique(val_batch_labels.numpy())}\"\n)\n\n\n# ============================================================\n# 17. FINAL SUMMARY\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 10A PIPELINE SUMMARY\")\nprint(\"=\" * 70)\n\nprint(f\"Input resolution : {IMG_SIZE} × {IMG_SIZE}\")\nprint(f\"Batch size       : {BATCH_SIZE}\")\n\nprint(\"\\nTraining:\")\nprint(\"  ✓ Image loading\")\nprint(\"  ✓ RGB decoding\")\nprint(\"  ✓ Conservative retinal crop\")\nprint(\"  ✓ Resize\")\nprint(\"  ✓ Normalization [0,1]\")\nprint(\"  ✓ Training augmentation\")\nprint(\"  ✓ Shuffle\")\nprint(\"  ✓ Batch\")\nprint(\"  ✓ Prefetch\")\n\nprint(\"\\nValidation:\")\nprint(\"  ✓ Image loading\")\nprint(\"  ✓ RGB decoding\")\nprint(\"  ✓ Conservative retinal crop\")\nprint(\"  ✓ Resize\")\nprint(\"  ✓ Normalization [0,1]\")\nprint(\"  ✓ NO random augmentation\")\nprint(\"  ✓ Batch\")\nprint(\"  ✓ Prefetch\")\n\nprint(\"\\n✓ STEP 10A PIPELINE TEST COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 10A-V — VISUAL PREPROCESSING VERIFICATION\n# ============================================================\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport tensorflow as tf\n\nprint(\"=\" * 70)\nprint(\"STEP 10A-V — VISUAL PREPROCESSING VERIFICATION\")\nprint(\"=\" * 70)\n\n# Take one batch from the validation dataset\n# Validation is used so there is NO random augmentation\nimages, labels = next(iter(val_ds))\n\n# Display 8 images\nplt.figure(figsize=(16, 8))\n\nfor i in range(8):\n    ax = plt.subplot(2, 4, i + 1)\n\n    plt.imshow(images[i].numpy())\n    plt.title(f\"DR Grade: {labels[i].numpy()}\")\n    plt.axis(\"off\")\n\nplt.tight_layout()\nplt.show()\n\nprint(\"\\n✓ Visual preprocessing inspection complete\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 10B — CONSERVATIVE RETINAL AUGMENTATION\n# ============================================================\n\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nprint(\"=\" * 70)\nprint(\"STEP 10B — CONSERVATIVE RETINAL AUGMENTATION\")\nprint(\"=\" * 70)\n\n# ------------------------------------------------------------\n# 1. AUGMENTATION LAYERS\n# ------------------------------------------------------------\n\n# Small rotation: approximately ±10 degrees\nrandom_rotation = tf.keras.layers.RandomRotation(\n    factor=(-10/360, 10/360),\n    fill_mode=\"constant\",\n    fill_value=0.0\n)\n\n# Small zoom: approximately ±10%\nrandom_zoom = tf.keras.layers.RandomZoom(\n    height_factor=(-0.10, 0.10),\n    width_factor=(-0.10, 0.10),\n    fill_mode=\"constant\",\n    fill_value=0.0\n)\n\n# ------------------------------------------------------------\n# 2. CONSERVATIVE AUGMENTATION FUNCTION\n# ------------------------------------------------------------\n\ndef augment_image(image):\n    # Small geometric variation\n    image = random_rotation(image, training=True)\n    image = random_zoom(image, training=True)\n\n    # Mild illumination variation\n    image = tf.image.random_brightness(\n        image,\n        max_delta=0.08\n    )\n\n    # Mild contrast variation\n    image = tf.image.random_contrast(\n        image,\n        lower=0.90,\n        upper=1.10\n    )\n\n    # Keep valid pixel range\n    image = tf.clip_by_value(image, 0.0, 1.0)\n\n    return image\n\n\n# ------------------------------------------------------------\n# 3. TEST ON VALIDATION IMAGES\n# ------------------------------------------------------------\n# We use validation images only for VISUAL inspection.\n# These images are NOT used for training here.\n\nimages, labels = next(iter(val_ds))\n\n# Select first 8 images\nsample_images = images[:8]\nsample_labels = labels[:8]\n\n# Apply augmentation multiple times\naugmented_images = []\n\nfor i in range(8):\n    aug_img = augment_image(sample_images[i])\n    augmented_images.append(aug_img)\n\naugmented_images = tf.stack(augmented_images)\n\n\n# ------------------------------------------------------------\n# 4. DISPLAY ORIGINAL VS AUGMENTED\n# ------------------------------------------------------------\n\nplt.figure(figsize=(16, 8))\n\nfor i in range(8):\n\n    # Original\n    ax = plt.subplot(2, 8, i + 1)\n    plt.imshow(sample_images[i].numpy())\n    plt.title(f\"Original\\nGrade {sample_labels[i].numpy()}\")\n    plt.axis(\"off\")\n\n    # Augmented\n    ax = plt.subplot(2, 8, i + 9)\n    plt.imshow(augmented_images[i].numpy())\n    plt.title(f\"Augmented\\nGrade {sample_labels[i].numpy()}\")\n    plt.axis(\"off\")\n\nplt.tight_layout()\nplt.show()\n\n\n# ------------------------------------------------------------\n# 5. SUMMARY\n# ------------------------------------------------------------\n\nprint()\nprint(\"=\" * 70)\nprint(\"STEP 10B AUGMENTATION SUMMARY\")\nprint(\"=\" * 70)\n\nprint(\"Rotation       : ±10°\")\nprint(\"Zoom           : ±10%\")\nprint(\"Brightness     : ±0.08\")\nprint(\"Contrast       : 0.90 → 1.10\")\nprint(\"Horizontal flip: DISABLED\")\nprint(\"Vertical flip  : DISABLED\")\nprint(\"Shear          : DISABLED\")\nprint(\"Large shift    : DISABLED\")\nprint(\"CLAHE          : DISABLED\")\nprint(\"Noise          : DISABLED\")\nprint(\"Color shifting : DISABLED\")\n\nprint()\nprint(\"✓ Augmentation implementation test complete\")\nprint(\"=\" * 70)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 10B-FINAL — FUNCTION SIGNATURE FIX\n# ============================================================\n\nprint(\"=\" * 70)\nprint(\"STEP 10B-FINAL — FIXING LOAD_IMAGE SIGNATURE\")\nprint(\"=\" * 70)\n\n\n# ------------------------------------------------------------\n# 1. FINAL TRAINING PREPROCESSING\n# ------------------------------------------------------------\n\ndef train_preprocess_final(path, label):\n\n    # load_image expects BOTH path and label\n    image, label = load_image(path, label)\n\n    # Approved augmentation\n    image = final_augment(image)\n\n    return image, label\n\n\n# ------------------------------------------------------------\n# 2. FINAL VALIDATION PREPROCESSING\n# ------------------------------------------------------------\n\ndef val_preprocess_final(path, label):\n\n    # load_image expects BOTH path and label\n    image, label = load_image(path, label)\n\n    # NO random augmentation\n    return image, label\n\n\n# ------------------------------------------------------------\n# 3. REBUILD TRAIN DATASET\n# ------------------------------------------------------------\n\ntrain_paths = train_df[\"image_path\"].values\ntrain_labels = train_df[\"diagnosis\"].values.astype(np.int32)\n\ntrain_ds = tf.data.Dataset.from_tensor_slices(\n    (train_paths, train_labels)\n)\n\ntrain_ds = (\n    train_ds\n    .shuffle(\n        buffer_size=len(train_df),\n        seed=SEED,\n        reshuffle_each_iteration=True\n    )\n    .map(\n        train_preprocess_final,\n        num_parallel_calls=AUTOTUNE\n    )\n    .batch(BATCH_SIZE)\n    .prefetch(AUTOTUNE)\n)\n\n\n# ------------------------------------------------------------\n# 4. REBUILD VALIDATION DATASET\n# ------------------------------------------------------------\n\nval_paths = val_df[\"image_path\"].values\nval_labels = val_df[\"diagnosis\"].values.astype(np.int32)\n\nval_ds = tf.data.Dataset.from_tensor_slices(\n    (val_paths, val_labels)\n)\n\nval_ds = (\n    val_ds\n    .map(\n        val_preprocess_final,\n        num_parallel_calls=AUTOTUNE\n    )\n    .batch(BATCH_SIZE)\n    .prefetch(AUTOTUNE)\n)\n\n\n# ------------------------------------------------------------\n# 5. TEST TRAINING PIPELINE\n# ------------------------------------------------------------\n\ntrain_images, train_labels_batch = next(iter(train_ds))\n\nprint()\nprint(\"[1] TRAINING PIPELINE TEST\")\nprint(\"-\" * 70)\n\nprint(\"Image batch shape :\", train_images.shape)\nprint(\"Label shape       :\", train_labels_batch.shape)\nprint(\"Image dtype       :\", train_images.dtype)\n\ntrain_min = tf.reduce_min(train_images).numpy()\ntrain_max = tf.reduce_max(train_images).numpy()\n\nprint(f\"Pixel range       : {train_min:.4f} → {train_max:.4f}\")\nprint(\"Unique labels     :\", np.unique(train_labels_batch.numpy()))\n\n\n# ------------------------------------------------------------\n# 6. TEST VALIDATION PIPELINE\n# ------------------------------------------------------------\n\nval_images, val_labels_batch = next(iter(val_ds))\n\nprint()\nprint(\"[2] VALIDATION PIPELINE TEST\")\nprint(\"-\" * 70)\n\nprint(\"Image batch shape :\", val_images.shape)\nprint(\"Label shape       :\", val_labels_batch.shape)\nprint(\"Image dtype       :\", val_images.dtype)\n\nval_min = tf.reduce_min(val_images).numpy()\nval_max = tf.reduce_max(val_images).numpy()\n\nprint(f\"Pixel range       : {val_min:.4f} → {val_max:.4f}\")\nprint(\"Unique labels     :\", np.unique(val_labels_batch.numpy()))\n\n\n# ------------------------------------------------------------\n# 7. FINAL SUMMARY\n# ------------------------------------------------------------\n\nprint()\nprint(\"=\" * 70)\nprint(\"STEP 10B-FINAL SUMMARY\")\nprint(\"=\" * 70)\n\nprint(\"Training samples   :\", len(train_df))\nprint(\"Validation samples :\", len(val_df))\nprint(\"Input resolution   :\", f\"{IMG_SIZE} × {IMG_SIZE}\")\nprint(\"Batch size         :\", BATCH_SIZE)\n\nprint()\nprint(\"Approved augmentation:\")\nprint(\"  ✓ Rotation        : ±10°\")\nprint(\"  ✓ Zoom            : ±10%\")\nprint(\"  ✓ Brightness      : ±0.08\")\nprint(\"  ✓ Contrast        : 0.90 → 1.10\")\n\nprint()\nprint(\"Disabled:\")\nprint(\"  ✓ Horizontal flip : OFF\")\nprint(\"  ✓ Vertical flip   : OFF\")\nprint(\"  ✓ CLAHE           : OFF\")\nprint(\"  ✓ Noise           : OFF\")\nprint(\"  ✓ Shear           : OFF\")\nprint(\"  ✓ Large shift     : OFF\")\n\nprint()\nprint(\"Validation:\")\nprint(\"  ✓ Random augmentation : OFF\")\nprint(\"  ✓ Deterministic       : YES\")\n\nprint()\nprint(\"✓ STEP 10B-FINAL COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 11A — EFFICIENTNETB0 ARCHITECTURE\n# ============================================================\n\nimport tensorflow as tf\nimport numpy as np\n\nprint(\"=\" * 70)\nprint(\"STEP 11A — EFFICIENTNETB0 ARCHITECTURE\")\nprint(\"=\" * 70)\n\n# ------------------------------------------------------------\n# 1. CONFIGURATION\n# ------------------------------------------------------------\n\nIMG_SIZE = 224\nNUM_CLASSES = 5\n\nprint()\nprint(\"[1] CONFIGURATION\")\nprint(\"-\" * 70)\nprint(\"Input size       :\", f\"{IMG_SIZE} × {IMG_SIZE} × 3\")\nprint(\"Number of classes:\", NUM_CLASSES)\nprint(\"Backbone         : EfficientNetB0\")\nprint(\"Pretrained       : ImageNet\")\nprint(\"Initial mode     : Frozen backbone\")\n\n\n# ------------------------------------------------------------\n# 2. CREATE INPUT\n# ------------------------------------------------------------\n\ninputs = tf.keras.Input(\n    shape=(IMG_SIZE, IMG_SIZE, 3),\n    name=\"fundus_input\"\n)\n\n\n# ------------------------------------------------------------\n# 3. INPUT RESCALING\n# ------------------------------------------------------------\n# Our tf.data pipeline outputs [0,1].\n# EfficientNet expects [0,255].\n# Therefore convert [0,1] -> [0,255].\n\nx = tf.keras.layers.Rescaling(\n    scale=255.0,\n    name=\"efficientnet_input_rescaling\"\n)(inputs)\n\n\n# ------------------------------------------------------------\n# 4. LOAD IMAGENET-PRETRAINED EFFICIENTNETB0\n# ------------------------------------------------------------\n\nbase_model = tf.keras.applications.EfficientNetB0(\n    include_top=False,\n    weights=\"imagenet\",\n    input_shape=(IMG_SIZE, IMG_SIZE, 3)\n)\n\n# Freeze backbone for initial transfer learning\nbase_model.trainable = False\n\n\n# ------------------------------------------------------------\n# 5. PASS THROUGH BACKBONE\n# ------------------------------------------------------------\n# training=False keeps BatchNormalization layers in inference\n# mode during the frozen transfer-learning stage.\n\nx = base_model(\n    x,\n    training=False\n)\n\n\n# ------------------------------------------------------------\n# 6. GLOBAL AVERAGE POOLING\n# ------------------------------------------------------------\n\nx = tf.keras.layers.GlobalAveragePooling2D(\n    name=\"global_average_pooling\"\n)(x)\n\n\n# ------------------------------------------------------------\n# 7. DROPOUT\n# ------------------------------------------------------------\n\nx = tf.keras.layers.Dropout(\n    rate=0.30,\n    name=\"classifier_dropout\"\n)(x)\n\n\n# ------------------------------------------------------------\n# 8. FINAL 5-CLASS CLASSIFIER\n# ------------------------------------------------------------\n\noutputs = tf.keras.layers.Dense(\n    NUM_CLASSES,\n    activation=\"softmax\",\n    name=\"dr_grade_output\"\n)(x)\n\n\n# ------------------------------------------------------------\n# 9. CREATE COMPLETE MODEL\n# ------------------------------------------------------------\n\nmodel = tf.keras.Model(\n    inputs=inputs,\n    outputs=outputs,\n    name=\"APTOS_EfficientNetB0\"\n)\n\n\n# ------------------------------------------------------------\n# 10. MODEL SUMMARY\n# ------------------------------------------------------------\n\nprint()\nprint(\"[2] MODEL ARCHITECTURE\")\nprint(\"-\" * 70)\n\nmodel.summary()\n\n\n# ------------------------------------------------------------\n# 11. PARAMETER COUNTS\n# ------------------------------------------------------------\n\ntotal_params = model.count_params()\n\ntrainable_params = np.sum(\n    [np.prod(v.shape) for v in model.trainable_weights]\n)\n\nnon_trainable_params = np.sum(\n    [np.prod(v.shape) for v in model.non_trainable_weights]\n)\n\nprint()\nprint(\"[3] PARAMETER COUNTS\")\nprint(\"-\" * 70)\n\nprint(\"Total parameters     :\", f\"{total_params:,}\")\nprint(\"Trainable parameters :\", f\"{trainable_params:,}\")\nprint(\"Frozen parameters    :\", f\"{non_trainable_params:,}\")\n\n\n# ------------------------------------------------------------\n# 12. VERIFY BACKBONE IS FROZEN\n# ------------------------------------------------------------\n\nprint()\nprint(\"[4] BACKBONE STATUS\")\nprint(\"-\" * 70)\n\nprint(\"Base model trainable :\", base_model.trainable)\n\nif not base_model.trainable:\n    print(\"✓ EfficientNetB0 backbone is frozen\")\nelse:\n    print(\"⚠ WARNING: Backbone is trainable\")\n\n\n# ------------------------------------------------------------\n# 13. FORWARD PASS TEST\n# ------------------------------------------------------------\n\nprint()\nprint(\"[5] FORWARD PASS TEST\")\nprint(\"-\" * 70)\n\n# Use one real batch from our existing validation pipeline\nsample_images, sample_labels = next(iter(val_ds))\n\npredictions = model(\n    sample_images,\n    training=False\n)\n\nprint(\"Input shape          :\", sample_images.shape)\nprint(\"Output shape         :\", predictions.shape)\nprint(\"Output dtype         :\", predictions.dtype)\n\nprint(\n    \"Probability range    : \"\n    f\"{tf.reduce_min(predictions).numpy():.6f}\"\n    \" → \"\n    f\"{tf.reduce_max(predictions).numpy():.6f}\"\n)\n\n# Verify probabilities sum to approximately 1\nprobability_sums = tf.reduce_sum(\n    predictions,\n    axis=1\n)\n\nprint(\n    \"Probability sum range: \"\n    f\"{tf.reduce_min(probability_sums).numpy():.6f}\"\n    \" → \"\n    f\"{tf.reduce_max(probability_sums).numpy():.6f}\"\n)\n\n\n# ------------------------------------------------------------\n# 14. PREDICTED CLASS TEST\n# ------------------------------------------------------------\n\npredicted_classes = tf.argmax(\n    predictions,\n    axis=1\n)\n\nprint()\nprint(\"Sample true labels   :\", sample_labels.numpy()[:10])\nprint(\"Sample predictions   :\", predicted_classes.numpy()[:10])\n\n\n# ------------------------------------------------------------\n# 15. FINAL VERIFICATION\n# ------------------------------------------------------------\n\nprint()\nprint(\"=\" * 70)\nprint(\"STEP 11A VERIFICATION\")\nprint(\"=\" * 70)\n\nchecks = {\n    \"Correct input shape\": model.input_shape == (None, IMG_SIZE, IMG_SIZE, 3),\n    \"Correct output shape\": model.output_shape == (None, NUM_CLASSES),\n    \"ImageNet weights loaded\": base_model.weights[0] is not None,\n    \"Backbone frozen\": base_model.trainable is False,\n    \"5-class softmax output\": model.output_shape[-1] == 5,\n    \"Forward pass successful\": predictions.shape == (32, NUM_CLASSES),\n}\n\nfor name, result in checks.items():\n    print(f\"{'✓' if result else '✗'} {name}\")\n\nif all(checks.values()):\n    print()\n    print(\"✓ STEP 11A ARCHITECTURE TEST PASSED\")\nelse:\n    print()\n    print(\"⚠ STEP 11A REQUIRES INVESTIGATION\")\n\nprint(\"=\" * 70)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 11B — LOSS, METRICS & TRAINING CONFIGURATION\n# ============================================================\n\nimport tensorflow as tf\nimport numpy as np\n\nprint(\"=\" * 70)\nprint(\"STEP 11B — LOSS, METRICS & TRAINING CONFIGURATION\")\nprint(\"=\" * 70)\n\n\n# ------------------------------------------------------------\n# 1. BASELINE LOSS\n# ------------------------------------------------------------\n# Controlled baseline:\n# Standard sparse categorical cross-entropy.\n#\n# We intentionally do NOT use class weights, focal loss,\n# oversampling, or label smoothing yet.\n#\n# This gives us a clean baseline experiment.\n\nloss_function = tf.keras.losses.SparseCategoricalCrossentropy(\n    from_logits=False\n)\n\n\n# ------------------------------------------------------------\n# 2. OPTIMIZER\n# ------------------------------------------------------------\n# Only the classifier head is trainable at this stage.\n\noptimizer = tf.keras.optimizers.Adam(\n    learning_rate=1e-3\n)\n\n\n# ------------------------------------------------------------\n# 3. COMPILE MODEL\n# ------------------------------------------------------------\n\nmodel.compile(\n    optimizer=optimizer,\n    loss=loss_function,\n    metrics=[\n        tf.keras.metrics.SparseCategoricalAccuracy(\n            name=\"accuracy\"\n        )\n    ]\n)\n\n\n# ------------------------------------------------------------\n# 4. MODEL STATUS\n# ------------------------------------------------------------\n\nprint()\nprint(\"[1] MODEL STATUS\")\nprint(\"-\" * 70)\n\nprint(\"Model name         :\", model.name)\nprint(\"Input shape        :\", model.input_shape)\nprint(\"Output shape       :\", model.output_shape)\n\nprint(\"Backbone trainable :\", base_model.trainable)\n\nprint(\n    \"Trainable params   :\",\n    f\"{model.count_params() - sum(np.prod(v.shape) for v in base_model.weights):,}\"\n)\n\n\n# ------------------------------------------------------------\n# 5. LOSS CONFIGURATION\n# ------------------------------------------------------------\n\nprint()\nprint(\"[2] LOSS CONFIGURATION\")\nprint(\"-\" * 70)\n\nprint(\"Loss               : SparseCategoricalCrossentropy\")\nprint(\"From logits        : False\")\nprint(\"Class weights       : NONE\")\nprint(\"Focal loss          : NONE\")\nprint(\"Oversampling        : NONE\")\nprint(\"Label smoothing     : NONE\")\n\n\n# ------------------------------------------------------------\n# 6. OPTIMIZER CONFIGURATION\n# ------------------------------------------------------------\n\nprint()\nprint(\"[3] OPTIMIZER CONFIGURATION\")\nprint(\"-\" * 70)\n\nprint(\"Optimizer          : Adam\")\nprint(\"Learning rate      : 0.001\")\nprint(\"Trainable layers   : Classification head only\")\n\n\n# ------------------------------------------------------------\n# 7. METRICS\n# ------------------------------------------------------------\n\nprint()\nprint(\"[4] BASE TRAINING METRIC\")\nprint(\"-\" * 70)\n\nprint(\"Training metric    : Sparse Categorical Accuracy\")\n\nprint()\nprint(\"Post-training evaluation will include:\")\nprint(\"  ✓ Accuracy\")\nprint(\"  ✓ Macro F1\")\nprint(\"  ✓ Weighted F1\")\nprint(\"  ✓ Balanced Accuracy\")\nprint(\"  ✓ Per-class Precision\")\nprint(\"  ✓ Per-class Recall\")\nprint(\"  ✓ Confusion Matrix\")\nprint(\"  ✓ Quadratic Weighted Kappa (QWK)\")\n\n\n# ------------------------------------------------------------\n# 8. VERIFY COMPILE\n# ------------------------------------------------------------\n\nprint()\nprint(\"[5] COMPILE VERIFICATION\")\nprint(\"-\" * 70)\n\nprint(\"Optimizer object   :\", type(model.optimizer).__name__)\nprint(\"Loss function      :\", type(model.loss).__name__)\n\nprint()\nprint(\"=\" * 70)\nprint(\"STEP 11B SUMMARY\")\nprint(\"=\" * 70)\n\nprint(\"Experiment         : B0 — Frozen EfficientNetB0 Baseline\")\nprint(\"Input resolution   : 224 × 224\")\nprint(\"Backbone           : ImageNet EfficientNetB0\")\nprint(\"Backbone status    : FROZEN\")\nprint(\"Loss               : Sparse Categorical Cross-Entropy\")\nprint(\"Optimizer          : Adam\")\nprint(\"Learning rate      : 1e-3\")\nprint(\"Class balancing    : OFF\")\nprint(\"Focal loss         : OFF\")\nprint(\"Oversampling       : OFF\")\nprint(\"Label smoothing    : OFF\")\n\nprint()\nprint(\"✓ Model compiled successfully\")\nprint(\"✓ Ready for controlled baseline training\")\nprint(\"=\" * 70)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 12 — B0 BASELINE TRAINING\n# ============================================================\n\nimport os\nimport tensorflow as tf\n\nprint(\"=\" * 70)\nprint(\"STEP 12 — B0 BASELINE TRAINING\")\nprint(\"=\" * 70)\n\n# ------------------------------------------------------------\n# 1. TRAINING CONFIGURATION\n# ------------------------------------------------------------\n\nEPOCHS = 15\n\nCHECKPOINT_PATH = \"/kaggle/working/b0_efficientnetb0_best.keras\"\n\nprint()\nprint(\"[1] TRAINING CONFIGURATION\")\nprint(\"-\" * 70)\n\nprint(\"Experiment          : B0\")\nprint(\"Backbone            : EfficientNetB0\")\nprint(\"Backbone trainable  : False\")\nprint(\"Input resolution    : 224 × 224\")\nprint(\"Batch size          :\", BATCH_SIZE)\nprint(\"Maximum epochs      :\", EPOCHS)\nprint(\"Loss                : SparseCategoricalCrossentropy\")\nprint(\"Optimizer           : Adam\")\nprint(\"Initial LR          : 0.001\")\nprint(\"Class weighting     : OFF\")\nprint(\"Focal loss          : OFF\")\nprint(\"Oversampling        : OFF\")\nprint(\"Checkpoint          :\", CHECKPOINT_PATH)\n\n\n# ------------------------------------------------------------\n# 2. CALLBACKS\n# ------------------------------------------------------------\n\ncheckpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n    filepath=CHECKPOINT_PATH,\n    monitor=\"val_loss\",\n    mode=\"min\",\n    save_best_only=True,\n    save_weights_only=False,\n    verbose=1\n)\n\nearly_stopping_callback = tf.keras.callbacks.EarlyStopping(\n    monitor=\"val_loss\",\n    mode=\"min\",\n    patience=4,\n    restore_best_weights=True,\n    verbose=1\n)\n\nreduce_lr_callback = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor=\"val_loss\",\n    mode=\"min\",\n    factor=0.3,\n    patience=2,\n    min_lr=1e-6,\n    verbose=1\n)\n\n\n# ------------------------------------------------------------\n# 3. TRAINING\n# ------------------------------------------------------------\n\nprint()\nprint(\"=\" * 70)\nprint(\"STARTING B0 TRAINING\")\nprint(\"=\" * 70)\n\nhistory = model.fit(\n    train_ds,\n    validation_data=val_ds,\n    epochs=EPOCHS,\n    callbacks=[\n        checkpoint_callback,\n        early_stopping_callback,\n        reduce_lr_callback\n    ],\n    verbose=1\n)\n\n\n# ------------------------------------------------------------\n# 4. TRAINING SUMMARY\n# ------------------------------------------------------------\n\nprint()\nprint(\"=\" * 70)\nprint(\"B0 TRAINING COMPLETE\")\nprint(\"=\" * 70)\n\ntrained_epochs = len(history.history[\"loss\"])\n\nbest_epoch = (\n    int(tf.argmin(history.history[\"val_loss\"]).numpy()) + 1\n)\n\nbest_val_loss = min(history.history[\"val_loss\"])\n\nbest_val_accuracy = max(\n    history.history[\"val_accuracy\"]\n)\n\nprint(\"Epochs completed      :\", trained_epochs)\nprint(\"Best epoch            :\", best_epoch)\nprint(f\"Best validation loss  : {best_val_loss:.6f}\")\nprint(f\"Best validation acc   : {best_val_accuracy:.6f}\")\n\nprint()\nprint(\"Final training loss   :\", history.history[\"loss\"][-1])\nprint(\"Final training acc    :\", history.history[\"accuracy\"][-1])\nprint(\"Final validation loss :\", history.history[\"val_loss\"][-1])\nprint(\"Final validation acc  :\", history.history[\"val_accuracy\"][-1])\n\nprint()\nprint(\"Best model saved to:\")\nprint(CHECKPOINT_PATH)\n\nprint()\nprint(\"✓ STEP 12 B0 TRAINING COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}