{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport shutil\n\noutput_dir = \"/kaggle/working/\"\nfor item in os.listdir(output_dir):\n    item_path = os.path.join(output_dir, item)\n    try:\n        if os.path.isfile(item_path):\n            os.remove(item_path)  # Delete individual file\n        elif os.path.isdir(item_path):\n            shutil.rmtree(item_path)  # Delete directory and its contents\n        print(f\"Deleted: {item_path}\")\n    except Exception as e:\n        print(f\"Error deleting {item_path}: {e}\")\n\nprint(\"Output directory cleared.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport random\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, confusion_matrix\nimport matplotlib.pyplot as plt\nfrom glob import glob\nimport math\n\n# Enable mixed precision\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16')\n\n\n# =====================\n# STEP 1: Load Data\n# =====================\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\n\nCSV_PATH = '/kaggle/input/aptos2019-blindness-detection/train.csv'\nIMAGE_DIR = '/kaggle/input/aptos2019-blindness-detection/train_images'\n\ndf = pd.read_csv(CSV_PATH)\ndf['image'] = df['id_code'] + '.png'\ndf.rename(columns={'diagnosis': 'level'}, inplace=True)\n\n# Split Data (80/10/10)\ntrain_df, temp_df = train_test_split(df, test_size=0.2, stratify=df['level'], random_state=SEED)\nval_df, test_df = train_test_split(temp_df, test_size=0.5, stratify=temp_df['level'], random_state=SEED)\nprint(\"Train:\", len(train_df), \"Val:\", len(val_df), \"Test:\", len(test_df))\n\n# =====================\n# STEP 2: Preprocessing + Augmentation\n# =====================\nIMG_SIZE = 224\nNB_CHANNELS = 3\n\ndef get_pad_width(im, new_shape, is_rgb=True):\n    pad_diff = new_shape - im.shape[0], new_shape - im.shape[1]\n    t, b = math.floor(pad_diff[0]/2), math.ceil(pad_diff[0]/2)\n    l, r = math.floor(pad_diff[1]/2), math.ceil(pad_diff[1]/2)\n    if is_rgb:\n        pad_width = ((t,b), (l,r), (0, 0))\n    else:\n        pad_width = ((t,b), (l,r))\n    return pad_width\n\ndef standardize(x):\n    x = x.astype(np.float32)\n    x = x / np.max(x)\n    return (x - np.mean(x)) / (np.std(x))\n\ndef normalize(img):\n    img = ((img - np.min(img)) / (np.max(img) - np.min(img))) * 255\n    return img.astype(np.uint8)\n\ndef crop_image(img, tol=10):\n    def crop_image_1(img):\n        mask = img > tol\n        return img[np.ix_(mask.any(1), mask.any(0))]\n    \n    if img.ndim == 2:\n        return crop_image_1(img)\n    \n    elif img.ndim == 3:\n        try:\n            img_cpy = img.copy()\n            h, w, _ = img.shape\n            img1 = cv2.resize(crop_image_1(img[:, :, 0]), (w, h))\n            img2 = cv2.resize(crop_image_1(img[:, :, 1]), (w, h))\n            img3 = cv2.resize(crop_image_1(img[:, :, 2]), (w, h))\n\n            img[:,:,0] = img1\n            img[:,:,1] = img2\n            img[:,:,2] = img3\n            return img\n        except:\n            return img_cpy\n\ndef preprocess_image(img_name, label=None, base_dir=IMAGE_DIR):\n    img_path = os.path.join(base_dir, img_name)\n    im = cv2.imread(img_path)\n    if im is None:\n        print(f\"Failed to load {img_path}\")\n        return None\n\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    im = normalize(im)\n    im = crop_image(im)\n    im = cv2.resize(im, (IMG_SIZE, IMG_SIZE))\n\n    \n\n    # Note: Applying CLAHE after resize to enhance contrast while preserving color\n    im_lab = cv2.cvtColor(im, cv2.COLOR_RGB2LAB)\n    l_channel, a_channel, b_channel = cv2.split(im_lab)\n    clahe = cv2.createCLAHE(clipLimit=0.1, tileGridSize=(2, 2))\n    l_channel = clahe.apply(l_channel)\n    im_lab = cv2.merge([l_channel, a_channel, b_channel])\n    im = cv2.cvtColor(im_lab, cv2.COLOR_LAB2RGB)\n    \n    im = cv2.addWeighted(im, 4, cv2.GaussianBlur(im, (0, 0), IMG_SIZE / 10), -4, 128)\n\n     # 🔳 Mask background to black using circular ROI\n    mask = np.zeros((IMG_SIZE, IMG_SIZE), dtype=np.uint8)  # ← new\n    cv2.circle(mask, (IMG_SIZE // 2, IMG_SIZE // 2), IMG_SIZE // 2, 255, -1)  # ← new\n    for c in range(3):  # ← new\n        im[:, :, c] = np.where(mask == 255, im[:, :, c], 0)  # ← new\n\n   \n    \n    return im.astype(np.uint8)\n\ndef augment_image(img):\n    datagen = tf.keras.preprocessing.image.ImageDataGenerator(\n        rotation_range=20,\n        horizontal_flip=True\n    )\n    img = img.reshape((1,) + img.shape)\n    return next(datagen.flow(img, batch_size=1))[0].astype(np.uint8).squeeze()\n\nprint(\"Applying preprocessing and augmentation for a single image...\")\n\nPREPROCESSED_DIR = 'processed_images'\nos.makedirs(PREPROCESSED_DIR, exist_ok=True)\nupdated_rows = []\n\n# Select a single image (e.g., the first row from train_df)\nrow = train_df.iloc[0]\nimg_name = row['image']\nlabel = row['level']\nimg = preprocess_image(img_name, label, IMAGE_DIR)\nif img is not None:\n    # Save the preprocessed image before augmentation for inspection\n    preprocessed_name = f\"{row['image'].split('.')[0]}_preprocessed.png\"\n    cv2.imwrite(os.path.join(PREPROCESSED_DIR, preprocessed_name), cv2.cvtColor(img, cv2.COLOR_RGB2BGR))\n    print(f\"Saved preprocessed image (before augmentation): {preprocessed_name}\")\n\n    # Apply augmentation\n    for i in range(2):\n        aug_img = augment_image(img)\n        new_name = f\"{row['image'].split('.')[0]}_aug{i}.png\"\n        cv2.imwrite(os.path.join(PREPROCESSED_DIR, new_name), cv2.cvtColor(aug_img, cv2.COLOR_RGB2BGR))\n        updated_rows.append({'image': new_name, 'level': row['level']})\n\naug_df = pd.DataFrame(updated_rows)\nprint(f\"Processed and augmented 1 image. Resulting DataFrame:\\n{aug_df}\")\n\n# =====================\n# Show preprocessed images from \"PREPROCESSED_DIR\"\n# =====================\nprint(\"\\nShowing preprocessed images from PREPROCESSED_DIR:\")\nplt.figure(figsize=(15, 5))\n\n# Show the preprocessed image (before augmentation)\npreprocessed_path = os.path.join(PREPROCESSED_DIR, preprocessed_name)\npreprocessed_img = cv2.imread(preprocessed_path)\nif preprocessed_img is not None:\n    preprocessed_img = cv2.cvtColor(preprocessed_img, cv2.COLOR_BGR2RGB)\n    plt.subplot(1, 3, 1)\n    plt.imshow(preprocessed_img)\n    plt.title(f\"Preprocessed (Label: {label})\")\n    plt.axis('off')\n\n# Show the augmented images\nfor i, row in enumerate(aug_df.itertuples(), 2):\n    img_path = os.path.join(PREPROCESSED_DIR, row.image)\n    img = cv2.imread(img_path)\n    if img is not None:\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        plt.subplot(1, 3, i)\n        plt.imshow(img)\n        plt.title(f\"Augmented {i-1} (Label: {row.level})\")\n        plt.axis('off')\n\nplt.tight_layout()\nplt.show()\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-23T16:06:05.104237Z","iopub.execute_input":"2025-04-23T16:06:05.104995Z","iopub.status.idle":"2025-04-23T16:06:06.553168Z","shell.execute_reply.started":"2025-04-23T16:06:05.104956Z","shell.execute_reply":"2025-04-23T16:06:06.552Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport random\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, confusion_matrix\nimport matplotlib.pyplot as plt\nfrom glob import glob\n\n#c\nimport gc\nfrom tensorflow.keras import mixed_precision\nmixed_precision.set_global_policy('mixed_float16')\n\n# =====================\n# STEP 1: Load Data\n# =====================\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\n\nCSV_PATH = '/kaggle/input/aptos2019-blindness-detection/train.csv'\nIMAGE_DIR = '/kaggle/input/aptos2019-blindness-detection/train_images'\n\ndf = pd.read_csv(CSV_PATH)\ndf['image'] = df['id_code'] + '.png'\ndf.rename(columns={'diagnosis': 'level'}, inplace=True)\n\n\n# Split Data (80/10/10)\ntrain_df, temp_df = train_test_split(df, test_size=0.2, stratify=df['level'], random_state=SEED)\nval_df, test_df = train_test_split(temp_df, test_size=0.5, stratify=temp_df['level'], random_state=SEED)\nprint(\"Train:\", len(train_df), \"Val:\", len(val_df), \"Test:\", len(test_df))\n\n\n# =====================\n# STEP 2: Preprocessing + Augmentation\n# =====================\nIMG_SIZE = 224\nNB_CHANNELS = 3\n\ndef get_pad_width(im, new_shape, is_rgb=True):\n    pad_diff = new_shape - im.shape[0], new_shape - im.shape[1]\n    t, b = math.floor(pad_diff[0]/2), math.ceil(pad_diff[0]/2)\n    l, r = math.floor(pad_diff[1]/2), math.ceil(pad_diff[1]/2)\n    if is_rgb:\n        pad_width = ((t,b), (l,r), (0, 0))\n    else:\n        pad_width = ((t,b), (l,r))\n    return pad_width\n\ndef standardize(x):\n    x = x.astype(np.float32)\n    x = x / np.max(x)\n    return (x - np.mean(x)) / (np.std(x))\n\ndef normalize(img):\n    img = ((img - np.min(img)) / (np.max(img) - np.min(img))) * 255\n    return img.astype(np.uint8)\n\ndef crop_image(img, tol=10):\n    def crop_image_1(img):\n        mask = img > tol\n        return img[np.ix_(mask.any(1), mask.any(0))]\n    \n    if img.ndim == 2:\n        return crop_image_1(img)\n    \n    elif img.ndim == 3:\n        try:\n            img_cpy = img.copy()\n            h, w, _ = img.shape\n            img1 = cv2.resize(crop_image_1(img[:, :, 0]), (w, h))\n            img2 = cv2.resize(crop_image_1(img[:, :, 1]), (w, h))\n            img3 = cv2.resize(crop_image_1(img[:, :, 2]), (w, h))\n\n            img[:,:,0] = img1\n            img[:,:,1] = img2\n            img[:,:,2] = img3\n            return img\n        except:\n            return img_cpy\n\ndef preprocess_image(img_name, label=None, base_dir=IMAGE_DIR):\n    img_path = os.path.join(base_dir, img_name)\n    im = cv2.imread(img_path)\n    if im is None:\n        print(f\"Failed to load {img_path}\")\n        return None\n\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    im = normalize(im)\n    im = crop_image(im)\n    im = cv2.resize(im, (IMG_SIZE, IMG_SIZE))\n\n    \n\n    # Note: Applying CLAHE after resize to enhance contrast while preserving color\n    im_lab = cv2.cvtColor(im, cv2.COLOR_RGB2LAB)\n    l_channel, a_channel, b_channel = cv2.split(im_lab)\n    clahe = cv2.createCLAHE(clipLimit=0.1, tileGridSize=(2, 2))\n    l_channel = clahe.apply(l_channel)\n    im_lab = cv2.merge([l_channel, a_channel, b_channel])\n    im = cv2.cvtColor(im_lab, cv2.COLOR_LAB2RGB)\n    \n    im = cv2.addWeighted(im, 4, cv2.GaussianBlur(im, (0, 0), IMG_SIZE / 10), -4, 128)\n\n     # 🔳 Mask background to black using circular ROI\n    mask = np.zeros((IMG_SIZE, IMG_SIZE), dtype=np.uint8)  # ← new\n    cv2.circle(mask, (IMG_SIZE // 2, IMG_SIZE // 2), IMG_SIZE // 2, 255, -1)  # ← new\n    for c in range(3):  # ← new\n        im[:, :, c] = np.where(mask == 255, im[:, :, c], 0)  # ← new\n\n   \n    \n    return im.astype(np.uint8)\n\ndef augment_image(img):\n    datagen = tf.keras.preprocessing.image.ImageDataGenerator(\n        rotation_range=20,\n        horizontal_flip=True\n    )\n    img = img.reshape((1,) + img.shape)\n    return next(datagen.flow(img, batch_size=1))[0].astype(np.uint8).squeeze()\n\nprint(\"Applying preprocessing and augmentation...\")\n\nPREPROCESSED_DIR = 'processed_images'\nos.makedirs(PREPROCESSED_DIR, exist_ok=True)\nupdated_rows = []\n\nfor idx, row in train_df.iterrows():\n    img_name = row['image']\n    label = row['level']\n    img = preprocess_image(img_name, label, IMAGE_DIR)\n    if img is None:\n        continue\n    for i in range(2):  # 2 augmentations per image\n        aug_img = augment_image(img)\n        new_name = f\"{row['image'].split('.')[0]}_aug{i}.png\"\n        cv2.imwrite(os.path.join(PREPROCESSED_DIR, new_name), aug_img)\n        updated_rows.append({'image': new_name, 'level': row['level']})\n\n#for idx, row in df.iterrows():\n#    img_name = row['image']\n#    label = row['level']\n#    img = preprocess_image(img_name, label, IMAGE_DIR)\n#    if img is not None:\n#        new_name = row['image'].replace('.jpg', '.png').replace('.jpeg', '.png')\n#        cv2.imwrite(os.path.join(PREPROCESSED_DIR, new_name), img)\n#        updated_rows.append({'image': new_name, 'level': row['level']})\n\naug_df = pd.DataFrame(updated_rows)\n\nprint(\"Done...\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-24T20:23:07.216206Z","iopub.execute_input":"2025-04-24T20:23:07.216477Z","iopub.status.idle":"2025-04-24T20:37:15.245037Z","shell.execute_reply.started":"2025-04-24T20:23:07.216456Z","shell.execute_reply":"2025-04-24T20:37:15.244345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n#  Show first 10 preprocessed images from \"PREPROCESSED_DIR\"\n# =====================\nprint(\"\\nShowing first 10 images from PREPROCESSED_DIR:\")\nplt.figure(figsize=(15, 5))\n\nfor i, row in enumerate(aug_df.head(10).itertuples(), 1):\n    img_path = os.path.join(PREPROCESSED_DIR, row.image)\n    img = cv2.imread(img_path)      #img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)\n    \n    if img is not None:\n        plt.subplot(2, 5, i)\n        plt.imshow(img)      #plt.imshow(img, cmap='gray')\n        plt.title(f\"Label: {row.level}\")\n        plt.axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-25T02:02:58.155124Z","iopub.execute_input":"2025-04-25T02:02:58.155387Z","iopub.status.idle":"2025-04-25T02:02:59.274397Z","shell.execute_reply.started":"2025-04-25T02:02:58.155367Z","shell.execute_reply":"2025-04-25T02:02:59.273504Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# STEP 3: DCGAN for Label 2, 3 & 4\n# =====================\nprint(\"Training DCGAN for synthetic image generation...\")\nLABELS_FOR_GAN = [4]\nGAN_DIR = 'synthetic_images'\nos.makedirs(GAN_DIR, exist_ok=True)\nLATENT_DIM = 100\nEPOCHS = 10\nBATCH_SIZE = 4\nSTEPS_PER_EPOCH = 160\n\n# Prepare dataset for GAN\ngan_imgs = []\nfor label in LABELS_FOR_GAN:\n    paths = aug_df[aug_df['level'] == label]['image'].tolist()\n    for path in paths[:100]:\n        img = cv2.imread(os.path.join(PREPROCESSED_DIR, path), cv2.IMREAD_COLOR)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (128, 128)).astype(np.float32)\n        img = (img / 127.5) - 1\n        gan_imgs.append(img)\n\n        #img = cv2.imread(os.path.join(PREPROCESSED_DIR, path), cv2.IMREAD_GRAYSCALE)\n        #img = cv2.resize(img, (128, 128)).astype(np.float32)\n        #img = (img / 127.5) - 1\n        #gan_imgs.append(img.reshape(128, 128, 1))\n\ngan_imgs = np.array(gan_imgs)\n\n# Define DCGAN\n\ndef build_generator():\n    model = models.Sequential([\n        layers.Dense(8*8*256, input_dim=LATENT_DIM),\n        layers.Reshape((8, 8, 256)),\n        layers.BatchNormalization(),\n        layers.LeakyReLU(),\n        layers.Conv2DTranspose(128, 5, strides=2, padding='same'),\n        layers.BatchNormalization(),\n        layers.LeakyReLU(),\n        layers.Conv2DTranspose(64, 5, strides=2, padding='same'),\n        layers.BatchNormalization(),\n        layers.LeakyReLU(),\n        layers.Conv2DTranspose(32, 5, strides=2, padding='same'),\n        layers.BatchNormalization(),\n        layers.LeakyReLU(),\n        layers.Conv2DTranspose(3, 5, strides=2, padding='same', activation='tanh')      #layers.Conv2DTranspose(1, 5, strides=2, padding='same', activation='tanh')\n    ])\n    return model\n\ndef build_discriminator():\n    model = models.Sequential([\n        layers.Conv2D(64, 5, strides=2, padding='same', input_shape=(128, 128, 3)),      #layers.Conv2D(64, 5, strides=2, padding='same', input_shape=(128, 128, 1)),\n        layers.LeakyReLU(0.2),\n        layers.Dropout(0.3),\n        layers.Conv2D(128, 5, strides=2, padding='same'),\n        layers.LeakyReLU(0.2),\n        layers.Dropout(0.3),\n        layers.Flatten(),\n        layers.Dense(1, activation='sigmoid')\n    ])\n    return model\n\ngenerator = build_generator()\ndiscriminator = build_discriminator()\noptimizer = tf.keras.optimizers.Adam(0.0002, beta_1=0.5)\ndiscriminator.compile(loss='binary_crossentropy', optimizer=optimizer)\ndiscriminator.trainable = False\n\nnoise_input = layers.Input(shape=(LATENT_DIM,))\nfake_img = generator(noise_input)\nvalidity = discriminator(fake_img)\ncombined = models.Model(noise_input, validity)\ncombined.compile(loss='binary_crossentropy', optimizer=optimizer)\n\n# Train GAN\nprint(\"Training DCGAN...\")\nfor epoch in range(EPOCHS):\n    for step in range(STEPS_PER_EPOCH):\n        idx = np.random.randint(0, gan_imgs.shape[0], BATCH_SIZE)\n        real_imgs = gan_imgs[idx]\n        noise = np.random.normal(0, 1, (BATCH_SIZE, LATENT_DIM))\n        gen_imgs = generator(noise, training=False).numpy()      #gen_imgs = generator.predict(noise)          \n\n        d_loss_real = discriminator.train_on_batch(real_imgs, np.ones((BATCH_SIZE, 1)))\n        d_loss_fake = discriminator.train_on_batch(gen_imgs, np.zeros((BATCH_SIZE, 1)))\n        d_loss = 0.5 * float(np.add(d_loss_real, d_loss_fake))\n\n        #garbage collect\n        del real_imgs, gen_imgs\n        gc.collect()\n    \n        noise = np.random.normal(0, 1, (BATCH_SIZE, LATENT_DIM))\n        g_loss = combined.train_on_batch(noise, np.ones((BATCH_SIZE, 1)))\n        if isinstance(g_loss, (list, tuple, np.ndarray)):\n            g_loss = g_loss[0]\n\n        #garbage collect\n        gc.collect()\n    \n        if step == 150:    #if step % 150 == 0:\n            print(f\"Epoch {epoch+1}/{EPOCHS}, Step {step}/{STEPS_PER_EPOCH}, D Loss: {d_loss:.4f}, G Loss: {g_loss:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-25T05:03:52.079526Z","iopub.execute_input":"2025-04-25T05:03:52.080141Z","iopub.status.idle":"2025-04-25T06:37:53.724225Z","shell.execute_reply.started":"2025-04-25T05:03:52.080115Z","shell.execute_reply":"2025-04-25T06:37:53.712165Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save synthetic images\nfor label in LABELS_FOR_GAN:\n    print(f\"Generating synthetic images for label {label}\")\n    for i in range(50):\n        noise = np.random.normal(0, 1, (1, LATENT_DIM))\n        gen_img = generator.predict(noise)[0]\n        \n        gen_img = ((gen_img + 1) * 127.5).astype(np.uint8)\n        filename = f\"synthetic_label{label}_{i}.png\"\n        path = os.path.join(GAN_DIR, filename)\n        gen_img_bgr = cv2.cvtColor(gen_img, cv2.COLOR_RGB2BGR)  # convert back to BGR for cv2.imwrite\n        cv2.imwrite(path, gen_img_bgr)\n\n        \n        #gen_img = ((gen_img + 1) * 127.5).astype(np.uint8)\n        #filename = f\"synthetic_label{label}_{i}.png\"\n        #path = os.path.join(GAN_DIR, filename)\n        #cv2.imwrite(path, gen_img.squeeze())\n        \n        aug_df = pd.concat(\n            [aug_df, pd.DataFrame([{'image': filename, 'level': label}])],\n            ignore_index=True\n        )\n\nprint(\"New label distribution after GAN:\")\nprint(aug_df['level'].value_counts())\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-25T06:38:01.225420Z","iopub.execute_input":"2025-04-25T06:38:01.225721Z","iopub.status.idle":"2025-04-25T06:38:04.684863Z","shell.execute_reply.started":"2025-04-25T06:38:01.225695Z","shell.execute_reply":"2025-04-25T06:38:04.684206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# Show first 10 preprocessed images from \"GAN_DIR\"\n# =====================\nprint(\"\\nShowing first 10 images from GAN_DIR:\")\nplt.figure(figsize=(15, 5))\n\n# Filter aug_df to only include synthetic images\nsynthetic_df = aug_df[aug_df['image'].str.startswith('synthetic_label')]\nprint(f\"Found {len(synthetic_df)} synthetic images in aug_df:\")\nprint(synthetic_df[['image', 'level']].head(10))\n\n# Debug: Verify GAN_DIR\nprint(f\"GAN_DIR: {GAN_DIR}\")\n\nfor i, row in enumerate(synthetic_df.head(10).itertuples(), 1):\n    img_path = os.path.join(GAN_DIR, row.image)\n    \n    print(f\"Attempting to load image: {img_path}\")\n    \n    if not os.path.exists(img_path):\n        print(f\"File not found: {img_path}\")\n        continue\n    \n    # Load the image in grayscale and resize to 224x224 for consistency\n    img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)\n    \n    if img is not None:\n        img = cv2.resize(img, (224, 224))  # Resize to match CNN input\n        plt.subplot(2, 5, i)\n        plt.imshow(img, cmap='gray')\n        plt.title(f\"Label: {row.level}\")\n        plt.axis('off')\n    else:\n        print(f\"Failed to load image: {img_path}\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-25T06:38:06.715410Z","iopub.execute_input":"2025-04-25T06:38:06.716178Z","iopub.status.idle":"2025-04-25T06:38:07.609907Z","shell.execute_reply.started":"2025-04-25T06:38:06.716149Z","shell.execute_reply":"2025-04-25T06:38:07.608774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_images_and_labels(df, *dirs):\n    images, labels = [], []\n    for _, row in df.iterrows():\n        filename = row['image']\n        label = row['level']\n        \n        # Try loading from each directory in order\n        for directory in dirs:\n            img_path = os.path.join(directory, filename)\n            if os.path.exists(img_path):\n                img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)\n                if img is None:\n                    continue\n                img = cv2.resize(img, (224, 224))\n                images.append(img)\n                labels.append(label)\n                break  # Stop after first successful load\n\n    X = np.array(images).reshape(-1, 224, 224, 1) / 255.0\n    y = to_categorical(labels, num_classes=5)\n    return X, y\n\n\n# =====================\n# STEP 4: Train Model\n# =====================\ndef residual_block(x, filters):\n    shortcut = x\n    x = layers.Conv2D(filters, 3, padding='same', activation='relu')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Conv2D(filters, 3, padding='same')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Add()([shortcut, x])\n    x = layers.ReLU()(x)\n    return x\n\ndef build_cnn_model():\n    inputs = layers.Input(shape=(224, 224, 1))\n\n    x = layers.Conv2D(32, kernel_size=3, padding='same', activation='relu')(inputs)\n    x = residual_block(x, 32)\n    x = layers.MaxPooling2D(pool_size=2)(x)\n\n    x = layers.Conv2D(64, kernel_size=3, padding='same', activation='relu')(x)\n    x = residual_block(x, 64)\n    x = layers.MaxPooling2D(pool_size=2)(x)\n\n    x = layers.Conv2D(128, kernel_size=3, padding='same', activation='relu')(x)\n    x = residual_block(x, 128)\n    x = layers.GlobalAveragePooling2D()(x)\n\n    x = layers.Dense(128, activation='relu')(x)\n    x = layers.Dropout(0.5)(x)\n    outputs = layers.Dense(5, activation='softmax')(x)\n\n    model = models.Model(inputs=inputs, outputs=outputs)\n    return model\n\nX, y = load_images_and_labels(aug_df, PREPROCESSED_DIR, GAN_DIR)\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=SEED, stratify=y)\n\nmodel = build_cnn_model()\nmodel.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),\n              loss='categorical_crossentropy',\n              metrics=['accuracy'])\n\ncallbacks = [\n    tf.keras.callbacks.EarlyStopping(patience=15, restore_best_weights=True),\n    tf.keras.callbacks.ReduceLROnPlateau(factor=0.1, patience=10)\n]\n\nhistory = model.fit(X_train, y_train, epochs=50, validation_data=(X_val, y_val), batch_size=32, callbacks=callbacks)\n\nmodel.save(\"best_model.keras\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# STEP 5: Confusion Matrix & Report\n# =====================\nX_test, y_test = load_images_and_labels(test_df, IMAGE_DIR)  # original split from step 1\ny_pred = np.argmax(model.predict(X_test), axis=1)\ny_true = np.argmax(y_test, axis=1)\n\n# Plot Confusion Matrix\nfrom sklearn.metrics import ConfusionMatrixDisplay\nplt.figure(figsize=(8, 6))\nConfusionMatrixDisplay.from_predictions(y_true, y_pred, cmap=plt.cm.Blues, display_labels=[0,1,2,3,4])\nplt.title(\"Confusion Matrix on Original Test Set\")\nplt.grid(False)\nplt.show()\n\nprint(\"\\nClassification Report:\")\nprint(classification_report(y_true, y_pred))\n\n# =====================\n# STEP 5.1: Plot Training & Validation Accuracy/Loss\n# =====================\nplt.figure(figsize=(12, 5))\n\nplt.subplot(1, 2, 1)\nplt.plot(history.history['accuracy'], label='Train Acc')\nplt.plot(history.history['val_accuracy'], label='Val Acc')\nplt.title('Model Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.grid(True)\n\nplt.subplot(1, 2, 2)\nplt.plot(history.history['loss'], label='Train Loss')\nplt.plot(history.history['val_loss'], label='Val Loss')\nplt.title('Model Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.grid(True)\n\nplt.tight_layout()\nplt.show()\n\n# =====================\n# STEP 6: Visualize Predictions\n# =====================\nprint(\"\\nSample predictions from each class:\")\nclass_samples = {i: 0 for i in range(5)}\nshown = 0\n\nfor i in range(len(X_test)):\n    label = y_true[i]\n    if class_samples[label] < 4:\n        pred = y_pred[i]\n        plt.imshow(X_test[i].reshape(224, 224), cmap='gray')\n        plt.title(f\"Actual: {label}, Predicted: {pred}\")\n        plt.axis('off')\n        plt.show()\n        class_samples[label] += 1\n        shown += 1\n    if shown == 20:\n        break\n\nprint(\"Done.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}