{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":10338,"databundleVersionId":862042,"sourceType":"competition"},{"sourceId":6717213,"sourceType":"datasetVersion","datasetId":1317048},{"sourceId":9681947,"sourceType":"datasetVersion","datasetId":5918106}],"dockerImageVersionId":30805,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"dataset_path = '/kaggle/input/chest-x-ray-dataset-4-categories/Chest X_Ray Dataset/'\n\nimage_paths = []\ncategories = []\n\ncategories_list = ['NORMAL', 'PNEUMONIA', 'TURBERCULOSIS']  \n\nfor category in categories_list:\n    category_path = os.path.join(dataset_path, category)\n    \n    if os.path.exists(category_path):\n        \n        for image_name in os.listdir(category_path):\n            image_path = os.path.join(category_path, image_name)\n            image_paths.append(image_path)\n            categories.append(category)\n\ndf = pd.DataFrame({'image_path': image_paths, 'category': categories})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:35:26.591789Z","iopub.execute_input":"2025-01-13T21:35:26.592583Z","iopub.status.idle":"2025-01-13T21:35:26.673463Z","shell.execute_reply.started":"2025-01-13T21:35:26.592546Z","shell.execute_reply":"2025-01-13T21:35:26.672619Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:35:23.721798Z","iopub.execute_input":"2025-01-13T21:35:23.722497Z","iopub.status.idle":"2025-01-13T21:35:23.726158Z","shell.execute_reply.started":"2025-01-13T21:35:23.722464Z","shell.execute_reply":"2025-01-13T21:35:23.725265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:35:32.873921Z","iopub.execute_input":"2025-01-13T21:35:32.874332Z","iopub.status.idle":"2025-01-13T21:35:32.891643Z","shell.execute_reply.started":"2025-01-13T21:35:32.874297Z","shell.execute_reply":"2025-01-13T21:35:32.890991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nsns.countplot(x=df.category)\nimport matplotlib.pyplot as plt\nplt.title(\"count of category\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:35:35.879349Z","iopub.execute_input":"2025-01-13T21:35:35.879678Z","iopub.status.idle":"2025-01-13T21:35:36.198944Z","shell.execute_reply.started":"2025-01-13T21:35:35.879643Z","shell.execute_reply":"2025-01-13T21:35:36.198116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:34:35.740994Z","iopub.execute_input":"2024-12-22T08:34:35.741543Z","iopub.status.idle":"2024-12-22T08:34:35.755837Z","shell.execute_reply.started":"2024-12-22T08:34:35.741492Z","shell.execute_reply":"2024-12-22T08:34:35.754943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:34:40.04219Z","iopub.execute_input":"2024-12-22T08:34:40.042538Z","iopub.status.idle":"2024-12-22T08:34:40.047863Z","shell.execute_reply.started":"2024-12-22T08:34:40.042494Z","shell.execute_reply":"2024-12-22T08:34:40.047083Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['category'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:34:42.544404Z","iopub.execute_input":"2024-12-22T08:34:42.545265Z","iopub.status.idle":"2024-12-22T08:34:42.550908Z","shell.execute_reply.started":"2024-12-22T08:34:42.545231Z","shell.execute_reply":"2024-12-22T08:34:42.550096Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['category'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:34:45.477191Z","iopub.execute_input":"2024-12-22T08:34:45.477549Z","iopub.status.idle":"2024-12-22T08:34:45.489623Z","shell.execute_reply.started":"2024-12-22T08:34:45.477503Z","shell.execute_reply":"2024-12-22T08:34:45.488592Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"category_counts = df['category'].value_counts()\nplt.figure(figsize=(8,8))\nplt.pie(category_counts.values, labels=category_counts.index, autopct='%1.1f%%', startangle=140, colors=sns.color_palette(\"viridis\", len(category_counts)))\nplt.title(\"Proportion of Images per Category\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:34:47.913304Z","iopub.execute_input":"2024-12-22T08:34:47.913681Z","iopub.status.idle":"2024-12-22T08:34:48.039195Z","shell.execute_reply.started":"2024-12-22T08:34:47.91365Z","shell.execute_reply":"2024-12-22T08:34:48.03833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nlabel_encoder = LabelEncoder()\n\ndf['category_encoded'] = label_encoder.fit_transform(df['category'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:35:48.910041Z","iopub.execute_input":"2025-01-13T21:35:48.91066Z","iopub.status.idle":"2025-01-13T21:35:48.916488Z","shell.execute_reply.started":"2025-01-13T21:35:48.910627Z","shell.execute_reply":"2025-01-13T21:35:48.91556Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:35:49.343525Z","iopub.execute_input":"2025-01-13T21:35:49.343801Z","iopub.status.idle":"2025-01-13T21:35:49.35332Z","shell.execute_reply.started":"2025-01-13T21:35:49.343774Z","shell.execute_reply":"2025-01-13T21:35:49.352391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from imblearn.over_sampling import RandomOverSampler\nros = RandomOverSampler(random_state=42)\nX_resampled, y_resampled = ros.fit_resample(df[['image_path']], df['category_encoded'])\ndf_resampled = pd.DataFrame(X_resampled, columns=['image_path'])\ndf_resampled['category_encoded'] = y_resampled","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:35:52.941045Z","iopub.execute_input":"2025-01-13T21:35:52.941883Z","iopub.status.idle":"2025-01-13T21:35:53.277061Z","shell.execute_reply.started":"2025-01-13T21:35:52.941846Z","shell.execute_reply":"2025-01-13T21:35:53.276413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nClass distribution after oversampling:\")\nprint(df_resampled['category_encoded'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:35:53.946888Z","iopub.execute_input":"2025-01-13T21:35:53.947392Z","iopub.status.idle":"2025-01-13T21:35:53.955975Z","shell.execute_reply.started":"2025-01-13T21:35:53.947363Z","shell.execute_reply":"2025-01-13T21:35:53.955163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_resampled","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:35:57.349416Z","iopub.execute_input":"2025-01-13T21:35:57.349753Z","iopub.status.idle":"2025-01-13T21:35:57.358864Z","shell.execute_reply.started":"2025-01-13T21:35:57.349721Z","shell.execute_reply":"2025-01-13T21:35:57.357982Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import time\nimport shutil\nimport pathlib\nimport itertools\nfrom PIL import Image\n\nimport cv2\nimport seaborn as sns\nsns.set_style('darkgrid')\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, classification_report\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam, Adamax\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Activation, Dropout, BatchNormalization\nfrom tensorflow.keras import regularizers\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nprint ('check')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:36:01.778909Z","iopub.execute_input":"2025-01-13T21:36:01.779277Z","iopub.status.idle":"2025-01-13T21:36:01.786155Z","shell.execute_reply.started":"2025-01-13T21:36:01.779245Z","shell.execute_reply":"2025-01-13T21:36:01.785281Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_resampled['category_encoded'] = df_resampled['category_encoded'].astype(str)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:36:04.277519Z","iopub.execute_input":"2025-01-13T21:36:04.277858Z","iopub.status.idle":"2025-01-13T21:36:04.285132Z","shell.execute_reply.started":"2025-01-13T21:36:04.277828Z","shell.execute_reply":"2025-01-13T21:36:04.284179Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df_new, temp_df_new = train_test_split(\n    df_resampled,\n    train_size=0.8,  \n    shuffle=True,\n    random_state=42,\n    stratify=df_resampled['category_encoded']  \n)\n\nvalid_df_new, test_df_new = train_test_split(\n    temp_df_new,\n    test_size=0.5,  \n    shuffle=True,\n    random_state=42,\n    stratify=temp_df_new['category_encoded'] \n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:36:09.132298Z","iopub.execute_input":"2025-01-13T21:36:09.132626Z","iopub.status.idle":"2025-01-13T21:36:09.153257Z","shell.execute_reply.started":"2025-01-13T21:36:09.132596Z","shell.execute_reply":"2025-01-13T21:36:09.152645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nbatch_size = 16\nimg_size = (224, 224)\nchannels = 3  \nimg_shape = (img_size[0], img_size[1], channels)\n\ntr_gen = ImageDataGenerator(rescale=1./255)  \nts_gen = ImageDataGenerator(rescale=1./255)\n\ntrain_gen_new = tr_gen.flow_from_dataframe(\n    train_df_new,\n    x_col='image_path',  \n    y_col='category_encoded',     \n    target_size=img_size,\n    class_mode='sparse',  \n    color_mode='rgb', \n    shuffle=True,\n    batch_size=batch_size\n)\n\nvalid_gen_new = ts_gen.flow_from_dataframe(\n    valid_df_new,\n    x_col='image_path',  \n    y_col='category_encoded',     \n    target_size=img_size,\n    class_mode='sparse',  \n    color_mode='rgb', \n    shuffle=True,\n    batch_size=batch_size\n)\n\ntest_gen_new = ts_gen.flow_from_dataframe(\n    test_df_new,\n    x_col='image_path', \n    y_col='category_encoded',    \n    target_size=img_size,\n    class_mode='sparse',  \n    color_mode='rgb', \n    shuffle=False,  \n    batch_size=batch_size\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:36:12.829988Z","iopub.execute_input":"2025-01-13T21:36:12.830803Z","iopub.status.idle":"2025-01-13T21:36:21.965263Z","shell.execute_reply.started":"2025-01-13T21:36:12.830768Z","shell.execute_reply":"2025-01-13T21:36:21.964484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:36:21.966517Z","iopub.execute_input":"2025-01-13T21:36:21.966784Z","iopub.status.idle":"2025-01-13T21:36:21.970916Z","shell.execute_reply.started":"2025-01-13T21:36:21.966757Z","shell.execute_reply":"2025-01-13T21:36:21.970027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.applications import VGG19\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Flatten, Dropout, GlobalAveragePooling2D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n\nbase_model = VGG19(weights='imagenet', include_top=False, input_shape=img_shape)\n\nbase_model.trainable = False\n\nx = base_model.output\nx = GlobalAveragePooling2D()(x) \nx = Dense(512, activation='relu')(x) \nx = Dropout(0.5)(x)  \noutput = Dense(4, activation='softmax')(x)  \n\nmodel = Model(inputs=base_model.input, outputs=output)\n\nmodel.compile(optimizer=Adam(learning_rate=0.0001), \n              loss='sparse_categorical_crossentropy', \n              metrics=['accuracy'])\n\ncallbacks = [\n    EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True),\n    ModelCheckpoint('vgg19_best_model.keras', save_best_only=True, monitor='val_loss')\n]\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:36:21.972017Z","iopub.execute_input":"2025-01-13T21:36:21.972367Z","iopub.status.idle":"2025-01-13T21:36:23.6634Z","shell.execute_reply.started":"2025-01-13T21:36:21.972329Z","shell.execute_reply":"2025-01-13T21:36:23.662738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history = model.fit(\n    train_gen_new,\n    validation_data=valid_gen_new,\n    epochs=9,\n    callbacks=callbacks,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:35:59.988509Z","iopub.execute_input":"2024-12-22T08:35:59.989137Z","iopub.status.idle":"2024-12-22T08:55:08.511113Z","shell.execute_reply.started":"2024-12-22T08:35:59.989095Z","shell.execute_reply":"2024-12-22T08:55:08.51022Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('Model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Validation'], loc='upper left')\nplt.show()\n\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Validation'], loc='upper left')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:55:08.512373Z","iopub.execute_input":"2024-12-22T08:55:08.51274Z","iopub.status.idle":"2024-12-22T08:55:09.179931Z","shell.execute_reply.started":"2024-12-22T08:55:08.512701Z","shell.execute_reply":"2024-12-22T08:55:09.179049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_labels = test_gen_new.classes  \npredictions = model.predict(test_gen_new)  \npredicted_classes = np.argmax(predictions, axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:55:09.181036Z","iopub.execute_input":"2024-12-22T08:55:09.181318Z","iopub.status.idle":"2024-12-22T08:55:26.278656Z","shell.execute_reply.started":"2024-12-22T08:55:09.181291Z","shell.execute_reply":"2024-12-22T08:55:26.277689Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"report = classification_report(test_labels, predicted_classes, target_names=list(test_gen_new.class_indices.keys()))\nprint(report)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:55:26.279974Z","iopub.execute_input":"2024-12-22T08:55:26.280792Z","iopub.status.idle":"2024-12-22T08:55:26.299981Z","shell.execute_reply.started":"2024-12-22T08:55:26.280742Z","shell.execute_reply":"2024-12-22T08:55:26.299249Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nprint(confusion_matrix(test_labels,predicted_classes))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:55:26.300855Z","iopub.execute_input":"2024-12-22T08:55:26.301121Z","iopub.status.idle":"2024-12-22T08:55:26.307543Z","shell.execute_reply.started":"2024-12-22T08:55:26.301083Z","shell.execute_reply":"2024-12-22T08:55:26.306808Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(15, 15))\nimages, labels = next(test_gen_new)\npredictions = model.predict(images)\npredicted_classes = np.argmax(predictions, axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:55:26.308621Z","iopub.execute_input":"2024-12-22T08:55:26.308956Z","iopub.status.idle":"2024-12-22T08:55:27.022773Z","shell.execute_reply.started":"2024-12-22T08:55:26.308921Z","shell.execute_reply":"2024-12-22T08:55:27.02189Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_gen_new.class_indices.keys()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:55:27.023855Z","iopub.execute_input":"2024-12-22T08:55:27.024161Z","iopub.status.idle":"2024-12-22T08:55:27.02988Z","shell.execute_reply.started":"2024-12-22T08:55:27.024126Z","shell.execute_reply":"2024-12-22T08:55:27.028883Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\nnum_images = 49\nclass_labels=list(sorted(test_gen_new.class_indices.keys()))\nindices = np.random.choice(len(test_gen_new), num_images)\n\nplt.figure(figsize=(15, 15))\n\ny_pred = model.predict(test_gen_new)\npredicted_classes = np.argmax(y_pred, axis=1)\n\ntrue_classes = test_gen_new.classes \n\nfor i, idx in enumerate(indices):\n    plt.subplot(7, 7, i + 1)  \n    img, label = test_gen_new[idx]  \n    plt.imshow(img[0]) \n    plt.title(f'Predit: {class_labels[predicted_classes[idx]]}\\nReal: {class_labels[true_classes[idx]]}')\n    plt.axis('off') \n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:55:27.031Z","iopub.execute_input":"2024-12-22T08:55:27.03171Z","iopub.status.idle":"2024-12-22T08:55:54.108609Z","shell.execute_reply.started":"2024-12-22T08:55:27.031666Z","shell.execute_reply":"2024-12-22T08:55:54.10771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport cv2\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:55:54.110851Z","iopub.execute_input":"2024-12-22T08:55:54.111127Z","iopub.status.idle":"2024-12-22T08:55:54.115182Z","shell.execute_reply.started":"2024-12-22T08:55:54.1111Z","shell.execute_reply":"2024-12-22T08:55:54.114327Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_gradcam_heatmap(img_array, model, last_conv_layer_name='block5_conv4', pred_index=None):\n    \"\"\"\n    Generates a Grad-CAM heatmap for a single image.\n\n    Parameters:\n    -----------\n    img_array : np.array\n        Preprocessed image array with shape (1, H, W, 3).\n    model : tf.keras.Model\n        The trained model.\n    last_conv_layer_name : str\n        Name of the last convolutional layer in the model (e.g., 'block5_conv4' in VGG19).\n    pred_index : int, optional\n        Index of the class for which Grad-CAM is computed. If None, uses top predicted class.\n\n    Returns:\n    --------\n    heatmap : np.array\n        The generated heatmap.\n    \"\"\"\n    # Create a model that maps the input image to the activations\n    # of the last conv layer and the model's predictions\n    grad_model = tf.keras.models.Model(\n        inputs=model.input,\n        outputs=[\n            model.get_layer(last_conv_layer_name).output,\n            model.output\n        ]\n    )\n\n    with tf.GradientTape() as tape:\n        # Forward pass\n        conv_outputs, predictions = grad_model(img_array)\n        if pred_index is None:\n            pred_index = tf.argmax(predictions[0])\n        class_channel = predictions[:, pred_index]\n\n    # Compute gradients of the target class wrt. conv layer outputs\n    grads = tape.gradient(class_channel, conv_outputs)\n\n    # Compute the mean intensity of the gradients for each feature map\n    pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))\n\n    # Multiply each feature map by its corresponding importance\n    conv_outputs = conv_outputs[0]\n    heatmap = conv_outputs @ pooled_grads[..., tf.newaxis]\n    heatmap = tf.squeeze(heatmap)\n\n    # Apply ReLU to the heatmap to keep only positive contributions\n    heatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)\n    return heatmap.numpy()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:59:38.538641Z","iopub.execute_input":"2024-12-22T08:59:38.538981Z","iopub.status.idle":"2024-12-22T08:59:38.545935Z","shell.execute_reply.started":"2024-12-22T08:59:38.538953Z","shell.execute_reply":"2024-12-22T08:59:38.545064Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def overlay_heatmap(original_img, heatmap, alpha=0.4, colormap=cv2.COLORMAP_JET):\n    \"\"\"\n    Overlays the Grad-CAM heatmap on the original image.\n\n    Parameters:\n    -----------\n    original_img : np.array\n        Original image as a NumPy array (H, W, 3).\n    heatmap : np.array\n        Heatmap with values in [0, 1].\n    alpha : float\n        Opacity of the heatmap overlay.\n    colormap : int\n        OpenCV colormap to apply (default is JET).\n\n    Returns:\n    --------\n    overlay_img : np.array\n        Image with heatmap overlay.\n    \"\"\"\n    # Convert heatmap to range [0, 255]\n    heatmap = np.uint8(255 * heatmap)\n\n    # Apply the specified colormap\n    heatmap_colored = cv2.applyColorMap(heatmap, colormap)\n\n    # Convert from BGR to RGB (for matplotlib)\n    heatmap_colored = cv2.cvtColor(heatmap_colored, cv2.COLOR_BGR2RGB)\n\n    # Overlay the heatmap on the original image\n    overlay_img = cv2.addWeighted(original_img, 1 - alpha, heatmap_colored, alpha, 0)\n    return overlay_img\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T08:59:49.95241Z","iopub.execute_input":"2024-12-22T08:59:49.95333Z","iopub.status.idle":"2024-12-22T08:59:49.960487Z","shell.execute_reply.started":"2024-12-22T08:59:49.953263Z","shell.execute_reply":"2024-12-22T08:59:49.95959Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Assume 'model' is your trained VGG19 model\n# Assume 'test_gen_new' is your test data generator\n\n# Retrieve one batch of test images and labels\nbatch_images, batch_labels = next(test_gen_new)\n\n# Select a specific image from the batch (e.g., the first image)\nimg = batch_images[0]\nlabel = batch_labels[0]\n\n# Expand dimensions to create a batch of one\nimg_array = np.expand_dims(img, axis=0)\n\n# Specify the name of the last convolutional layer in VGG19\nlast_conv_layer_name = 'block5_conv4'  # Verify this with model.summary()\n\n# Generate Grad-CAM heatmap\nheatmap = make_gradcam_heatmap(\n    img_array=img_array,\n    model=model,\n    last_conv_layer_name=last_conv_layer_name,\n    pred_index=None  # None uses the predicted class\n)\n\n# Convert the image back to original scale (0-255) and type\noriginal_img = (img * 255).astype('uint8')\n\n# Resize heatmap to match the original image size\nheatmap_resized = cv2.resize(heatmap, (original_img.shape[1], original_img.shape[0]))\n\n# Overlay the heatmap on the original image\noverlay_img = overlay_heatmap(original_img, heatmap_resized, alpha=0.4)\n\n# Plot the original image, heatmap, and overlay\nfig, ax = plt.subplots(1, 3, figsize=(18, 6))\n\nax[0].imshow(original_img)\nax[0].set_title(\"Original Image\")\nax[0].axis(\"off\")\n\nax[1].imshow(heatmap_resized, cmap='jet')\nax[1].set_title(\"Grad-CAM Heatmap\")\nax[1].axis(\"off\")\n\nax[2].imshow(overlay_img)\nax[2].set_title(\"Overlay Image\")\nax[2].axis(\"off\")\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T09:00:01.107699Z","iopub.execute_input":"2024-12-22T09:00:01.108044Z","iopub.status.idle":"2024-12-22T09:00:05.664241Z","shell.execute_reply.started":"2024-12-22T09:00:01.108012Z","shell.execute_reply":"2024-12-22T09:00:05.663422Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport cv2\n\n# Define helper functions\ndef make_gradcam_heatmap(img_array, model, last_conv_layer_name='block5_conv4', pred_index=None):\n    \"\"\"\n    Generates a Grad-CAM heatmap for a single image.\n\n    Parameters:\n    -----------\n    img_array : np.array\n        Preprocessed image array with shape (1, H, W, 3).\n    model : tf.keras.Model\n        The trained model.\n    last_conv_layer_name : str\n        Name of the last convolutional layer in the model (e.g., 'block5_conv4' in VGG19).\n    pred_index : int, optional\n        Index of the class for which Grad-CAM is computed. If None, uses top predicted class.\n\n    Returns:\n    --------\n    heatmap : np.array\n        The generated heatmap.\n    \"\"\"\n    # Create a model that maps the input image to the activations\n    # of the last conv layer and the model's predictions\n    grad_model = tf.keras.models.Model(\n        inputs=model.input,\n        outputs=[\n            model.get_layer(last_conv_layer_name).output,\n            model.output\n        ]\n    )\n\n    with tf.GradientTape() as tape:\n        # Forward pass\n        conv_outputs, predictions = grad_model(img_array)\n        if pred_index is None:\n            pred_index = tf.argmax(predictions[0])\n        class_channel = predictions[:, pred_index]\n\n    # Compute gradients of the target class wrt. conv layer outputs\n    grads = tape.gradient(class_channel, conv_outputs)\n\n    # Compute the mean intensity of the gradients for each feature map\n    pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))\n\n    # Multiply each feature map by its corresponding importance\n    conv_outputs = conv_outputs[0]\n    heatmap = conv_outputs @ pooled_grads[..., tf.newaxis]\n    heatmap = tf.squeeze(heatmap)\n\n    # Apply ReLU to the heatmap to keep only positive contributions\n    heatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)\n    return heatmap.numpy()\n\ndef overlay_heatmap(original_img, heatmap, alpha=0.4, colormap=cv2.COLORMAP_JET):\n    \"\"\"\n    Overlays the Grad-CAM heatmap on the original image.\n\n    Parameters:\n    -----------\n    original_img : np.array\n        Original image as a NumPy array (H, W, 3).\n    heatmap : np.array\n        Heatmap with values in [0, 1].\n    alpha : float\n        Opacity of the heatmap overlay.\n    colormap : int\n        OpenCV colormap to apply (default is JET).\n\n    Returns:\n    --------\n    overlay_img : np.array\n        Image with heatmap overlay.\n    \"\"\"\n    # Convert heatmap to range [0, 255]\n    heatmap = np.uint8(255 * heatmap)\n\n    # Apply the specified colormap\n    heatmap_colored = cv2.applyColorMap(heatmap, colormap)\n\n    # Convert from BGR to RGB (for matplotlib)\n    heatmap_colored = cv2.cvtColor(heatmap_colored, cv2.COLOR_BGR2RGB)\n\n    # Overlay the heatmap on the original image\n    overlay_img = cv2.addWeighted(original_img, 1 - alpha, heatmap_colored, alpha, 0)\n    return overlay_img\n\n# Retrieve class labels from the generator\nclass_indices = test_gen_new.class_indices\n# Create a mapping from index to class label\nidx_to_class = {v: k for k, v in class_indices.items()}\n\n# Example Usage of Grad-CAM on one image from the test set\nbatch_images, batch_labels = next(test_gen_new)\n\n# Select a specific image from the batch (e.g., the first image)\nimg = batch_images[0]\nlabel = batch_labels[0]\n\n# Expand dims to create a batch of one\nimg_array = np.expand_dims(img, axis=0)\n\n# Specify the name of the last convolutional layer in VGG19\nlast_conv_layer_name = 'block5_conv4'  # Verify this with model.summary()\n\n# Generate Grad-CAM heatmap\nheatmap = make_gradcam_heatmap(\n    img_array=img_array,\n    model=model,\n    last_conv_layer_name=last_conv_layer_name,\n    pred_index=None  # None uses the predicted class\n)\n\n# Determine the predicted class\npredictions = model.predict(img_array)\npredicted_class_idx = np.argmax(predictions[0])\npredicted_class = idx_to_class[predicted_class_idx]\n\n# Get the original class label\noriginal_class = idx_to_class[int(label)]\n\n# Convert the image back to original scale (0-255) and type\noriginal_img = (img * 255).astype('uint8')\n\n# Resize heatmap to match the original image size\nheatmap_resized = cv2.resize(heatmap, (original_img.shape[1], original_img.shape[0]))\n\n# Overlay the heatmap on the original image\noverlay_img = overlay_heatmap(original_img, heatmap_resized, alpha=0.4)\n\n# Plot the original image, heatmap, and overlay with labels\nfig, ax = plt.subplots(1, 3, figsize=(18, 6))\n\n# Original Image with Original Label\nax[0].imshow(original_img)\nax[0].set_title(f\"Original Image\\nLabel: {original_class}\")\nax[0].axis(\"off\")\n\n# Grad-CAM Heatmap\nax[1].imshow(heatmap_resized, cmap='jet')\nax[1].set_title(\"Grad-CAM Heatmap\")\nax[1].axis(\"off\")\n\n# Overlay Image with Predicted Label\nax[2].imshow(overlay_img)\nax[2].set_title(f\"Overlay Image\\nPredicted: {predicted_class}\")\nax[2].axis(\"off\")\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T09:01:19.496838Z","iopub.execute_input":"2024-12-22T09:01:19.497217Z","iopub.status.idle":"2024-12-22T09:01:22.487553Z","shell.execute_reply.started":"2024-12-22T09:01:19.497183Z","shell.execute_reply":"2024-12-22T09:01:22.486692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport cv2\n\n# Define helper functions\ndef make_gradcam_heatmap(img_array, model, last_conv_layer_name='block5_conv4', pred_index=None):\n    \"\"\"\n    Generates a Grad-CAM heatmap for a single image.\n\n    Parameters:\n    -----------\n    img_array : np.array\n        Preprocessed image array with shape (1, H, W, 3).\n    model : tf.keras.Model\n        The trained model.\n    last_conv_layer_name : str\n        Name of the last convolutional layer in the model (e.g., 'block5_conv4' in VGG19).\n    pred_index : int, optional\n        Index of the class for which Grad-CAM is computed. If None, uses top predicted class.\n\n    Returns:\n    --------\n    heatmap : np.array\n        The generated heatmap.\n    \"\"\"\n    # Create a model that maps the input image to the activations\n    # of the last conv layer and the model's predictions\n    grad_model = tf.keras.models.Model(\n        inputs=model.input,\n        outputs=[\n            model.get_layer(last_conv_layer_name).output,\n            model.output\n        ]\n    )\n\n    with tf.GradientTape() as tape:\n        # Forward pass\n        conv_outputs, predictions = grad_model(img_array)\n        if pred_index is None:\n            pred_index = tf.argmax(predictions[0])\n        class_channel = predictions[:, pred_index]\n\n    # Compute gradients of the target class wrt. conv layer outputs\n    grads = tape.gradient(class_channel, conv_outputs)\n\n    # Compute the mean intensity of the gradients for each feature map\n    pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))\n\n    # Multiply each feature map by its corresponding importance\n    conv_outputs = conv_outputs[0]\n    heatmap = conv_outputs @ pooled_grads[..., tf.newaxis]\n    heatmap = tf.squeeze(heatmap)\n\n    # Apply ReLU to the heatmap to keep only positive contributions\n    heatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)\n    return heatmap.numpy()\n\ndef overlay_heatmap(original_img, heatmap, alpha=0.4, colormap=cv2.COLORMAP_JET):\n    \"\"\"\n    Overlays the Grad-CAM heatmap on the original image.\n\n    Parameters:\n    -----------\n    original_img : np.array\n        Original image as a NumPy array (H, W, 3).\n    heatmap : np.array\n        Heatmap with values in [0, 1].\n    alpha : float\n        Opacity of the heatmap overlay.\n    colormap : int\n        OpenCV colormap to apply (default is JET).\n\n    Returns:\n    --------\n    overlay_img : np.array\n        Image with heatmap overlay.\n    \"\"\"\n    # Convert heatmap to range [0, 255]\n    heatmap = np.uint8(255 * heatmap)\n\n    # Apply the specified colormap\n    heatmap_colored = cv2.applyColorMap(heatmap, colormap)\n\n    # Convert from BGR to RGB (for matplotlib)\n    heatmap_colored = cv2.cvtColor(heatmap_colored, cv2.COLOR_BGR2RGB)\n\n    # Overlay the heatmap on the original image\n    overlay_img = cv2.addWeighted(original_img, 1 - alpha, heatmap_colored, alpha, 0)\n    return overlay_img\n\n# Retrieve class labels from the generator\nclass_indices = test_gen_new.class_indices\n# Create a mapping from index to class label\nidx_to_class = {v: k for k, v in class_indices.items()}\n\n# Number of images to visualize\nnum_images = 15\n\n# Initialize counters\nimages_visualized = 0\nbatch_count = 0\n\n# Iterate through the test generator until we visualize the desired number of images\nfor batch_images, batch_labels in test_gen_new:\n    for i in range(len(batch_images)):\n        if images_visualized >= num_images:\n            break  # Exit if we've visualized enough images\n\n        img = batch_images[i]\n        label = batch_labels[i]\n\n        # Expand dims to create a batch of one\n        img_array = np.expand_dims(img, axis=0)\n\n        # Specify the name of the last convolutional layer in VGG19\n        last_conv_layer_name = 'block5_conv4'  # Verify this with model.summary()\n\n        # Generate Grad-CAM heatmap\n        heatmap = make_gradcam_heatmap(\n            img_array=img_array,\n            model=model,\n            last_conv_layer_name=last_conv_layer_name,\n            pred_index=None  # None uses the predicted class\n        )\n\n        # Determine the predicted class\n        predictions = model.predict(img_array)\n        predicted_class_idx = np.argmax(predictions[0])\n        predicted_class = idx_to_class[predicted_class_idx]\n\n        # Get the original class label\n        original_class = idx_to_class[int(label)]\n\n        # Convert the image back to original scale (0-255) and type\n        original_img = (img * 255).astype('uint8')\n\n        # Resize heatmap to match the original image size\n        heatmap_resized = cv2.resize(heatmap, (original_img.shape[1], original_img.shape[0]))\n\n        # Overlay the heatmap on the original image\n        overlay_img = overlay_heatmap(original_img, heatmap_resized, alpha=0.4)\n\n        # Plot the original image, heatmap, and overlay with labels\n        fig, ax = plt.subplots(1, 3, figsize=(18, 6))\n\n        # Original Image with Original Label\n        ax[0].imshow(original_img)\n        ax[0].set_title(f\"Original Image\\nLabel: {original_class}\")\n        ax[0].axis(\"off\")\n\n        # Grad-CAM Heatmap\n        ax[1].imshow(heatmap_resized, cmap='jet')\n        ax[1].set_title(\"Grad-CAM Heatmap\")\n        ax[1].axis(\"off\")\n\n        # Overlay Image with Predicted Label\n        ax[2].imshow(overlay_img)\n        ax[2].set_title(f\"Overlay Image\\nPredicted: {predicted_class}\")\n        ax[2].axis(\"off\")\n\n        plt.tight_layout()\n        plt.show()\n\n        images_visualized += 1\n\n    batch_count += 1\n    if images_visualized >= num_images:\n        break  # Exit the outer loop as well\n\n# Reset the test generator for future use\ntest_gen_new.reset()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T09:03:10.125769Z","iopub.execute_input":"2024-12-22T09:03:10.12617Z","iopub.status.idle":"2024-12-22T09:03:26.880949Z","shell.execute_reply.started":"2024-12-22T09:03:10.126133Z","shell.execute_reply":"2024-12-22T09:03:26.880071Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# VGG 16","metadata":{}},{"cell_type":"code","source":"import time\nimport shutil\nimport pathlib\nimport itertools\nimport numpy as np\nimport pandas as pd  # Assuming you are using pandas for df_resampled\nfrom PIL import Image\n\nimport cv2\nimport seaborn as sns\nsns.set_style('darkgrid')\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, classification_report\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom tensorflow.keras.layers import Dense, Dropout, GlobalAveragePooling1D\nfrom tensorflow.keras.models import Model\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Import Vision Transformer from vit-keras\nfrom vit_keras import vit\n\n# ---------------------------\n# Data Preparation\n# ---------------------------\n\n# Ensure df_resampled is defined before this point\n# For example:\n# df_resampled = pd.read_csv('your_data.csv')\n\nprint('Check')\n\n# Encode the category column as string\ndf_resampled['category_encoded'] = df_resampled['category_encoded'].astype(str)\n\n# Split the data into training, validation, and test sets\ntrain_df_new, temp_df_new = train_test_split(\n    df_resampled,\n    train_size=0.8,  \n    shuffle=True,\n    random_state=42,\n    stratify=df_resampled['category_encoded']  \n)\n\nvalid_df_new, test_df_new = train_test_split(\n    temp_df_new,\n    test_size=0.5,  \n    shuffle=True,\n    random_state=42,\n    stratify=temp_df_new['category_encoded'] \n)\n\n# ---------------------------\n# Data Generators\n# ---------------------------\n\n# Define image parameters\nbatch_size = 16\nimg_size = (224, 224)  # ViT models often use 224x224\nchannels = 3  \nimg_shape = (img_size[0], img_size[1], channels)\n\n# Define ImageDataGenerators with data augmentation for training\ntr_gen = ImageDataGenerator(\n    rescale=1./255,\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    shear_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    fill_mode='nearest'\n)  \n\nts_gen = ImageDataGenerator(rescale=1./255)\n\n# Create data generators from dataframes\ntrain_gen_new = tr_gen.flow_from_dataframe(\n    train_df_new,\n    x_col='image_path',  \n    y_col='category_encoded',     \n    target_size=img_size,\n    class_mode='sparse',  \n    color_mode='rgb', \n    shuffle=True,\n    batch_size=batch_size\n)\n\nvalid_gen_new = ts_gen.flow_from_dataframe(\n    valid_df_new,\n    x_col='image_path',  \n    y_col='category_encoded',     \n    target_size=img_size,\n    class_mode='sparse',  \n    color_mode='rgb', \n    shuffle=True,\n    batch_size=batch_size\n)\n\ntest_gen_new = ts_gen.flow_from_dataframe(\n    test_df_new,\n    x_col='image_path', \n    y_col='category_encoded',    \n    target_size=img_size,\n    class_mode='sparse',  \n    color_mode='rgb', \n    shuffle=False,  \n    batch_size=batch_size\n)\n\n# ---------------------------\n# Model Definition\n# ---------------------------\n\n# Load the pre-trained Vision Transformer model\n# Here, we use ViT-B/16. Adjust the parameters as needed.\nvit_model = vit.vit_b16(\n    image_size=img_size[0],\n    activation='softmax',\n    pretrained=True,\n    include_top=False,\n    pretrained_top=False\n)\n\n# Freeze the ViT base model\nvit_model.trainable = False\n\n# Add custom classification head\nx = vit_model.output\nx = GlobalAveragePooling1D()(x)  # Aggregates the sequence of embeddings\nx = Dense(512, activation='relu')(x)\nx = Dropout(0.5)(x)\noutput = Dense(len(train_gen_new.class_indices), activation='softmax')(x)\n\n# Define the complete model\nmodel = Model(inputs=vit_model.input, outputs=output)\n\n# Compile the model with appropriate optimizer, loss, and metrics\nmodel.compile(optimizer=keras.optimizers.Adam(learning_rate=0.0001), \n              loss='sparse_categorical_crossentropy', \n              metrics=['accuracy'])\n\n# Define callbacks for early stopping and saving the best model\ncallbacks = [\n    EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True),\n    ModelCheckpoint('vit_best_model.keras', save_best_only=True, monitor='val_loss')\n]\n\n# ---------------------------\n# Model Training\n# ---------------------------\n\n# Train the model\nhistory = model.fit(\n    train_gen_new,\n    validation_data=valid_gen_new,\n    epochs=20,  # Adjust as needed\n    callbacks=callbacks,\n)\n\n# ---------------------------\n# Visualization of Training History\n# ---------------------------\n\n# Plot training & validation accuracy and loss values\nplt.figure(figsize=(12, 5))\n\n# Accuracy plot\nplt.subplot(1, 2, 1)\nplt.plot(history.history['accuracy'], marker='o', label='Train Accuracy')\nplt.plot(history.history['val_accuracy'], marker='o', label='Validation Accuracy')\nplt.title('Model Accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(loc='upper left')\n\n# Loss plot\nplt.subplot(1, 2, 2)\nplt.plot(history.history['loss'], marker='o', label='Train Loss')\nplt.plot(history.history['val_loss'], marker='o', label='Validation Loss')\nplt.title('Model Loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(loc='upper left')\n\nplt.tight_layout()\nplt.show()\n\n# ---------------------------\n# Model Evaluation\n# ---------------------------\n\n# Evaluate the model on the test set\ntest_labels = test_gen_new.classes  \npredictions = model.predict(test_gen_new)  \npredicted_classes = np.argmax(predictions, axis=1)\n\n# Generate classification report\nreport = classification_report(\n    test_labels, \n    predicted_classes, \n    target_names=list(test_gen_new.class_indices.keys())\n)\nprint(\"Classification Report:\\n\", report)\n\n# Generate confusion matrix\nconf_matrix = confusion_matrix(test_labels, predicted_classes)\nprint(\"Confusion Matrix:\\n\", conf_matrix)\n\n# Plot confusion matrix using seaborn heatmap\nplt.figure(figsize=(10, 8))\nsns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues',\n            xticklabels=list(test_gen_new.class_indices.keys()),\n            yticklabels=list(test_gen_new.class_indices.keys()))\nplt.ylabel('Actual')\nplt.xlabel('Predicted')\nplt.title('Confusion Matrix')\nplt.show()\n\n# ---------------------------\n# Visualize Some Predictions\n# ---------------------------\n\nnum_images = 49\nclass_labels = list(sorted(test_gen_new.class_indices.keys()))\nindices = np.random.choice(len(test_gen_new), num_images)\n\nplt.figure(figsize=(15, 15))\n\ny_pred = model.predict(test_gen_new)\npredicted_classes = np.argmax(y_pred, axis=1)\n\ntrue_classes = test_gen_new.classes \n\nfor i, idx in enumerate(indices):\n    img, label = test_gen_new[idx]  \n    plt.subplot(7, 7, i + 1)  \n    plt.imshow(img[0]) \n    plt.title(f'Pred: {class_labels[predicted_classes[idx]]}\\nTrue: {class_labels[true_classes[idx]]}', fontsize=8)\n    plt.axis('off') \n\nplt.tight_layout()\nplt.show()\n\n# ---------------------------\n# Explainability: Attention Map Visualization\n# ---------------------------\n\ndef visualize_attention(model, img, class_labels, last_layer='transformer'):\n    \"\"\"\n    Visualizes the attention maps from the Vision Transformer model.\n    \n    Parameters:\n    -----------\n    model : tf.keras.Model\n        The trained model.\n    img : np.array\n        Preprocessed image array with shape (1, H, W, 3).\n    class_labels : list\n        List of class names.\n    last_layer : str\n        Name of the last transformer layer to extract attention from.\n    \"\"\"\n    import tensorflow as tf\n\n    # Create a sub-model to extract attention weights\n    attention_model = Model(inputs=model.input, \n                            outputs=[layer.output for layer in model.layers if 'MultiHeadAttention' in layer.name])\n\n    # Get the attention weights\n    attention_weights = attention_model.predict(img)\n    \n    # Select the last attention layer\n    last_attention = attention_weights[-1]  # Shape: (batch_size, num_heads, seq_length, seq_length)\n    num_heads = last_attention.shape[1]\n    \n    # Average the attention weights across all heads\n    avg_attention = np.mean(last_attention, axis=1)[0]  # Shape: (seq_length, seq_length)\n    \n    # Exclude the [CLS] token\n    avg_attention = avg_attention[1:, 1:]\n    \n    # Reshape to 2D (patches)\n    grid_size = int(np.sqrt(avg_attention.shape[0]))\n    attention_map = avg_attention.reshape(grid_size, grid_size)\n    \n    # Resize attention map to match the image size\n    attention_map = cv2.resize(attention_map, img_size, interpolation=cv2.INTER_LINEAR)\n    \n    # Normalize the attention map\n    attention_map = (attention_map - attention_map.min()) / (attention_map.max() - attention_map.min())\n    \n    # Convert image back to original scale\n    original_img = (img[0] * 255).astype('uint8')\n    \n    # Apply heatmap\n    heatmap = cv2.applyColorMap(np.uint8(255 * attention_map), cv2.COLORMAP_JET)\n    heatmap = cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB)\n    \n    # Overlay the heatmap on the original image\n    overlay_img = cv2.addWeighted(original_img, 0.6, heatmap, 0.4, 0)\n    \n    # Plot the results\n    plt.figure(figsize=(15, 5))\n    \n    plt.subplot(1, 3, 1)\n    plt.imshow(original_img)\n    plt.title(\"Original Image\")\n    plt.axis('off')\n    \n    plt.subplot(1, 3, 2)\n    plt.imshow(attention_map, cmap='jet')\n    plt.title(\"Attention Map\")\n    plt.axis('off')\n    \n    plt.subplot(1, 3, 3)\n    plt.imshow(overlay_img)\n    plt.title(\"Overlay Image\")\n    plt.axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n\n# Example Attention Map Visualization\ndef visualize_sample_attention(model, test_gen, class_labels):\n    \"\"\"\n    Visualizes attention maps for a random sample from the test generator.\n    \n    Parameters:\n    -----------\n    model : tf.keras.Model\n        The trained ViT model.\n    test_gen : DirectoryIterator\n        The test data generator.\n    class_labels : list\n        List of class names.\n    \"\"\"\n    # Retrieve one batch of test images and labels\n    batch_images, batch_labels = next(test_gen)\n\n    # Select a specific image from the batch (e.g., the first image)\n    img = batch_images[0]\n    label = batch_labels[0]\n\n    # Expand dimensions to create a batch of one\n    img_array = np.expand_dims(img, axis=0)\n\n    # Predict the class\n    prediction = model.predict(img_array)\n    predicted_class = np.argmax(prediction, axis=1)[0]\n    true_class = label\n\n    print(f\"Predicted: {class_labels[predicted_class]}, True: {class_labels[true_class]}\")\n\n    # Visualize attention\n    visualize_attention(model, img_array, class_labels)\n\n# Call the attention visualization function on a sample\nvisualize_sample_attention(model, test_gen_new, class_labels)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T21:36:31.522815Z","iopub.execute_input":"2025-01-13T21:36:31.523138Z","iopub.status.idle":"2025-01-13T21:36:36.370247Z","shell.execute_reply.started":"2025-01-13T21:36:31.523092Z","shell.execute_reply":"2025-01-13T21:36:36.369111Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom PIL import Image\nimport tensorflow as tf\nfrom tensorflow.keras.applications import VGG16\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Dropout\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport gradio as gr\nimport lime\nfrom lime import lime_image\nfrom skimage.segmentation import mark_boundaries\n\n# Set random seed for reproducibility\ntf.random.set_seed(42)\nnp.random.seed(42)\n\n# Paths\nNORMAL_PATH = '/kaggle/input/chest-x-ray-dataset-4-categories/Chest X_Ray Dataset/NORMAL'\nPNEUMONIA_PATH = '/kaggle/input/chest-x-ray-dataset-4-categories/Chest X_Ray Dataset/PNEUMONIA'\n\ndef load_and_preprocess_image(path, target_size=(224, 224)):\n    \"\"\"Load and preprocess a single image\"\"\"\n    img = Image.open(path).convert('RGB')\n    img = img.resize(target_size)\n    img = np.array(img) / 255.0\n    return img\n\ndef plot_sample_images():\n    \"\"\"Plot sample images from both classes\"\"\"\n    plt.figure(figsize=(15, 6))\n    \n    # Plot normal samples\n    normal_samples = os.listdir(NORMAL_PATH)[:3]\n    for i, img_name in enumerate(normal_samples, 1):\n        plt.subplot(2, 3, i)\n        img = load_and_preprocess_image(os.path.join(NORMAL_PATH, img_name))\n        plt.imshow(img)\n        plt.title('Normal')\n        plt.axis('off')\n    \n    # Plot pneumonia samples\n    pneumonia_samples = os.listdir(PNEUMONIA_PATH)[:3]\n    for i, img_name in enumerate(pneumonia_samples, 1):\n        plt.subplot(2, 3, i+3)\n        img = load_and_preprocess_image(os.path.join(PNEUMONIA_PATH, img_name))\n        plt.imshow(img)\n        plt.title('Pneumonia')\n        plt.axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n\ndef create_model():\n    \"\"\"Create and compile the VGG16 model\"\"\"\n    base_model = VGG16(weights='imagenet', include_top=False, input_shape=(224, 224, 3))\n    \n    # Freeze the base model layers\n    for layer in base_model.layers:\n        layer.trainable = False\n    \n    # Add custom layers\n    x = base_model.output\n    x = GlobalAveragePooling2D()(x)\n    x = Dense(512, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    predictions = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inputs=base_model.input, outputs=predictions)\n    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n    \n    return model\n\ndef create_data_generators():\n    \"\"\"Create training and validation data generators with augmentation\"\"\"\n    train_datagen = ImageDataGenerator(\n        rescale=1./255,\n        rotation_range=20,\n        width_shift_range=0.2,\n        height_shift_range=0.2,\n        shear_range=0.2,\n        zoom_range=0.2,\n        horizontal_flip=True,\n        validation_split=0.2\n    )\n    \n    train_generator = train_datagen.flow_from_directory(\n        '/kaggle/input/chest-x-ray-dataset-4-categories/Chest X_Ray Dataset',\n        target_size=(224, 224),\n        batch_size=32,\n        class_mode='binary',\n        subset='training',\n        classes=['NORMAL', 'PNEUMONIA']\n    )\n    \n    validation_generator = train_datagen.flow_from_directory(\n        '/kaggle/input/chest-x-ray-dataset-4-categories/Chest X_Ray Dataset',\n        target_size=(224, 224),\n        batch_size=32,\n        class_mode='binary',\n        subset='validation',\n        classes=['NORMAL', 'PNEUMONIA']\n    )\n    \n    return train_generator, validation_generator\n\ndef train_model(model, train_generator, validation_generator):\n    \"\"\"Train the model\"\"\"\n    history = model.fit(\n        train_generator,\n        epochs=10,\n        validation_data=validation_generator,\n        callbacks=[\n            tf.keras.callbacks.EarlyStopping(patience=3, restore_best_weights=True)\n        ]\n    )\n    return history\n\ndef plot_training_history(history):\n    \"\"\"Plot training history\"\"\"\n    plt.figure(figsize=(12, 4))\n    \n    plt.subplot(1, 2, 1)\n    plt.plot(history.history['loss'], label='Training Loss')\n    plt.plot(history.history['val_loss'], label='Validation Loss')\n    plt.title('Model Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend()\n    \n    plt.subplot(1, 2, 2)\n    plt.plot(history.history['accuracy'], label='Training Accuracy')\n    plt.plot(history.history['val_accuracy'], label='Validation Accuracy')\n    plt.title('Model Accuracy')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    \n    plt.tight_layout()\n    plt.show()\n\ndef predict_and_visualize(model, num_samples=5):\n    \"\"\"Make predictions and visualize results\"\"\"\n    # Get random samples\n    normal_samples = np.random.choice(os.listdir(NORMAL_PATH), num_samples)\n    pneumonia_samples = np.random.choice(os.listdir(PNEUMONIA_PATH), num_samples)\n    \n    plt.figure(figsize=(15, 10))\n    \n    # Plot normal samples\n    for i, img_name in enumerate(normal_samples):\n        img_path = os.path.join(NORMAL_PATH, img_name)\n        img = load_and_preprocess_image(img_path)\n        pred = model.predict(np.expand_dims(img, axis=0))[0][0]\n        \n        plt.subplot(2, num_samples, i+1)\n        plt.imshow(img)\n        plt.title(f'True: Normal\\nPred: {\"Pneumonia\" if pred > 0.5 else \"Normal\"}\\n{pred:.2f}')\n        plt.axis('off')\n    \n    # Plot pneumonia samples\n    for i, img_name in enumerate(pneumonia_samples):\n        img_path = os.path.join(PNEUMONIA_PATH, img_name)\n        img = load_and_preprocess_image(img_path)\n        pred = model.predict(np.expand_dims(img, axis=0))[0][0]\n        \n        plt.subplot(2, num_samples, num_samples+i+1)\n        plt.imshow(img)\n        plt.title(f'True: Pneumonia\\nPred: {\"Pneumonia\" if pred > 0.5 else \"Normal\"}\\n{pred:.2f}')\n        plt.axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n\ndef explain_prediction(model, img_path):\n    \"\"\"Generate LIME explanation for a prediction\"\"\"\n    explainer = lime_image.LimeImageExplainer()\n    \n    img = load_and_preprocess_image(img_path)\n    explanation = explainer.explain_instance(\n        img,\n        lambda x: model.predict(x),\n        top_labels=1,\n        hide_color=0,\n        num_samples=1000\n    )\n    \n    # Get the image and mask\n    temp, mask = explanation.get_image_and_mask(\n        explanation.top_labels[0],\n        positive_only=True,\n        num_features=5,\n        hide_rest=True\n    )\n    \n    # Plot the explanation\n    plt.figure(figsize=(10, 5))\n    \n    plt.subplot(1, 2, 1)\n    plt.imshow(img)\n    plt.title('Original Image')\n    plt.axis('off')\n    \n    plt.subplot(1, 2, 2)\n    plt.imshow(mark_boundaries(temp / 2 + 0.5, mask))\n    plt.title('LIME Explanation')\n    plt.axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n\ndef main():\n    # 1. EDA\n    print(\"Performing EDA...\")\n    plot_sample_images()\n    \n    # 2. Model Training\n    print(\"\\nCreating and training model...\")\n    model = create_model()\n    train_generator, validation_generator = create_data_generators()\n    history = train_model(model, train_generator, validation_generator)\n    plot_training_history(history)\n    \n    # 3. Model Inference\n    print(\"\\nMaking predictions...\")\n    predict_and_visualize(model)\n    \n    # 4. Model Explainability\n    print(\"\\nGenerating explanations...\")\n    # Generate explanation for one normal and one pneumonia case\n    normal_sample = os.path.join(NORMAL_PATH, os.listdir(NORMAL_PATH)[0])\n    pneumonia_sample = os.path.join(PNEUMONIA_PATH, os.listdir(PNEUMONIA_PATH)[0])\n    \n    explain_prediction(model, normal_sample)\n    explain_prediction(model, pneumonia_sample)\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T19:52:03.786428Z","iopub.execute_input":"2025-01-14T19:52:03.786785Z","iopub.status.idle":"2025-01-14T20:11:45.968053Z","shell.execute_reply.started":"2025-01-14T19:52:03.786731Z","shell.execute_reply":"2025-01-14T20:11:45.967073Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom PIL import Image\nimport tensorflow as tf\nfrom tensorflow.keras.applications import VGG16\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Dropout\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport lime\nfrom lime import lime_image\nfrom skimage.segmentation import mark_boundaries\nimport shap\nfrom matplotlib.colors import LinearSegmentedColormap\nfrom scipy.optimize import differential_evolution\nimport random\n\n# Set random seed for reproducibility\ntf.random.set_seed(42)\nnp.random.seed(42)\n\n# Paths\nNORMAL_PATH = '/kaggle/input/chest-x-ray-dataset-4-categories/Chest X_Ray Dataset/NORMAL'\nPNEUMONIA_PATH = '/kaggle/input/chest-x-ray-dataset-4-categories/Chest X_Ray Dataset/PNEUMONIA'\n\ndef load_and_preprocess_image(path, target_size=(224, 224)):\n    \"\"\"Load and preprocess a single image\"\"\"\n    img = Image.open(path).convert('RGB')\n    img = img.resize(target_size)\n    img = np.array(img) / 255.0\n    return img\n\ndef plot_sample_images():\n    \"\"\"Plot sample images from both classes\"\"\"\n    plt.figure(figsize=(15, 6))\n    \n    # Plot normal samples\n    normal_samples = os.listdir(NORMAL_PATH)[:3]\n    for i, img_name in enumerate(normal_samples, 1):\n        plt.subplot(2, 3, i)\n        img = load_and_preprocess_image(os.path.join(NORMAL_PATH, img_name))\n        plt.imshow(img)\n        plt.title('Normal')\n        plt.axis('off')\n    \n    # Plot pneumonia samples\n    pneumonia_samples = os.listdir(PNEUMONIA_PATH)[:3]\n    for i, img_name in enumerate(pneumonia_samples, 1):\n        plt.subplot(2, 3, i+3)\n        img = load_and_preprocess_image(os.path.join(PNEUMONIA_PATH, img_name))\n        plt.imshow(img)\n        plt.title('Pneumonia')\n        plt.axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n\ndef create_model():\n    \"\"\"Create and compile the VGG16 model\"\"\"\n    base_model = VGG16(weights='imagenet', include_top=False, input_shape=(224, 224, 3))\n    \n    # Freeze the base model layers\n    for layer in base_model.layers:\n        layer.trainable = False\n    \n    # Add custom layers\n    x = base_model.output\n    x = GlobalAveragePooling2D()(x)\n    x = Dense(512, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    predictions = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inputs=base_model.input, outputs=predictions)\n    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n    \n    return model\n\ndef create_data_generators():\n    \"\"\"Create training and validation data generators with augmentation\"\"\"\n    train_datagen = ImageDataGenerator(\n        rescale=1./255,\n        rotation_range=20,\n        width_shift_range=0.2,\n        height_shift_range=0.2,\n        shear_range=0.2,\n        zoom_range=0.2,\n        horizontal_flip=True,\n        validation_split=0.2\n    )\n    \n    train_generator = train_datagen.flow_from_directory(\n        '/kaggle/input/chest-x-ray-dataset-4-categories/Chest X_Ray Dataset',\n        target_size=(224, 224),\n        batch_size=32,\n        class_mode='binary',\n        subset='training',\n        classes=['NORMAL', 'PNEUMONIA']\n    )\n    \n    validation_generator = train_datagen.flow_from_directory(\n        '/kaggle/input/chest-x-ray-dataset-4-categories/Chest X_Ray Dataset',\n        target_size=(224, 224),\n        batch_size=32,\n        class_mode='binary',\n        subset='validation',\n        classes=['NORMAL', 'PNEUMONIA']\n    )\n    \n    return train_generator, validation_generator\n\ndef train_model(model, train_generator, validation_generator):\n    \"\"\"Train the model\"\"\"\n    history = model.fit(\n        train_generator,\n        epochs=10,\n        validation_data=validation_generator,\n        callbacks=[\n            tf.keras.callbacks.EarlyStopping(patience=3, restore_best_weights=True)\n        ]\n    )\n    return history\n\ndef plot_training_history(history):\n    \"\"\"Plot training history\"\"\"\n    plt.figure(figsize=(12, 4))\n    \n    plt.subplot(1, 2, 1)\n    plt.plot(history.history['loss'], label='Training Loss')\n    plt.plot(history.history['val_loss'], label='Validation Loss')\n    plt.title('Model Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend()\n    \n    plt.subplot(1, 2, 2)\n    plt.plot(history.history['accuracy'], label='Training Accuracy')\n    plt.plot(history.history['val_accuracy'], label='Validation Accuracy')\n    plt.title('Model Accuracy')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    \n    plt.tight_layout()\n    plt.show()\n\ndef explain_with_lime(model, img_path):\n    \"\"\"Generate LIME explanation for a prediction\"\"\"\n    explainer = lime_image.LimeImageExplainer()\n    \n    img = load_and_preprocess_image(img_path)\n    explanation = explainer.explain_instance(\n        img,\n        lambda x: model.predict(x),\n        top_labels=1,\n        hide_color=0,\n        num_samples=1000\n    )\n    \n    # Get the image and mask\n    temp, mask = explanation.get_image_and_mask(\n        explanation.top_labels[0],\n        positive_only=True,\n        num_features=5,\n        hide_rest=True\n    )\n    \n    return img, temp, mask\n\ndef explain_with_shap(model, img_path):\n    \"\"\"Generate SHAP explanation for a prediction\"\"\"\n    img = load_and_preprocess_image(img_path)\n    \n    # Get prediction\n    pred = model.predict(np.expand_dims(img, axis=0))[0][0]\n    pred_label = \"Pneumonia\" if pred > 0.5 else \"Normal\"\n    true_label = \"Pneumonia\" if \"PNEUMONIA\" in img_path else \"Normal\"\n    \n    # Create background distribution for SHAP\n    background = np.zeros((10, 224, 224, 3))\n    \n    # Modify model to handle SHAP requirements\n    def predict(x):\n        return model.predict(x)\n    \n    # Initialize SHAP explainer with modified prediction function\n    explainer = shap.GradientExplainer(\n        (model.input, model.output),\n        background\n    )\n    \n    # Calculate SHAP values\n    shap_values = explainer.shap_values(\n        np.expand_dims(img, axis=0),\n        nsamples=50\n    )\n    \n    if isinstance(shap_values, list):\n        shap_values = shap_values[0]\n    \n    return img, shap_values, pred, pred_label, true_label\n\ndef plot_explanations(model, img_path):\n    \"\"\"Plot both LIME and SHAP explanations\"\"\"\n    plt.figure(figsize=(15, 5))\n    \n    # LIME explanation\n    img, temp, mask = explain_with_lime(model, img_path)\n    \n    plt.subplot(1, 3, 1)\n    plt.imshow(mark_boundaries(temp / 2 + 0.5, mask))\n    plt.title('LIME Explanation')\n    plt.axis('off')\n    \n    # SHAP explanation\n    img, shap_values, pred, pred_label, true_label = explain_with_shap(model, img_path)\n    \n    plt.subplot(1, 3, 2)\n    shap_img = np.abs(shap_values).mean(axis=-1)[0]\n    plt.imshow(shap_img, cmap='hot')\n    plt.title(f'SHAP Values\\nTrue: {true_label}\\nPred: {pred_label} ({pred:.3f})')\n    plt.axis('off')\n    \n    # Original image\n    plt.subplot(1, 3, 3)\n    plt.imshow(img)\n    plt.title('Original Image')\n    plt.axis('off')\n    \n    plt.tight_layout()\n    plt.show()\nclass OnePixelAttack:\n    def __init__(self, model, image, target_size=(224, 224)):\n        self.model = model\n        self.image = image\n        self.target_size = target_size\n    \n    def perturb_image(self, coords):\n        \"\"\"Perturb the image with pixels at the given coordinates\"\"\"\n        perturbed = self.image.copy()\n        \n        # Reshape coordinates to handle multiple pixels\n        coords = coords.reshape(-1, 5)\n        \n        for pixel in coords:\n            x, y = int(pixel[0]), int(pixel[1])\n            rgb = pixel[2:5]\n            perturbed[y, x] = rgb\n            \n        return perturbed\n    \n    def predict_class(self, coords):\n        \"\"\"Get model prediction for perturbed image\"\"\"\n        perturbed = self.perturb_image(coords)\n        pred = self.model.predict(np.expand_dims(perturbed, axis=0))[0][0]\n        return pred\n    \n    def attack(self, num_pixels=1):\n        \"\"\"Perform the one-pixel attack\"\"\"\n        bounds = []\n        for _ in range(num_pixels):\n            bounds.extend([(0, self.target_size[0]), (0, self.target_size[1])])  # x, y bounds\n            bounds.extend([(0, 1), (0, 1), (0, 1)])  # RGB bounds\n        \n        result = differential_evolution(\n            func=self.predict_class,\n            bounds=bounds,\n            maxiter=100,\n            popsize=10\n        )\n        \n        return result\n\ndef run_pixel_attacks(model, img_path, max_pixels=10):\n    \"\"\"Run pixel attacks with increasing number of pixels\"\"\"\n    original_img = load_and_preprocess_image(img_path)\n    original_pred = model.predict(np.expand_dims(original_img, axis=0))[0][0]\n    results = []\n    \n    plt.figure(figsize=(15, max_pixels * 3))\n    plt.subplot(max_pixels, 3, 1)\n    plt.imshow(original_img)\n    plt.title(f'Original\\nPred: {\"Pneumonia\" if original_pred > 0.5 else \"Normal\"} ({original_pred:.3f})')\n    plt.axis('off')\n    \n    for n_pixels in range(1, max_pixels + 1):\n        print(f\"\\nRunning {n_pixels}-pixel attack...\")\n        attack = OnePixelAttack(model, original_img)\n        result = attack.attack(num_pixels=n_pixels)\n        \n        perturbed = attack.perturb_image(result.x)\n        pred = model.predict(np.expand_dims(perturbed, axis=0))[0][0]\n        \n        results.append({\n            'num_pixels': n_pixels,\n            'success': (pred > 0.5) != (original_pred > 0.5),\n            'original_pred': original_pred,\n            'attacked_pred': pred\n        })\n        \n        plt.subplot(max_pixels, 3, n_pixels * 3 - 1)\n        plt.imshow(perturbed)\n        plt.title(f'{n_pixels}-Pixel Attack\\nPred: {\"Pneumonia\" if pred > 0.5 else \"Normal\"} ({pred:.3f})')\n        plt.axis('off')\n        \n        # Plot difference\n        plt.subplot(max_pixels, 3, n_pixels * 3)\n        plt.imshow(np.abs(original_img - perturbed))\n        plt.title('Difference')\n        plt.axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n    \n    return results\n\ndef print_attack_results(results):\n    \"\"\"Print summary of attack results\"\"\"\n    print(\"\\nPixel Attack Results:\")\n    print(\"=\" * 50)\n    print(\"Pixels | Original Pred | Attacked Pred | Success\")\n    print(\"-\" * 50)\n    \n    for result in results:\n        print(f\"{result['num_pixels']:6d} | {result['original_pred']:.3f} | {result['attacked_pred']:.3f} | {'Yes' if result['success'] else 'No'}\")\n\ndef main():\n    # 1. EDA\n    print(\"Performing EDA...\")\n    plot_sample_images()\n    \n    # 2. Model Training\n    print(\"\\nCreating and training model...\")\n    model = create_model()\n    train_generator, validation_generator = create_data_generators()\n    history = train_model(model, train_generator, validation_generator)\n    plot_training_history(history)\n    \n    # 3. Model Explainability\n    print(\"\\nGenerating explanations...\")\n    normal_sample = os.path.join(NORMAL_PATH, os.listdir(NORMAL_PATH)[0])\n    pneumonia_sample = os.path.join(PNEUMONIA_PATH, os.listdir(PNEUMONIA_PATH)[0])\n    \n    print(\"\\nNormal case explanations:\")\n    plot_explanations(model, normal_sample)\n    \n    print(\"\\nPneumonia case explanations:\")\n    plot_explanations(model, pneumonia_sample)\n    \n    # 4. Pixel Attacks\n    print(\"\\nRunning pixel attacks...\")\n    print(\"\\nAttacking normal case:\")\n    normal_results = run_pixel_attacks(model, normal_sample)\n    print_attack_results(normal_results)\n    \n    print(\"\\nAttacking pneumonia case:\")\n    pneumonia_results = run_pixel_attacks(model, pneumonia_sample)\n    print_attack_results(pneumonia_results)\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T20:20:06.484963Z","iopub.execute_input":"2025-01-14T20:20:06.485371Z","iopub.status.idle":"2025-01-14T20:40:36.859505Z","shell.execute_reply.started":"2025-01-14T20:20:06.485338Z","shell.execute_reply":"2025-01-14T20:40:36.858051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# Chunk 1: Imports\n# ================================\n\nimport os\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom PIL import Image\n\nimport tensorflow as tf\nfrom tensorflow.keras.applications import VGG16\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Dropout\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nfrom sklearn.metrics import confusion_matrix, classification_report\n\nimport lime\nfrom lime import lime_image\nfrom skimage.segmentation import mark_boundaries\n\nimport shap\nfrom matplotlib.colors import LinearSegmentedColormap\nfrom scipy.optimize import differential_evolution\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T20:45:06.378079Z","iopub.execute_input":"2025-01-14T20:45:06.378519Z","iopub.status.idle":"2025-01-14T20:45:06.385786Z","shell.execute_reply.started":"2025-01-14T20:45:06.378484Z","shell.execute_reply":"2025-01-14T20:45:06.384508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# Chunk 2: Configuration & Seed Setting\n# ================================\n\n# Set random seed for reproducibility\nSEED = 42\ntf.random.set_seed(SEED)\nnp.random.seed(SEED)\nrandom.seed(SEED)  # Ensures reproducibility for random operations\n\n# Define dataset paths\nDATASET_DIR = '/kaggle/input/chest-x-ray-dataset-4-categories/Chest X_Ray Dataset'\nNORMAL_PATH = os.path.join(DATASET_DIR, 'NORMAL')\nPNEUMONIA_PATH = os.path.join(DATASET_DIR, 'PNEUMONIA')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T20:45:16.916809Z","iopub.execute_input":"2025-01-14T20:45:16.9172Z","iopub.status.idle":"2025-01-14T20:45:16.93683Z","shell.execute_reply.started":"2025-01-14T20:45:16.917166Z","shell.execute_reply":"2025-01-14T20:45:16.935717Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# Chunk 3: Data Loading & Preprocessing\n# ================================\n\ndef load_and_preprocess_image(path, target_size=(224, 224)):\n    \"\"\"Load and preprocess a single image.\"\"\"\n    img = Image.open(path).convert('RGB')\n    img = img.resize(target_size)\n    img = np.array(img) / 255.0  # Normalize to [0,1]\n    return img\n\ndef plot_sample_images():\n    \"\"\"Plot sample images from both classes.\"\"\"\n    plt.figure(figsize=(15, 6))\n    \n    # Plot normal samples\n    normal_samples = os.listdir(NORMAL_PATH)[:3]\n    for i, img_name in enumerate(normal_samples, 1):\n        plt.subplot(2, 3, i)\n        img_path = os.path.join(NORMAL_PATH, img_name)\n        img = load_and_preprocess_image(img_path)\n        plt.imshow(img)\n        plt.title('Normal')\n        plt.axis('off')\n    \n    # Plot pneumonia samples\n    pneumonia_samples = os.listdir(PNEUMONIA_PATH)[:3]\n    for i, img_name in enumerate(pneumonia_samples, 1):\n        plt.subplot(2, 3, i+3)\n        img_path = os.path.join(PNEUMONIA_PATH, img_name)\n        img = load_and_preprocess_image(img_path)\n        plt.imshow(img)\n        plt.title('Pneumonia')\n        plt.axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n\n# Execute EDA\nprint(\"Performing Exploratory Data Analysis (EDA)...\")\nplot_sample_images()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T20:45:25.636938Z","iopub.execute_input":"2025-01-14T20:45:25.637297Z","iopub.status.idle":"2025-01-14T20:45:26.90385Z","shell.execute_reply.started":"2025-01-14T20:45:25.637267Z","shell.execute_reply":"2025-01-14T20:45:26.902894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# Chunk 4: Data Generators\n# ================================\n\ndef create_data_generators():\n    \"\"\"Create training and validation data generators with augmentation.\"\"\"\n    train_datagen = ImageDataGenerator(\n        rescale=1./255,\n        rotation_range=20,\n        width_shift_range=0.2,\n        height_shift_range=0.2,\n        shear_range=0.2,\n        zoom_range=0.2,\n        horizontal_flip=True,\n        validation_split=0.2  # 20% for validation\n    )\n    \n    train_generator = train_datagen.flow_from_directory(\n        DATASET_DIR,\n        target_size=(224, 224),\n        batch_size=32,\n        class_mode='binary',\n        subset='training',\n        classes=['NORMAL', 'PNEUMONIA'],\n        seed=SEED\n    )\n    \n    validation_generator = train_datagen.flow_from_directory(\n        DATASET_DIR,\n        target_size=(224, 224),\n        batch_size=32,\n        class_mode='binary',\n        subset='validation',\n        classes=['NORMAL', 'PNEUMONIA'],\n        seed=SEED\n    )\n    \n    return train_generator, validation_generator\n\n# Create data generators\nprint(\"\\nCreating data generators...\")\ntrain_generator, validation_generator = create_data_generators()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T20:45:38.12042Z","iopub.execute_input":"2025-01-14T20:45:38.120839Z","iopub.status.idle":"2025-01-14T20:45:39.743412Z","shell.execute_reply.started":"2025-01-14T20:45:38.120803Z","shell.execute_reply":"2025-01-14T20:45:39.742392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# Chunk 5: Model Creation\n# ================================\n\ndef create_model():\n    \"\"\"Create and compile the VGG16 model with custom layers.\"\"\"\n    base_model = VGG16(weights='imagenet', include_top=False, input_shape=(224, 224, 3))\n    \n    # Freeze the base model layers to prevent training\n    for layer in base_model.layers:\n        layer.trainable = False\n    \n    # Add custom layers on top of the base model\n    x = base_model.output\n    x = GlobalAveragePooling2D()(x)\n    x = Dense(512, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    predictions = Dense(1, activation='sigmoid')(x)  # Binary classification\n    \n    # Define the complete model\n    model = Model(inputs=base_model.input, outputs=predictions)\n    \n    # Compile the model\n    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n    \n    return model\n\n# Create the model\nprint(\"\\nCreating the model...\")\nmodel = create_model()\nmodel.summary()  # Optional: View the model architecture\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T20:45:48.767523Z","iopub.execute_input":"2025-01-14T20:45:48.767929Z","iopub.status.idle":"2025-01-14T20:45:49.16743Z","shell.execute_reply.started":"2025-01-14T20:45:48.767891Z","shell.execute_reply":"2025-01-14T20:45:49.166458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# Chunk 6: Model Training\n# ================================\n\ndef train_model(model, train_generator, validation_generator, epochs=10):\n    \"\"\"Train the model with early stopping.\"\"\"\n    history = model.fit(\n        train_generator,\n        epochs=epochs,\n        validation_data=validation_generator,\n        callbacks=[\n            tf.keras.callbacks.EarlyStopping(patience=3, restore_best_weights=True)\n        ]\n    )\n    return history\n\ndef plot_training_history(history):\n    \"\"\"Plot training and validation loss and accuracy.\"\"\"\n    plt.figure(figsize=(12, 4))\n    \n    # Plot Loss\n    plt.subplot(1, 2, 1)\n    plt.plot(history.history['loss'], label='Training Loss', marker='o')\n    plt.plot(history.history['val_loss'], label='Validation Loss', marker='o')\n    plt.title('Model Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend()\n    \n    # Plot Accuracy\n    plt.subplot(1, 2, 2)\n    plt.plot(history.history['accuracy'], label='Training Accuracy', marker='o')\n    plt.plot(history.history['val_accuracy'], label='Validation Accuracy', marker='o')\n    plt.title('Model Accuracy')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    \n    plt.tight_layout()\n    plt.show()\n\n# Train the model\nprint(\"\\nTraining the model...\")\nhistory = train_model(model, train_generator, validation_generator, epochs=10)\n\n# Plot training history\nprint(\"\\nPlotting training history...\")\nplot_training_history(history)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T20:45:59.326608Z","iopub.execute_input":"2025-01-14T20:45:59.327606Z","iopub.status.idle":"2025-01-14T21:03:56.833595Z","shell.execute_reply.started":"2025-01-14T20:45:59.327564Z","shell.execute_reply":"2025-01-14T21:03:56.832524Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# vision transformer","metadata":{}},{"cell_type":"code","source":"import os\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom transformers import ViTForImageClassification, ViTFeatureExtractor\nfrom torch.optim import AdamW\nfrom torch.nn import CrossEntropyLoss\nfrom tqdm import tqdm\n\n# Define the dataset class\nclass PneumoniaDataset(Dataset):\n    def __init__(self, file_paths, labels, transform=None):\n        self.file_paths = file_paths\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.file_paths)\n\n    def __getitem__(self, idx):\n        img_path = self.file_paths[idx]\n        label = self.labels[idx]\n        image = Image.open(img_path).convert(\"RGB\")\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        return image, label\n\n# Directories\nnormal_dir = \"/kaggle/input/chest-x-ray-dataset-4-categories/Chest X_Ray Dataset/NORMAL\"\npneumonia_dir = \"/kaggle/input/chest-x-ray-dataset-4-categories/Chest X_Ray Dataset/PNEUMONIA\"\n\n# Prepare data\nnormal_images = [os.path.join(normal_dir, img) for img in os.listdir(normal_dir) if img.endswith(\".jpeg\")]\npneumonia_images = [os.path.join(pneumonia_dir, img) for img in os.listdir(pneumonia_dir) if img.endswith(\".jpeg\")]\n\n# Assign labels: 0 for NORMAL, 1 for PNEUMONIA\nnormal_labels = [0] * len(normal_images)\npneumonia_labels = [1] * len(pneumonia_images)\n\nfile_paths = normal_images + pneumonia_images\nlabels = normal_labels + pneumonia_labels\n\n# Split the dataset into train and validation sets\ntrain_paths, val_paths, train_labels, val_labels = train_test_split(file_paths, labels, test_size=0.2, random_state=42)\n\n# Define transformations\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])\n])\n\n# Create datasets and dataloaders\ntrain_dataset = PneumoniaDataset(train_paths, train_labels, transform=transform)\nval_dataset = PneumoniaDataset(val_paths, val_labels, transform=transform)\n\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=32, shuffle=False)\n\n# Load ViT model\nmodel_name = \"google/vit-base-patch16-224\"\nfeature_extractor = ViTFeatureExtractor.from_pretrained(model_name)\n\n# Add ignore_mismatched_sizes=True to handle size mismatch\nmodel = ViTForImageClassification.from_pretrained(\n    model_name,\n    num_labels=2,  # Binary classification: NORMAL (0), PNEUMONIA (1)\n    ignore_mismatched_sizes=True\n)\n\n# Define optimizer and loss function\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\noptimizer = AdamW(model.parameters(), lr=5e-5)\ncriterion = CrossEntropyLoss()\n\n# Training loop\ndef train_model(model, train_loader, val_loader, criterion, optimizer, epochs=5):\n    for epoch in range(epochs):\n        model.train()\n        train_loss = 0\n        correct = 0\n        total = 0\n\n        for images, labels in tqdm(train_loader):\n            images, labels = images.to(device), labels.to(device)\n            optimizer.zero_grad()\n\n            outputs = model(images).logits\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n\n            train_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n        train_accuracy = 100 * correct / total\n        print(f\"Epoch {epoch + 1}/{epochs}, Loss: {train_loss:.4f}, Accuracy: {train_accuracy:.2f}%\")\n\n        # Validation\n        model.eval()\n        val_loss = 0\n        correct = 0\n        total = 0\n\n        with torch.no_grad():\n            for images, labels in val_loader:\n                images, labels = images.to(device), labels.to(device)\n\n                outputs = model(images).logits\n                loss = criterion(outputs, labels)\n                val_loss += loss.item()\n\n                _, predicted = torch.max(outputs, 1)\n                total += labels.size(0)\n                correct += (predicted == labels).sum().item()\n\n        val_accuracy = 100 * correct / total\n        print(f\"Validation Loss: {val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%\")\n\n# Train the model\ntrain_model(model, train_loader, val_loader, criterion, optimizer, epochs=5)\n\n# Save the trained model\nmodel.save_pretrained(\"/kaggle/working/pneumonia_detection_vit\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-24T19:15:52.565264Z","iopub.execute_input":"2025-01-24T19:15:52.565849Z","iopub.status.idle":"2025-01-24T19:39:48.16282Z","shell.execute_reply.started":"2025-01-24T19:15:52.565814Z","shell.execute_reply":"2025-01-24T19:39:48.162099Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport torch\nfrom torchvision import transforms\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom transformers import ViTForImageClassification, ViTFeatureExtractor\nfrom torch.nn.functional import softmax\n\n# Load the trained model\nmodel_path = \"/kaggle/working/pneumonia_detection_vit\"\nmodel = ViTForImageClassification.from_pretrained(model_path)\nmodel.eval()\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n# Load feature extractor\nfeature_extractor = ViTFeatureExtractor.from_pretrained(\"google/vit-base-patch16-224\")\n\n# Define transformation\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])\n])\n\n# Define the dataset for validation\nval_images = val_paths  # Use validation paths from your previous code\nval_labels = val_labels\n\n# Function to visualize attention maps\ndef visualize_attention(model, image_path, label):\n    image = Image.open(image_path).convert(\"RGB\")\n    input_tensor = transform(image).unsqueeze(0).to(device)\n\n    # Forward pass\n    with torch.no_grad():\n        outputs = model(input_tensor, output_attentions=True)\n        logits = outputs.logits\n        attentions = outputs.attentions\n\n    # Get predicted class and confidence\n    probs = softmax(logits, dim=1).detach().cpu().numpy()[0]\n    pred_class = np.argmax(probs)\n\n    # Extract the last attention map\n    attention_map = attentions[-1]  # Shape: (batch_size, num_heads, seq_len, seq_len)\n    mean_attention = attention_map[0].mean(dim=0).detach().cpu().numpy()  # Average over heads\n\n    # Remove the CLS token's attention\n    cls_attention = mean_attention[0, 1:]\n    patch_attention = cls_attention.reshape(14, 14)  # Reshape for visualization (ViT uses 14x14 patches)\n\n    # Upscale attention map to match image size\n    attention_resized = np.uint8(255 * patch_attention / patch_attention.max())\n    attention_resized = Image.fromarray(attention_resized).resize(image.size, resample=Image.BILINEAR)\n    attention_resized = np.array(attention_resized)\n\n    # Overlay attention map on the original image\n    overlay = np.array(image) * 0.5 + plt.cm.jet(attention_resized / 255.0)[:, :, :3] * 0.5\n    overlay = np.uint8(overlay / overlay.max() * 255)\n\n    # Plot results\n    plt.figure(figsize=(12, 4))\n    plt.subplot(1, 3, 1)\n    plt.imshow(image)\n    plt.title(\"Original Image\")\n    plt.axis(\"off\")\n\n    plt.subplot(1, 3, 2)\n    plt.imshow(patch_attention, cmap=\"jet\")\n    plt.title(\"Attention Map\")\n    plt.axis(\"off\")\n\n    plt.subplot(1, 3, 3)\n    plt.imshow(overlay)\n    plt.title(\"Overlay\")\n    plt.axis(\"off\")\n\n    plt.show()\n\n    print(f\"True Label: {'Pneumonia' if label == 1 else 'Normal'}, Predicted: {'Pneumonia' if pred_class == 1 else 'Normal'}, Confidence: {probs[pred_class]:.2f}\")\n\n# Visualize attention for some validation images\nfor img_path, label in zip(val_images[:5], val_labels[:5]):\n    visualize_attention(model, img_path, label)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-24T19:58:41.963855Z","iopub.execute_input":"2025-01-24T19:58:41.964684Z","iopub.status.idle":"2025-01-24T19:58:46.429727Z","shell.execute_reply.started":"2025-01-24T19:58:41.964643Z","shell.execute_reply":"2025-01-24T19:58:46.428871Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ----------------------------------------------------------------------------------------------------------------------------------------------------","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}