{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":26680,"databundleVersionId":2283525,"sourceType":"competition"},{"sourceId":2283573,"sourceType":"datasetVersion","datasetId":1351789},{"sourceId":2283993,"sourceType":"datasetVersion","datasetId":1351786}],"dockerImageVersionId":30096,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## covid19-detection with Vision Transformer and HeatMap visualization","metadata":{}},{"cell_type":"markdown","source":"![intro](https://www.ucsfhealth.org/-/media/project/ucsf/ucsf-health/medical-tests/hero/chest-x-ray-2x.jpg)","metadata":{}},{"cell_type":"markdown","source":"## Note\n\nThis implementation is inspired from official keras example.\n 1. [**Image classification with Vision Transformer**](https://keras.io/examples/vision/image_classification_with_vision_transformer/)\n 2. [**Grad-CAM class activation visualization**](https://keras.io/examples/vision/grad_cam/)","metadata":{}},{"cell_type":"markdown","source":"## Introduction\nThis example implements the [Vision Transformer (ViT)](https://arxiv.org/abs/2010.11929) model by Alexey Dosovitskiy et al. for image classification, and demonstrates it on the CIFAR-100 dataset. The ViT model applies the Transformer architecture with self-attention to sequences of image patches, without using convolution layers.\n\n![vit](https://neurohive.io/wp-content/uploads/2020/10/rsz_cov.png)","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport gc\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\n\n# Display\nfrom IPython.display import Image, display\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\n\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2025-03-06T01:57:44.885416Z","iopub.execute_input":"2025-03-06T01:57:44.885799Z","iopub.status.idle":"2025-03-06T01:57:51.194643Z","shell.execute_reply.started":"2025-03-06T01:57:44.885709Z","shell.execute_reply":"2025-03-06T01:57:51.193934Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BASE_PATH = '../input/siim-covid19-detection/'\ntrain_study = pd.read_csv(BASE_PATH + 'train_study_level.csv')\ntrain_study.head()","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:57:51.195882Z","iopub.execute_input":"2025-03-06T01:57:51.196108Z","iopub.status.idle":"2025-03-06T01:57:51.237361Z","shell.execute_reply.started":"2025-03-06T01:57:51.196085Z","shell.execute_reply":"2025-03-06T01:57:51.236486Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_image = pd.read_csv(BASE_PATH + 'train_image_level.csv')\ntrain_image.head()","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:57:51.238763Z","iopub.execute_input":"2025-03-06T01:57:51.239066Z","iopub.status.idle":"2025-03-06T01:57:51.291708Z","shell.execute_reply.started":"2025-03-06T01:57:51.239040Z","shell.execute_reply":"2025-03-06T01:57:51.290901Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_study['id'] = train_study['id'].str.replace('_study',\"\")\ntrain_study.rename({'id': 'StudyInstanceUID'},axis=1, inplace=True)\ntrain_study.head(3)\n# df_std.sort_values(by=['StudyInstanceUID'],inplace=True)\ntrain_study.head()","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:57:51.293062Z","iopub.execute_input":"2025-03-06T01:57:51.293304Z","iopub.status.idle":"2025-03-06T01:57:51.310072Z","shell.execute_reply.started":"2025-03-06T01:57:51.293280Z","shell.execute_reply":"2025-03-06T01:57:51.309097Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_size = 512\nBASE_PATH = \"../input/siimcovid19-{size}-jpg-image-dataset\".format(size=img_size)\ncollection = pd.read_csv(os.path.join(BASE_PATH,\"train.csv\" ))\ncollection['filepath'] = [os.path.join(BASE_PATH,\"train\",id_+'.jpg')for id_ in collection['image_id']]\ncollection.head()","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:57:51.311205Z","iopub.execute_input":"2025-03-06T01:57:51.311453Z","iopub.status.idle":"2025-03-06T01:57:51.400196Z","shell.execute_reply.started":"2025-03-06T01:57:51.311428Z","shell.execute_reply":"2025-03-06T01:57:51.399341Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target = np.array(collection[['Negative for Pneumonia','Typical Appearance','Indeterminate Appearance','Atypical Appearance']])","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:57:51.401256Z","iopub.execute_input":"2025-03-06T01:57:51.401536Z","iopub.status.idle":"2025-03-06T01:57:51.406107Z","shell.execute_reply.started":"2025-03-06T01:57:51.401509Z","shell.execute_reply":"2025-03-06T01:57:51.405311Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_test, y_train, y_test  = train_test_split(collection.filepath, target, test_size=0.33, random_state=42)\nprint(f\"train shape: {X_train.shape}- y_train shape: {y_train.shape}\")\nprint(f\"test shape: {X_test.shape}- y_test shape: {y_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:57:51.407191Z","iopub.execute_input":"2025-03-06T01:57:51.407458Z","iopub.status.idle":"2025-03-06T01:57:51.420659Z","shell.execute_reply.started":"2025-03-06T01:57:51.407433Z","shell.execute_reply":"2025-03-06T01:57:51.419743Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_classes = 4\ninput_shape = (512, 512, 1)","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:57:51.421863Z","iopub.execute_input":"2025-03-06T01:57:51.422175Z","iopub.status.idle":"2025-03-06T01:57:51.429401Z","shell.execute_reply.started":"2025-03-06T01:57:51.422149Z","shell.execute_reply":"2025-03-06T01:57:51.428662Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Configure the hyperparameters","metadata":{}},{"cell_type":"code","source":"learning_rate = 1e-4 #0.001\nweight_decay = 0.0001\nbatch_size = 256\nnum_epochs = 100\n# We'll resize input images to this size\nimage_size =  256 \n# Size of the patches to be extract from the input images\npatch_size = 20  \nnum_patches = (image_size // patch_size) ** 2\nprojection_dim = 128 #64\nnum_heads = 6 #4\n# Size of the transformer layers\ntransformer_units = [\n    projection_dim * 2,\n    projection_dim,\n]  \ntransformer_layers = 3 #8\n# Size of the dense layers of the final classifier\nmlp_head_units = [256] #[1024, 512]  ","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:57:51.432161Z","iopub.execute_input":"2025-03-06T01:57:51.432435Z","iopub.status.idle":"2025-03-06T01:57:51.440502Z","shell.execute_reply.started":"2025-03-06T01:57:51.432409Z","shell.execute_reply":"2025-03-06T01:57:51.439711Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@tf.function\ndef load(image_file, target):\n    image = tf.io.read_file(image_file)\n    image = tf.image.decode_jpeg(image)\n\n    image_ = tf.cast(image, tf.float32)\n    return image_, target","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:57:51.442314Z","iopub.execute_input":"2025-03-06T01:57:51.442669Z","iopub.status.idle":"2025-03-06T01:57:51.453791Z","shell.execute_reply.started":"2025-03-06T01:57:51.442634Z","shell.execute_reply":"2025-03-06T01:57:51.452941Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_loader = (\n    tf.data.Dataset\n    .from_tensor_slices((X_train,y_train))\n    .map(load, num_parallel_calls=AUTOTUNE)\n    .shuffle(7)\n    .batch(batch_size)\n)\ntest_loader = (\n    tf.data.Dataset\n    .from_tensor_slices((X_test,y_test))\n    .map(load, num_parallel_calls=AUTOTUNE)\n    .shuffle(7)\n    .batch(batch_size)\n)","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:57:51.454910Z","iopub.execute_input":"2025-03-06T01:57:51.455259Z","iopub.status.idle":"2025-03-06T01:57:53.724718Z","shell.execute_reply.started":"2025-03-06T01:57:51.455233Z","shell.execute_reply":"2025-03-06T01:57:53.723742Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_batch = (\n    tf.data.Dataset\n    .from_tensor_slices((X_train,y_train))\n    .map(load, num_parallel_calls=AUTOTUNE)\n    .shuffle(7)\n    .batch(X_train.shape[0]-100)\n)\n#next(iter(train_batch))[0].shape","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:57:53.725872Z","iopub.execute_input":"2025-03-06T01:57:53.726156Z","iopub.status.idle":"2025-03-06T01:57:53.740837Z","shell.execute_reply.started":"2025-03-06T01:57:53.726129Z","shell.execute_reply":"2025-03-06T01:57:53.740063Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Augmentation","metadata":{}},{"cell_type":"code","source":"data_augmentation = keras.Sequential(\n    [\n        layers.experimental.preprocessing.Normalization(),\n        layers.experimental.preprocessing.Resizing(image_size, image_size),\n        layers.experimental.preprocessing.RandomFlip(\"horizontal\"),\n        layers.experimental.preprocessing.RandomRotation(factor=0.02),\n        layers.experimental.preprocessing.RandomZoom(\n            height_factor = 0.2, width_factor = 0.2\n        ),\n    ],\n     name=\"data_augmentation\",\n)\n# Compute the mean and the variance of the training data for normalization.\nCompleteBatchData  =next(iter(train_batch))[0]\ndata_augmentation.layers[0].adapt(CompleteBatchData)","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:57:53.741926Z","iopub.execute_input":"2025-03-06T01:57:53.742234Z","iopub.status.idle":"2025-03-06T01:58:26.299810Z","shell.execute_reply.started":"2025-03-06T01:57:53.742208Z","shell.execute_reply":"2025-03-06T01:58:26.299023Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del CompleteBatchData\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:58:26.300934Z","iopub.execute_input":"2025-03-06T01:58:26.301168Z","iopub.status.idle":"2025-03-06T01:58:26.890202Z","shell.execute_reply.started":"2025-03-06T01:58:26.301146Z","shell.execute_reply":"2025-03-06T01:58:26.889345Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Implementing multilayer perceptron (MLP)","metadata":{}},{"cell_type":"code","source":"print('hello')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-06T01:58:26.891524Z","iopub.execute_input":"2025-03-06T01:58:26.891907Z","iopub.status.idle":"2025-03-06T01:58:26.908514Z","shell.execute_reply.started":"2025-03-06T01:58:26.891867Z","shell.execute_reply":"2025-03-06T01:58:26.907629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def mlp(x, hidden_units, dropout_rate):\n    for units in hidden_units:\n        x = layers.Dense(units, activation = tf.nn.gelu)(x)\n        x = layers.Dropout(dropout_rate)(x)\n    return x","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:58:26.909692Z","iopub.execute_input":"2025-03-06T01:58:26.910102Z","iopub.status.idle":"2025-03-06T01:58:26.920806Z","shell.execute_reply.started":"2025-03-06T01:58:26.910065Z","shell.execute_reply":"2025-03-06T01:58:26.919907Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Implement patch creation as a layer","metadata":{}},{"cell_type":"code","source":"class Patches(layers.Layer):\n    def __init__(self, patch_size):\n        super(Patches, self).__init__()\n        self.patch_size = patch_size\n        \n    def call(self, images):\n        batch_size = tf.shape(images)[0]\n        patches = tf.image.extract_patches(\n            images = images,\n            sizes = [1, self.patch_size, self.patch_size, 1],\n            strides=[1, self.patch_size, self.patch_size, 1],\n            rates=[1, 1, 1, 1],\n            padding=\"VALID\",\n        )\n        patch_dims = patches.shape[-1]\n        #print(patches.shape)\n        patches = tf.reshape(patches, [batch_size, -1, patch_dims])\n        return patches","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:58:26.922072Z","iopub.execute_input":"2025-03-06T01:58:26.922355Z","iopub.status.idle":"2025-03-06T01:58:26.938800Z","shell.execute_reply.started":"2025-03-06T01:58:26.922312Z","shell.execute_reply":"2025-03-06T01:58:26.938138Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(8, 8))\nimage = next(iter(train_loader))[0][5]\nplt.imshow(image, cmap='gray')\nplt.axis(\"off\")\n\nresized_image = tf.image.resize(\n    tf.convert_to_tensor([image]), size=(image_size, image_size)\n)\n#print(resized_image.shape)\npatches = Patches(patch_size)(resized_image)\nprint(f\"Image size: {image_size} X {image_size}\")\nprint(f\"Patch size: {patch_size} X {patch_size}\")\nprint(f\"Patches per image: {patches.shape[1]}\")\nprint(f\"Elements per patch: {patches.shape[-1]}\")\n\nn = int(np.sqrt(patches.shape[1]))\n\nplt.figure(figsize=(4, 4))\nfor i, patch in enumerate(patches[0]):\n    ax = plt.subplot(n, n, i + 1)\n    patch_img = tf.reshape(patch, (patch_size, patch_size, 1))\n    plt.imshow(patch_img,cmap='gray')\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:58:26.939931Z","iopub.execute_input":"2025-03-06T01:58:26.940179Z","iopub.status.idle":"2025-03-06T01:58:32.888849Z","shell.execute_reply.started":"2025-03-06T01:58:26.940154Z","shell.execute_reply":"2025-03-06T01:58:32.887941Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## The patch encoding layer\n\nThe **PatchEncoder** layer will linearly transform a **patch** by projecting it into a vector of size **projection_dim**. In addition, it adds a learnable position embedding to the projected vector.","metadata":{}},{"cell_type":"code","source":"class PatchEncoder(layers.Layer):\n    def __init__(self, num_of_patches, projection_dim):\n        super(PatchEncoder, self).__init__()\n        self.num_patches = num_patches\n        self.projection = layers.Dense(units = projection_dim)\n        self.position_embedding = layers.Embedding(\n            input_dim = num_patches, output_dim = projection_dim\n        )\n        \n    def call(self, patch):\n        positions = tf.range(start=0, limit=self.num_patches, delta=1)\n        encode = self.projection(patch) + self.position_embedding(positions)\n        return encode","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:58:32.890023Z","iopub.execute_input":"2025-03-06T01:58:32.890264Z","iopub.status.idle":"2025-03-06T01:58:32.895773Z","shell.execute_reply.started":"2025-03-06T01:58:32.890241Z","shell.execute_reply":"2025-03-06T01:58:32.894618Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##  The ViT model\n\nThe ViT model consists of multiple Transformer blocks, which use the **layers.MultiHeadAttention layer** as a self-attention mechanism applied to the sequence of patches. The Transformer blocks produce a **[batch_size, num_patches, projection_dim]** tensor, which is processed via an classifier head with softmax to produce the final class probabilities output.<br>\nUnlike the technique described in the [paper](https://arxiv.org/abs/2010.11929), which prepends a learnable embedding to the sequence of encoded patches to serve as the image representation, all the outputs of the final Transformer block are reshaped with **layers.Flatten()** and used as the image representation input to the classifier head. Note that the **layers.GlobalAveragePooling1D** layer could also be used instead to aggregate the outputs of the Transformer block, especially when the number of patches and the projection dimensions are large.","metadata":{}},{"cell_type":"code","source":"def vit_model():\n    inputs = layers.Input(shape=input_shape)\n    # Augment data.\n    augmented = data_augmentation(inputs)\n    # Create patches.\n    patches = Patches(patch_size)(augmented)\n    # Encode patches.\n    encoded_patches = PatchEncoder(num_patches, projection_dim)(patches)\n    \n    # Create multiple layers of the Transformer block.\n    for _ in range(transformer_layers):\n        # Layer normalization 1.\n        x1 = layers.BatchNormalization()(encoded_patches)\n        # create a multi-head attention layer\n        attention_output = layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=projection_dim, dropout=0.1\n        )(x1, x1)\n        # Skip connection 1.\n        x2 = layers.Add()([attention_output, encoded_patches])\n        # Layer normalization 2.\n        x3 = layers.BatchNormalization()(x2)\n        # MLP.\n        x3 = mlp(x3, hidden_units=transformer_units, dropout_rate=0.1)\n        # Skip connection 2.\n        encoded_patches = layers.Add()([x3, x2])\n        \n    # Create a [batch_size, projection_dim] tensor.\n    representation = layers.LayerNormalization()(encoded_patches)\n    representation = layers.Flatten()(representation)\n    representation = layers.Dropout(0.5)(representation)\n    # Add MLP\n    features = mlp(representation, hidden_units = mlp_head_units, dropout_rate=0.5)\n    # Classify outputs.\n    logits = layers.Dense(num_classes, activation='softmax')(features)\n    # create keras model\n    model = keras.Model(inputs=inputs, outputs=logits)\n    return model","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:58:32.897131Z","iopub.execute_input":"2025-03-06T01:58:32.897506Z","iopub.status.idle":"2025-03-06T01:58:32.914672Z","shell.execute_reply.started":"2025-03-06T01:58:32.897458Z","shell.execute_reply":"2025-03-06T01:58:32.914057Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def experiment(model):\n    optimizer = tfa.optimizers.AdamW(\n        learning_rate=learning_rate, weight_decay=weight_decay\n    )\n    \n    model.compile(\n        optimizer=optimizer,\n        loss=keras.losses.CategoricalCrossentropy(from_logits=True),\n        metrics=[\n            keras.metrics.CategoricalAccuracy(name=\"accuracy\"),\n            keras.metrics.AUC( name=\"AUC\"),\n        ],\n     )\n    checkpoint_filepath = \"./tmp/checkpoint\"\n    checkpoint_callback = keras.callbacks.ModelCheckpoint(\n        checkpoint_filepath,\n        monitor=\"val_accuracy\",\n        save_best_only=True,\n        save_weights_only=True,\n    )\n\n    history = model.fit(train_loader ,\n                        batch_size=batch_size,\n                        epochs=num_epochs,\n                        validation_data=test_loader,\n                        callbacks=[checkpoint_callback],)\n    model.load_weights(checkpoint_filepath)\n    _, accuracy, auc = model.evaluate(test_loader)\n    print(f\"Test accuracy: {round(accuracy * 100, 2)}%\")\n    print(f\"Test AUC: {round(auc * 100, 2)}%\")\n\n    return history","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:58:32.915692Z","iopub.execute_input":"2025-03-06T01:58:32.915969Z","iopub.status.idle":"2025-03-06T01:58:32.929636Z","shell.execute_reply.started":"2025-03-06T01:58:32.915935Z","shell.execute_reply":"2025-03-06T01:58:32.929043Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vit_classifier = vit_model()\nvit_classifier.summary()","metadata":{"execution":{"iopub.status.busy":"2025-03-06T01:58:32.930558Z","iopub.execute_input":"2025-03-06T01:58:32.930777Z","iopub.status.idle":"2025-03-06T01:58:33.659644Z","shell.execute_reply.started":"2025-03-06T01:58:32.930756Z","shell.execute_reply":"2025-03-06T01:58:33.658556Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history = experiment(vit_classifier)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-03-06T01:58:33.660923Z","iopub.execute_input":"2025-03-06T01:58:33.661188Z","iopub.status.idle":"2025-03-06T02:28:45.215764Z","shell.execute_reply.started":"2025-03-06T01:58:33.661163Z","shell.execute_reply":"2025-03-06T02:28:45.214933Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model Performance Visulization","metadata":{}},{"cell_type":"code","source":"# list all data in history\nprint(history.history.keys())\n# summarize history for accuracy\nplt.figure(figsize=(12,10))\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n# summarize history for loss\nplt.figure(figsize=(12,10))\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:28:45.217153Z","iopub.execute_input":"2025-03-06T02:28:45.217458Z","iopub.status.idle":"2025-03-06T02:28:45.587675Z","shell.execute_reply.started":"2025-03-06T02:28:45.217427Z","shell.execute_reply":"2025-03-06T02:28:45.586744Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# summarize history for loss\nplt.figure(figsize=(12,10))\nplt.plot(history.history['AUC'])\nplt.plot(history.history['val_AUC'])\nplt.title('model AUC')\nplt.ylabel('AUC')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:28:45.589131Z","iopub.execute_input":"2025-03-06T02:28:45.589536Z","iopub.status.idle":"2025-03-06T02:28:45.776621Z","shell.execute_reply.started":"2025-03-06T02:28:45.589497Z","shell.execute_reply":"2025-03-06T02:28:45.775686Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vit_classifier.load_weights(\"./tmp/checkpoint\")","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:28:45.777696Z","iopub.execute_input":"2025-03-06T02:28:45.777958Z","iopub.status.idle":"2025-03-06T02:28:46.086869Z","shell.execute_reply.started":"2025-03-06T02:28:45.777931Z","shell.execute_reply":"2025-03-06T02:28:46.085870Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_img_array(img):\n    \n    # `array` is a float32 Numpy array of shape (299, 299, 3)\n    array = keras.preprocessing.image.img_to_array(img)\n    # We add a dimension to transform our array into a \"batch\"\n    # of size (1, 299, 299, 3)\n    array = np.expand_dims(array, axis=0)\n    return array","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:28:46.090301Z","iopub.execute_input":"2025-03-06T02:28:46.090588Z","iopub.status.idle":"2025-03-06T02:28:46.094882Z","shell.execute_reply.started":"2025-03-06T02:28:46.090561Z","shell.execute_reply":"2025-03-06T02:28:46.093879Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## The Grad-CAM algorithm","metadata":{}},{"cell_type":"code","source":"def gradcam_heatmap(img_array, model, last_conv_layer_name, pred_index=None):\n    # First, we create a model that maps the input image to the activations\n    # of the last conv layer as well as the output predictions\n    grad_model = tf.keras.models.Model(\n        [model.input], [model.get_layer(last_conv_layer_name).output,  model.output]\n    )\n    \n    # Then, we compute the gradient of the top predicted class for our input image\n    # with respect to the activations of the last conv layer\n    with tf.GradientTape() as tape:\n        last_conv_layer_output, preds = grad_model(img_array)\n        if pred_index is None:\n            pred_index = tf.argmax(preds[0])\n        class_channel = preds[:, pred_index]\n        \n        \n    # This is the gradient of the output neuron (top predicted or chosen)\n    # with regard to the output feature map of the last conv layer\n    grads = tape.gradient(class_channel, last_conv_layer_output)\n\n    # This is a vector where each entry is the mean intensity of the gradient\n    # over a specific feature map channel\n    pooled_grads = tf.reduce_mean(grads, axis=(0, 1))\n    # We multiply each channel in the feature map array\n    # by \"how important this channel is\" with regard to the top predicted class\n    # then sum all the channels to obtain the heatmap class activation\n    last_conv_layer_output = last_conv_layer_output#[0]\n    #print(np.expand_dims(last_conv_layer_output,axis=0))\n    #print(pooled_grads[..., tf.newaxis])\n    heatmap = last_conv_layer_output @ pooled_grads[..., tf.newaxis]\n    heatmap = tf.squeeze(heatmap)\n    \n    # For visualization purpose, we will also normalize the heatmap between 0 & 1\n    heatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)\n    return heatmap.numpy()","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:28:46.096322Z","iopub.execute_input":"2025-03-06T02:28:46.096573Z","iopub.status.idle":"2025-03-06T02:28:46.106086Z","shell.execute_reply.started":"2025-03-06T02:28:46.096549Z","shell.execute_reply":"2025-03-06T02:28:46.105156Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## superimposed visualization","metadata":{}},{"cell_type":"code","source":"def display_gradcam(img, heatmap, cam_path=\"cam.jpg\", alpha=0.4,preds=[0,0,0,0], plot=None):\n\n    # Rescale heatmap to a range 0-255\n    heatmap = np.uint8(255 * heatmap)\n\n    # Use jet colormap to colorize heatmap\n    jet = cm.get_cmap(\"jet\")\n\n    # Use RGB values of the colormap\n    jet_colors = jet(np.arange(256))[:, :3]\n    jet_heatmap = jet_colors[heatmap]\n\n    # Create an image with RGB colorized heatmap\n    jet_heatmap = keras.preprocessing.image.array_to_img(jet_heatmap)\n    jet_heatmap = jet_heatmap.resize((img.shape[1], img.shape[0]))\n    jet_heatmap = keras.preprocessing.image.img_to_array(jet_heatmap)\n\n    # Superimpose the heatmap on original image\n    superimposed_img = jet_heatmap * alpha + img\n    superimposed_img = keras.preprocessing.image.array_to_img(superimposed_img)\n\n    # Save the superimposed image\n    #superimposed_img.save(cam_path)\n\n    # Display Grad CAM\n    #display(Image(cam_path))\n    #plt.figure(figsize=(8,8))\n    plot.imshow(superimposed_img)\n    plot.set(title =\n        \"Negative for Pneumonia: \\\n        {:.3f}\\nTypical Appearance: \\\n        {:.3f}\\nIndeterminate Appearance: \\\n        {:.3f}\\nAtypical Appearance: \\\n        {:.3f}\".format(preds[0], \\\n                    preds[1], \\\n                    preds[2], \\\n                    preds[3])\n    )\n    plot.axis('off')\n    #plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:28:46.107220Z","iopub.execute_input":"2025-03-06T02:28:46.107560Z","iopub.status.idle":"2025-03-06T02:28:46.122073Z","shell.execute_reply.started":"2025-03-06T02:28:46.107532Z","shell.execute_reply":"2025-03-06T02:28:46.121196Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Implement","metadata":{}},{"cell_type":"code","source":"test_image = next(iter(test_loader))[0][5]\n# Prepare image\nimg_array =get_img_array(test_image)\n\nlast_conv_layer_name = 'layer_normalization'\n# Remove last layer's softmax\nvit_classifier.layers[-1].activation = None\n# Print what the top predicted class is\npreds = vit_classifier.predict(img_array)\nprint(\"Predicted:\\n\" +\"Negative for Pneumonia: \\\n    {p1}\\nTypical Appearance: {p2}\\nIndeterminate Appearance: \\\n    {p3}\\nAtypical Appearance: {p4}\".format(p1=preds[0][0], \\\n                                            p2=preds[0][1],p3=preds[0][2],p4=preds[0][3]))\n# Generate class activation heatmap\nheatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\nheatmap = np.reshape(heatmap, (12,12))\n# Display heatmap\nplt.matshow(heatmap)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:28:46.123252Z","iopub.execute_input":"2025-03-06T02:28:46.123521Z","iopub.status.idle":"2025-03-06T02:28:47.638480Z","shell.execute_reply.started":"2025-03-06T02:28:46.123483Z","shell.execute_reply":"2025-03-06T02:28:47.636995Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Heat-Map Visualization over Test-set","metadata":{}},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][:6], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:28:47.640365Z","iopub.execute_input":"2025-03-06T02:28:47.640946Z","iopub.status.idle":"2025-03-06T02:28:50.077577Z","shell.execute_reply.started":"2025-03-06T02:28:47.640887Z","shell.execute_reply":"2025-03-06T02:28:50.076427Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][20:27], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:28:50.078941Z","iopub.execute_input":"2025-03-06T02:28:50.079284Z","iopub.status.idle":"2025-03-06T02:28:52.519890Z","shell.execute_reply.started":"2025-03-06T02:28:50.079253Z","shell.execute_reply":"2025-03-06T02:28:52.518824Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][50:57], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:28:52.521168Z","iopub.execute_input":"2025-03-06T02:28:52.521475Z","iopub.status.idle":"2025-03-06T02:28:54.899725Z","shell.execute_reply.started":"2025-03-06T02:28:52.521443Z","shell.execute_reply":"2025-03-06T02:28:54.898659Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][60:67], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:28:54.901050Z","iopub.execute_input":"2025-03-06T02:28:54.901379Z","iopub.status.idle":"2025-03-06T02:28:57.270265Z","shell.execute_reply.started":"2025-03-06T02:28:54.901347Z","shell.execute_reply":"2025-03-06T02:28:57.269344Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][70:77], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:28:57.271629Z","iopub.execute_input":"2025-03-06T02:28:57.271986Z","iopub.status.idle":"2025-03-06T02:28:59.683070Z","shell.execute_reply.started":"2025-03-06T02:28:57.271951Z","shell.execute_reply":"2025-03-06T02:28:59.681984Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][100:107], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:28:59.684422Z","iopub.execute_input":"2025-03-06T02:28:59.684762Z","iopub.status.idle":"2025-03-06T02:29:02.096950Z","shell.execute_reply.started":"2025-03-06T02:28:59.684727Z","shell.execute_reply":"2025-03-06T02:29:02.096000Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][200:207], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:29:02.098251Z","iopub.execute_input":"2025-03-06T02:29:02.098578Z","iopub.status.idle":"2025-03-06T02:29:04.546577Z","shell.execute_reply.started":"2025-03-06T02:29:02.098546Z","shell.execute_reply":"2025-03-06T02:29:04.545618Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][250:257], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:29:04.547719Z","iopub.execute_input":"2025-03-06T02:29:04.548030Z","iopub.status.idle":"2025-03-06T02:29:06.956316Z","shell.execute_reply.started":"2025-03-06T02:29:04.548000Z","shell.execute_reply":"2025-03-06T02:29:06.955059Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][150:157], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2025-03-06T02:29:06.957573Z","iopub.execute_input":"2025-03-06T02:29:06.957897Z","iopub.status.idle":"2025-03-06T02:29:10.096236Z","shell.execute_reply.started":"2025-03-06T02:29:06.957859Z","shell.execute_reply":"2025-03-06T02:29:10.095079Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][170:177], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-06T02:29:10.097644Z","iopub.execute_input":"2025-03-06T02:29:10.098035Z","iopub.status.idle":"2025-03-06T02:29:12.482639Z","shell.execute_reply.started":"2025-03-06T02:29:10.097998Z","shell.execute_reply":"2025-03-06T02:29:12.481761Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Thank You.","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}