{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## covid19-detection with Vision Transformer and HeatMap visualization","metadata":{}},{"cell_type":"markdown","source":"![intro](https://www.ucsfhealth.org/-/media/project/ucsf/ucsf-health/medical-tests/hero/chest-x-ray-2x.jpg)","metadata":{}},{"cell_type":"markdown","source":"## Note\n\nThis implementation is inspired from official keras example.\n 1. [**Image classification with Vision Transformer**](https://keras.io/examples/vision/image_classification_with_vision_transformer/)\n 2. [**Grad-CAM class activation visualization**](https://keras.io/examples/vision/grad_cam/)","metadata":{}},{"cell_type":"markdown","source":"## Introduction\nThis example implements the [Vision Transformer (ViT)](https://arxiv.org/abs/2010.11929) model by Alexey Dosovitskiy et al. for image classification, and demonstrates it on the CIFAR-100 dataset. The ViT model applies the Transformer architecture with self-attention to sequences of image patches, without using convolution layers.\n\n![vit](https://neurohive.io/wp-content/uploads/2020/10/rsz_cov.png)","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport gc\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\n\n# Display\nfrom IPython.display import Image, display\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\n\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-15T06:40:43.415755Z","iopub.execute_input":"2021-06-15T06:40:43.416376Z","iopub.status.idle":"2021-06-15T06:40:51.229447Z","shell.execute_reply.started":"2021-06-15T06:40:43.416225Z","shell.execute_reply":"2021-06-15T06:40:51.228263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_PATH = '../input/siim-covid19-detection/'\ntrain_study = pd.read_csv(BASE_PATH + 'train_study_level.csv')\ntrain_study.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:40:51.231778Z","iopub.execute_input":"2021-06-15T06:40:51.232132Z","iopub.status.idle":"2021-06-15T06:40:51.282234Z","shell.execute_reply.started":"2021-06-15T06:40:51.232096Z","shell.execute_reply":"2021-06-15T06:40:51.281157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image = pd.read_csv(BASE_PATH + 'train_image_level.csv')\ntrain_image.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:40:51.284168Z","iopub.execute_input":"2021-06-15T06:40:51.284494Z","iopub.status.idle":"2021-06-15T06:40:51.342761Z","shell.execute_reply.started":"2021-06-15T06:40:51.284461Z","shell.execute_reply":"2021-06-15T06:40:51.341727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study['id'] = train_study['id'].str.replace('_study',\"\")\ntrain_study.rename({'id': 'StudyInstanceUID'},axis=1, inplace=True)\ntrain_study.head(3)\n# df_std.sort_values(by=['StudyInstanceUID'],inplace=True)\ntrain_study.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:40:51.344583Z","iopub.execute_input":"2021-06-15T06:40:51.345013Z","iopub.status.idle":"2021-06-15T06:40:51.368307Z","shell.execute_reply.started":"2021-06-15T06:40:51.344963Z","shell.execute_reply":"2021-06-15T06:40:51.367163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_size = 512\nBASE_PATH = \"../input/siimcovid19-{size}-jpg-image-dataset\".format(size=img_size)\ncollection = pd.read_csv(os.path.join(BASE_PATH,\"train.csv\" ))\ncollection['filepath'] = [os.path.join(BASE_PATH,\"train\",id_+'.jpg')for id_ in collection['image_id']]\ncollection.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:40:51.369845Z","iopub.execute_input":"2021-06-15T06:40:51.370189Z","iopub.status.idle":"2021-06-15T06:40:51.475919Z","shell.execute_reply.started":"2021-06-15T06:40:51.370157Z","shell.execute_reply":"2021-06-15T06:40:51.47484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = np.array(collection[['Negative for Pneumonia','Typical Appearance','Indeterminate Appearance','Atypical Appearance']])","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:40:51.477311Z","iopub.execute_input":"2021-06-15T06:40:51.477643Z","iopub.status.idle":"2021-06-15T06:40:51.484039Z","shell.execute_reply.started":"2021-06-15T06:40:51.477608Z","shell.execute_reply":"2021-06-15T06:40:51.482923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test  = train_test_split(collection.filepath, target, test_size=0.33, random_state=42)\nprint(f\"train shape: {X_train.shape}- y_train shape: {y_train.shape}\")\nprint(f\"test shape: {X_test.shape}- y_test shape: {y_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:40:51.486789Z","iopub.execute_input":"2021-06-15T06:40:51.487189Z","iopub.status.idle":"2021-06-15T06:40:51.503304Z","shell.execute_reply.started":"2021-06-15T06:40:51.487154Z","shell.execute_reply":"2021-06-15T06:40:51.502002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_classes = 4\ninput_shape = (512, 512, 1)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:40:51.506675Z","iopub.execute_input":"2021-06-15T06:40:51.507085Z","iopub.status.idle":"2021-06-15T06:40:51.514553Z","shell.execute_reply.started":"2021-06-15T06:40:51.507048Z","shell.execute_reply":"2021-06-15T06:40:51.51366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configure the hyperparameters","metadata":{}},{"cell_type":"code","source":"learning_rate = 1e-4 #0.001\nweight_decay = 0.0001\nbatch_size = 256\nnum_epochs = 100\n# We'll resize input images to this size\nimage_size =  256 \n# Size of the patches to be extract from the input images\npatch_size = 20  \nnum_patches = (image_size // patch_size) ** 2\nprojection_dim = 128 #64\nnum_heads = 6 #4\n# Size of the transformer layers\ntransformer_units = [\n    projection_dim * 2,\n    projection_dim,\n]  \ntransformer_layers = 3 #8\n# Size of the dense layers of the final classifier\nmlp_head_units = [256] #[1024, 512]  ","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:40:51.515862Z","iopub.execute_input":"2021-06-15T06:40:51.516156Z","iopub.status.idle":"2021-06-15T06:40:51.530071Z","shell.execute_reply.started":"2021-06-15T06:40:51.516128Z","shell.execute_reply":"2021-06-15T06:40:51.529004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef load(image_file, target):\n    image = tf.io.read_file(image_file)\n    image = tf.image.decode_jpeg(image)\n\n    image_ = tf.cast(image, tf.float32)\n    return image_, target","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:40:51.531648Z","iopub.execute_input":"2021-06-15T06:40:51.532003Z","iopub.status.idle":"2021-06-15T06:40:51.550188Z","shell.execute_reply.started":"2021-06-15T06:40:51.531971Z","shell.execute_reply":"2021-06-15T06:40:51.54897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = (\n    tf.data.Dataset\n    .from_tensor_slices((X_train,y_train))\n    .map(load, num_parallel_calls=AUTOTUNE)\n    .shuffle(7)\n    .batch(batch_size)\n)\ntest_loader = (\n    tf.data.Dataset\n    .from_tensor_slices((X_test,y_test))\n    .map(load, num_parallel_calls=AUTOTUNE)\n    .shuffle(7)\n    .batch(batch_size)\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:40:51.551665Z","iopub.execute_input":"2021-06-15T06:40:51.552036Z","iopub.status.idle":"2021-06-15T06:40:51.957016Z","shell.execute_reply.started":"2021-06-15T06:40:51.552002Z","shell.execute_reply":"2021-06-15T06:40:51.955995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_batch = (\n    tf.data.Dataset\n    .from_tensor_slices((X_train,y_train))\n    .map(load, num_parallel_calls=AUTOTUNE)\n    .shuffle(7)\n    .batch(X_train.shape[0]-100)\n)\n#next(iter(train_batch))[0].shape","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:40:51.95878Z","iopub.execute_input":"2021-06-15T06:40:51.959262Z","iopub.status.idle":"2021-06-15T06:40:51.979124Z","shell.execute_reply.started":"2021-06-15T06:40:51.959209Z","shell.execute_reply":"2021-06-15T06:40:51.978052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Augmentation","metadata":{}},{"cell_type":"code","source":"data_augmentation = keras.Sequential(\n    [\n        layers.experimental.preprocessing.Normalization(),\n        layers.experimental.preprocessing.Resizing(image_size, image_size),\n        layers.experimental.preprocessing.RandomFlip(\"horizontal\"),\n        layers.experimental.preprocessing.RandomRotation(factor=0.02),\n        layers.experimental.preprocessing.RandomZoom(\n            height_factor = 0.2, width_factor = 0.2\n        ),\n    ],\n     name=\"data_augmentation\",\n)\n# Compute the mean and the variance of the training data for normalization.\nCompleteBatchData  =next(iter(train_batch))[0]\ndata_augmentation.layers[0].adapt(CompleteBatchData)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:40:51.980717Z","iopub.execute_input":"2021-06-15T06:40:51.981172Z","iopub.status.idle":"2021-06-15T06:41:26.24062Z","shell.execute_reply.started":"2021-06-15T06:40:51.981125Z","shell.execute_reply":"2021-06-15T06:41:26.239692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del CompleteBatchData\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:41:26.241736Z","iopub.execute_input":"2021-06-15T06:41:26.242211Z","iopub.status.idle":"2021-06-15T06:41:26.636576Z","shell.execute_reply.started":"2021-06-15T06:41:26.242177Z","shell.execute_reply":"2021-06-15T06:41:26.635681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Implementing multilayer perceptron (MLP)","metadata":{}},{"cell_type":"code","source":"def mlp(x, hidden_units, dropout_rate):\n    for units in hidden_units:\n        x = layers.Dense(units, activation = tf.nn.gelu)(x)\n        x = layers.Dropout(dropout_rate)(x)\n    return x","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:41:26.637832Z","iopub.execute_input":"2021-06-15T06:41:26.638161Z","iopub.status.idle":"2021-06-15T06:41:26.652344Z","shell.execute_reply.started":"2021-06-15T06:41:26.638132Z","shell.execute_reply":"2021-06-15T06:41:26.651077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Implement patch creation as a layer","metadata":{}},{"cell_type":"code","source":"class Patches(layers.Layer):\n    def __init__(self, patch_size):\n        super(Patches, self).__init__()\n        self.patch_size = patch_size\n        \n    def call(self, images):\n        batch_size = tf.shape(images)[0]\n        patches = tf.image.extract_patches(\n            images = images,\n            sizes = [1, self.patch_size, self.patch_size, 1],\n            strides=[1, self.patch_size, self.patch_size, 1],\n            rates=[1, 1, 1, 1],\n            padding=\"VALID\",\n        )\n        patch_dims = patches.shape[-1]\n        #print(patches.shape)\n        patches = tf.reshape(patches, [batch_size, -1, patch_dims])\n        return patches","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:41:26.653766Z","iopub.execute_input":"2021-06-15T06:41:26.654249Z","iopub.status.idle":"2021-06-15T06:41:26.668489Z","shell.execute_reply.started":"2021-06-15T06:41:26.65421Z","shell.execute_reply":"2021-06-15T06:41:26.667387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8, 8))\nimage = next(iter(train_loader))[0][5]\nplt.imshow(image, cmap='gray')\nplt.axis(\"off\")\n\nresized_image = tf.image.resize(\n    tf.convert_to_tensor([image]), size=(image_size, image_size)\n)\n#print(resized_image.shape)\npatches = Patches(patch_size)(resized_image)\nprint(f\"Image size: {image_size} X {image_size}\")\nprint(f\"Patch size: {patch_size} X {patch_size}\")\nprint(f\"Patches per image: {patches.shape[1]}\")\nprint(f\"Elements per patch: {patches.shape[-1]}\")\n\nn = int(np.sqrt(patches.shape[1]))\n\nplt.figure(figsize=(4, 4))\nfor i, patch in enumerate(patches[0]):\n    ax = plt.subplot(n, n, i + 1)\n    patch_img = tf.reshape(patch, (patch_size, patch_size, 1))\n    plt.imshow(patch_img,cmap='gray')\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:41:26.670231Z","iopub.execute_input":"2021-06-15T06:41:26.670677Z","iopub.status.idle":"2021-06-15T06:41:33.695245Z","shell.execute_reply.started":"2021-06-15T06:41:26.67063Z","shell.execute_reply":"2021-06-15T06:41:33.693985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The patch encoding layer\n\nThe **PatchEncoder** layer will linearly transform a **patch** by projecting it into a vector of size **projection_dim**. In addition, it adds a learnable position embedding to the projected vector.","metadata":{}},{"cell_type":"code","source":"class PatchEncoder(layers.Layer):\n    def __init__(self, num_of_patches, projection_dim):\n        super(PatchEncoder, self).__init__()\n        self.num_patches = num_patches\n        self.projection = layers.Dense(units = projection_dim)\n        self.position_embedding = layers.Embedding(\n            input_dim = num_patches, output_dim = projection_dim\n        )\n        \n    def call(self, patch):\n        positions = tf.range(start=0, limit=self.num_patches, delta=1)\n        encode = self.projection(patch) + self.position_embedding(positions)\n        return encode","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:42:45.623725Z","iopub.execute_input":"2021-06-15T06:42:45.624146Z","iopub.status.idle":"2021-06-15T06:42:45.632315Z","shell.execute_reply.started":"2021-06-15T06:42:45.62411Z","shell.execute_reply":"2021-06-15T06:42:45.63092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##  The ViT model\n\nThe ViT model consists of multiple Transformer blocks, which use the **layers.MultiHeadAttention layer** as a self-attention mechanism applied to the sequence of patches. The Transformer blocks produce a **[batch_size, num_patches, projection_dim]** tensor, which is processed via an classifier head with softmax to produce the final class probabilities output.<br>\nUnlike the technique described in the [paper](https://arxiv.org/abs/2010.11929), which prepends a learnable embedding to the sequence of encoded patches to serve as the image representation, all the outputs of the final Transformer block are reshaped with **layers.Flatten()** and used as the image representation input to the classifier head. Note that the **layers.GlobalAveragePooling1D** layer could also be used instead to aggregate the outputs of the Transformer block, especially when the number of patches and the projection dimensions are large.","metadata":{}},{"cell_type":"code","source":"def vit_model():\n    inputs = layers.Input(shape=input_shape)\n    # Augment data.\n    augmented = data_augmentation(inputs)\n    # Create patches.\n    patches = Patches(patch_size)(augmented)\n    # Encode patches.\n    encoded_patches = PatchEncoder(num_patches, projection_dim)(patches)\n    \n    # Create multiple layers of the Transformer block.\n    for _ in range(transformer_layers):\n        # Layer normalization 1.\n        x1 = layers.BatchNormalization()(encoded_patches)\n        # create a multi-head attention layer\n        attention_output = layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=projection_dim, dropout=0.1\n        )(x1, x1)\n        # Skip connection 1.\n        x2 = layers.Add()([attention_output, encoded_patches])\n        # Layer normalization 2.\n        x3 = layers.BatchNormalization()(x2)\n        # MLP.\n        x3 = mlp(x3, hidden_units=transformer_units, dropout_rate=0.1)\n        # Skip connection 2.\n        encoded_patches = layers.Add()([x3, x2])\n        \n    # Create a [batch_size, projection_dim] tensor.\n    representation = layers.LayerNormalization()(encoded_patches)\n    representation = layers.Flatten()(representation)\n    representation = layers.Dropout(0.5)(representation)\n    # Add MLP\n    features = mlp(representation, hidden_units = mlp_head_units, dropout_rate=0.5)\n    # Classify outputs.\n    logits = layers.Dense(num_classes, activation='softmax')(features)\n    # create keras model\n    model = keras.Model(inputs=inputs, outputs=logits)\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:42:47.391467Z","iopub.execute_input":"2021-06-15T06:42:47.391848Z","iopub.status.idle":"2021-06-15T06:42:47.401473Z","shell.execute_reply.started":"2021-06-15T06:42:47.391807Z","shell.execute_reply":"2021-06-15T06:42:47.400351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def experiment(model):\n    optimizer = tfa.optimizers.AdamW(\n        learning_rate=learning_rate, weight_decay=weight_decay\n    )\n    \n    model.compile(\n        optimizer=optimizer,\n        loss=keras.losses.CategoricalCrossentropy(from_logits=True),\n        metrics=[\n            keras.metrics.CategoricalAccuracy(name=\"accuracy\"),\n            keras.metrics.AUC( name=\"AUC\"),\n        ],\n     )\n    checkpoint_filepath = \"./tmp/checkpoint\"\n    checkpoint_callback = keras.callbacks.ModelCheckpoint(\n        checkpoint_filepath,\n        monitor=\"val_accuracy\",\n        save_best_only=True,\n        save_weights_only=True,\n    )\n\n    history = model.fit(train_loader ,\n                        batch_size=batch_size,\n                        epochs=num_epochs,\n                        validation_data=test_loader,\n                        callbacks=[checkpoint_callback],)\n    model.load_weights(checkpoint_filepath)\n    _, accuracy, auc = model.evaluate(test_loader)\n    print(f\"Test accuracy: {round(accuracy * 100, 2)}%\")\n    print(f\"Test AUC: {round(auc * 100, 2)}%\")\n\n    return history","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:42:48.192737Z","iopub.execute_input":"2021-06-15T06:42:48.193268Z","iopub.status.idle":"2021-06-15T06:42:48.204104Z","shell.execute_reply.started":"2021-06-15T06:42:48.193218Z","shell.execute_reply":"2021-06-15T06:42:48.202315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vit_classifier = vit_model()\nvit_classifier.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:42:49.044763Z","iopub.execute_input":"2021-06-15T06:42:49.045182Z","iopub.status.idle":"2021-06-15T06:42:49.918069Z","shell.execute_reply.started":"2021-06-15T06:42:49.045146Z","shell.execute_reply":"2021-06-15T06:42:49.916856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = experiment(vit_classifier)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-15T06:42:49.919936Z","iopub.execute_input":"2021-06-15T06:42:49.920275Z","iopub.status.idle":"2021-06-15T10:14:32.539569Z","shell.execute_reply.started":"2021-06-15T06:42:49.920242Z","shell.execute_reply":"2021-06-15T10:14:32.537089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Performance Visulization","metadata":{}},{"cell_type":"code","source":"# list all data in history\nprint(history.history.keys())\n# summarize history for accuracy\nplt.figure(figsize=(12,10))\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n# summarize history for loss\nplt.figure(figsize=(12,10))\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.926644Z","iopub.status.idle":"2021-06-15T06:38:41.927307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summarize history for loss\nplt.figure(figsize=(12,10))\nplt.plot(history.history['AUC'])\nplt.plot(history.history['val_AUC'])\nplt.title('model AUC')\nplt.ylabel('AUC')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.928321Z","iopub.status.idle":"2021-06-15T06:38:41.928904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vit_classifier.load_weights(\"./tmp/checkpoint\")","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.929868Z","iopub.status.idle":"2021-06-15T06:38:41.930456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_img_array(img):\n    \n    # `array` is a float32 Numpy array of shape (299, 299, 3)\n    array = keras.preprocessing.image.img_to_array(img)\n    # We add a dimension to transform our array into a \"batch\"\n    # of size (1, 299, 299, 3)\n    array = np.expand_dims(array, axis=0)\n    return array","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.931367Z","iopub.status.idle":"2021-06-15T06:38:41.931918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The Grad-CAM algorithm","metadata":{}},{"cell_type":"code","source":"def gradcam_heatmap(img_array, model, last_conv_layer_name, pred_index=None):\n    # First, we create a model that maps the input image to the activations\n    # of the last conv layer as well as the output predictions\n    grad_model = tf.keras.models.Model(\n        [model.input], [model.get_layer(last_conv_layer_name).output,  model.output]\n    )\n    \n    # Then, we compute the gradient of the top predicted class for our input image\n    # with respect to the activations of the last conv layer\n    with tf.GradientTape() as tape:\n        last_conv_layer_output, preds = grad_model(img_array)\n        if pred_index is None:\n            pred_index = tf.argmax(preds[0])\n        class_channel = preds[:, pred_index]\n        \n        \n    # This is the gradient of the output neuron (top predicted or chosen)\n    # with regard to the output feature map of the last conv layer\n    grads = tape.gradient(class_channel, last_conv_layer_output)\n\n    # This is a vector where each entry is the mean intensity of the gradient\n    # over a specific feature map channel\n    pooled_grads = tf.reduce_mean(grads, axis=(0, 1))\n    # We multiply each channel in the feature map array\n    # by \"how important this channel is\" with regard to the top predicted class\n    # then sum all the channels to obtain the heatmap class activation\n    last_conv_layer_output = last_conv_layer_output#[0]\n    #print(np.expand_dims(last_conv_layer_output,axis=0))\n    #print(pooled_grads[..., tf.newaxis])\n    heatmap = last_conv_layer_output @ pooled_grads[..., tf.newaxis]\n    heatmap = tf.squeeze(heatmap)\n    \n    # For visualization purpose, we will also normalize the heatmap between 0 & 1\n    heatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)\n    return heatmap.numpy()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.932945Z","iopub.status.idle":"2021-06-15T06:38:41.933562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## superimposed visualization","metadata":{}},{"cell_type":"code","source":"def display_gradcam(img, heatmap, cam_path=\"cam.jpg\", alpha=0.4,preds=[0,0,0,0], plot=None):\n\n    # Rescale heatmap to a range 0-255\n    heatmap = np.uint8(255 * heatmap)\n\n    # Use jet colormap to colorize heatmap\n    jet = cm.get_cmap(\"jet\")\n\n    # Use RGB values of the colormap\n    jet_colors = jet(np.arange(256))[:, :3]\n    jet_heatmap = jet_colors[heatmap]\n\n    # Create an image with RGB colorized heatmap\n    jet_heatmap = keras.preprocessing.image.array_to_img(jet_heatmap)\n    jet_heatmap = jet_heatmap.resize((img.shape[1], img.shape[0]))\n    jet_heatmap = keras.preprocessing.image.img_to_array(jet_heatmap)\n\n    # Superimpose the heatmap on original image\n    superimposed_img = jet_heatmap * alpha + img\n    superimposed_img = keras.preprocessing.image.array_to_img(superimposed_img)\n\n    # Save the superimposed image\n    #superimposed_img.save(cam_path)\n\n    # Display Grad CAM\n    #display(Image(cam_path))\n    #plt.figure(figsize=(8,8))\n    plot.imshow(superimposed_img)\n    plot.set(title =\n        \"Negative for Pneumonia: \\\n        {:.3f}\\nTypical Appearance: \\\n        {:.3f}\\nIndeterminate Appearance: \\\n        {:.3f}\\nAtypical Appearance: \\\n        {:.3f}\".format(preds[0], \\\n                    preds[1], \\\n                    preds[2], \\\n                    preds[3])\n    )\n    plot.axis('off')\n    #plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.934563Z","iopub.status.idle":"2021-06-15T06:38:41.935172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Implement","metadata":{}},{"cell_type":"code","source":"test_image = next(iter(test_loader))[0][5]\n# Prepare image\nimg_array =get_img_array(test_image)\n\nlast_conv_layer_name = 'layer_normalization'\n# Remove last layer's softmax\nvit_classifier.layers[-1].activation = None\n# Print what the top predicted class is\npreds = vit_classifier.predict(img_array)\nprint(\"Predicted:\\n\" +\"Negative for Pneumonia: \\\n    {p1}\\nTypical Appearance: {p2}\\nIndeterminate Appearance: \\\n    {p3}\\nAtypical Appearance: {p4}\".format(p1=preds[0][0], \\\n                                            p2=preds[0][1],p3=preds[0][2],p4=preds[0][3]))\n# Generate class activation heatmap\nheatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\nheatmap = np.reshape(heatmap, (12,12))\n# Display heatmap\nplt.matshow(heatmap)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.936137Z","iopub.status.idle":"2021-06-15T06:38:41.936758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Heat-Map Visualization over Test-set","metadata":{}},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][:6], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.937734Z","iopub.status.idle":"2021-06-15T06:38:41.938369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][20:27], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.939378Z","iopub.status.idle":"2021-06-15T06:38:41.939988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][50:57], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.940932Z","iopub.status.idle":"2021-06-15T06:38:41.941575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][60:67], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.94255Z","iopub.status.idle":"2021-06-15T06:38:41.943158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][70:77], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.944156Z","iopub.status.idle":"2021-06-15T06:38:41.944776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][100:107], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.945726Z","iopub.status.idle":"2021-06-15T06:38:41.946361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][200:207], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.947306Z","iopub.status.idle":"2021-06-15T06:38:41.947907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][250:257], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.948868Z","iopub.status.idle":"2021-06-15T06:38:41.949497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][150:157], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2021-06-15T06:38:41.95048Z","iopub.status.idle":"2021-06-15T06:38:41.95108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(20, 20))\nfor images, ax in zip(next(iter(test_loader))[0][170:177], axis.flat):\n    img_array =get_img_array(images)\n    # Remove last layer's softmax\n    vit_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = vit_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, vit_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (12,12))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Thank You.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}