{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport gc\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\n\n# Display\nfrom IPython.display import Image, display\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\n\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-08T06:52:46.611176Z","iopub.execute_input":"2023-09-08T06:52:46.611952Z","iopub.status.idle":"2023-09-08T06:52:58.952847Z","shell.execute_reply.started":"2023-09-08T06:52:46.611898Z","shell.execute_reply":"2023-09-08T06:52:58.951878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH = '../input/aptos2019-blindness-detection/train_images/'\nDF_TRAIN = pd.read_csv('../input/aptos2019-blindness-detection/train.csv', dtype='str')\nDF_TRAIN['image_path'] = TRAIN_PATH + \"/\" + DF_TRAIN[\"id_code\"] + \".png\"\n \nDF_TRAIN.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:06.496515Z","iopub.execute_input":"2023-09-08T06:53:06.497400Z","iopub.status.idle":"2023-09-08T06:53:06.539823Z","shell.execute_reply.started":"2023-09-08T06:53:06.497357Z","shell.execute_reply":"2023-09-08T06:53:06.538751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DF_TRAIN['image_path'][0]","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:07.281827Z","iopub.execute_input":"2023-09-08T06:53:07.282498Z","iopub.status.idle":"2023-09-08T06:53:07.290908Z","shell.execute_reply.started":"2023-09-08T06:53:07.282450Z","shell.execute_reply":"2023-09-08T06:53:07.289472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'../input/aptos2019-blindness-detection/train_images/000c1434d8d7.png'.split('/')[-1]","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:07.697086Z","iopub.execute_input":"2023-09-08T06:53:07.697586Z","iopub.status.idle":"2023-09-08T06:53:07.705553Z","shell.execute_reply.started":"2023-09-08T06:53:07.697547Z","shell.execute_reply":"2023-09-08T06:53:07.704317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes = {0 : \"No DR\",\n           1 : \"Mild\",\n           2 : \"Moderate\",\n           3 : \"Severe\",\n           4 : \"Proliferative\"}","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:08.160382Z","iopub.execute_input":"2023-09-08T06:53:08.160833Z","iopub.status.idle":"2023-09-08T06:53:08.166277Z","shell.execute_reply.started":"2023-09-08T06:53:08.160801Z","shell.execute_reply":"2023-09-08T06:53:08.164953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH_RS = '../input/aptos2019-blindness-detection/train_images/'\nDF_TRAIN_RS = pd.read_csv('../input/aptos2019-blindness-detection/train.csv', dtype='str')\nDF_TRAIN_RS['image_path'] = TRAIN_PATH_RS +\"/\"+ DF_TRAIN_RS[\"id_code\"] + \".png\" \nDF_TRAIN_RS.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:08.549559Z","iopub.execute_input":"2023-09-08T06:53:08.550591Z","iopub.status.idle":"2023-09-08T06:53:08.573285Z","shell.execute_reply.started":"2023-09-08T06:53:08.550546Z","shell.execute_reply":"2023-09-08T06:53:08.572198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Source: https://stackoverflow.com/questions/37292872/how-can-i-one-hot-encode-in-python\ndef encode_and_bind(original_dataframe, feature_to_encode):\n    dummies = pd.get_dummies(original_dataframe[[feature_to_encode]])\n    res = pd.concat([original_dataframe, dummies], axis=1)\n    res = res.drop([feature_to_encode], axis=1)\n    return(res) ","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:09.019819Z","iopub.execute_input":"2023-09-08T06:53:09.020298Z","iopub.status.idle":"2023-09-08T06:53:09.027653Z","shell.execute_reply.started":"2023-09-08T06:53:09.020259Z","shell.execute_reply":"2023-09-08T06:53:09.026372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = encode_and_bind(DF_TRAIN_RS, 'diagnosis')","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:09.416691Z","iopub.execute_input":"2023-09-08T06:53:09.417182Z","iopub.status.idle":"2023-09-08T06:53:09.441550Z","shell.execute_reply.started":"2023-09-08T06:53:09.417126Z","shell.execute_reply":"2023-09-08T06:53:09.440421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:09.818854Z","iopub.execute_input":"2023-09-08T06:53:09.819689Z","iopub.status.idle":"2023-09-08T06:53:09.833727Z","shell.execute_reply.started":"2023-09-08T06:53:09.819649Z","shell.execute_reply":"2023-09-08T06:53:09.832728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes = {\"diagnosis_0\" : \"No DR\",\n           \"diagnosis_1\" : \"Mild\",\n           \"diagnosis_2\" : \"Moderate\",\n           \"diagnosis_3\" : \"Severe\",\n           \"diagnosis_4\" : \"Proliferative\"}","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:10.424408Z","iopub.execute_input":"2023-09-08T06:53:10.425250Z","iopub.status.idle":"2023-09-08T06:53:10.431023Z","shell.execute_reply.started":"2023-09-08T06:53:10.425187Z","shell.execute_reply":"2023-09-08T06:53:10.429755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res.rename(columns=classes, inplace=True)\nres.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:10.556204Z","iopub.execute_input":"2023-09-08T06:53:10.557610Z","iopub.status.idle":"2023-09-08T06:53:10.573039Z","shell.execute_reply.started":"2023-09-08T06:53:10.557557Z","shell.execute_reply":"2023-09-08T06:53:10.572062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res.columns","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:10.966237Z","iopub.execute_input":"2023-09-08T06:53:10.967232Z","iopub.status.idle":"2023-09-08T06:53:10.976004Z","shell.execute_reply.started":"2023-09-08T06:53:10.967175Z","shell.execute_reply":"2023-09-08T06:53:10.974668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = np.array(res[['No DR', 'Mild', 'Moderate', 'Severe', 'Proliferative']])","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:11.336784Z","iopub.execute_input":"2023-09-08T06:53:11.337271Z","iopub.status.idle":"2023-09-08T06:53:11.343784Z","shell.execute_reply.started":"2023-09-08T06:53:11.337232Z","shell.execute_reply":"2023-09-08T06:53:11.342538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_classes = 5\ninput_shape = (32, 32, 3)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:12.468916Z","iopub.execute_input":"2023-09-08T06:53:12.469444Z","iopub.status.idle":"2023-09-08T06:53:12.475255Z","shell.execute_reply.started":"2023-09-08T06:53:12.469403Z","shell.execute_reply":"2023-09-08T06:53:12.474221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimage_path = Image.open('../input/aptos2019-blindness-detection/train_images/000c1434d8d7.png')\nplt.imshow(image_path)\nplt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:13.285258Z","iopub.execute_input":"2023-09-08T06:53:13.286499Z","iopub.status.idle":"2023-09-08T06:53:15.337938Z","shell.execute_reply.started":"2023-09-08T06:53:13.286427Z","shell.execute_reply":"2023-09-08T06:53:15.336986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load and preprocess the data\n(x_train, x_test, y_train, y_test) = train_test_split(DF_TRAIN['image_path'], DF_TRAIN['diagnosis'], test_size=0.33, random_state=42)\n\n# Convert labels to integers (assuming 'diagnosis' is a column in your DataFrame)\ny_train = y_train.astype(int)\ny_test = y_test.astype(int)\n\n# One-hot encode the labels\ny_train = keras.utils.to_categorical(y_train, num_classes)\ny_test = keras.utils.to_categorical(y_test, num_classes)\n\nprint(f\"x_train shape: {x_train.shape} - y_train shape: {y_train.shape}\")\nprint(f\"x_test shape: {x_test.shape} - y_test shape: {y_test.shape}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:19.292208Z","iopub.execute_input":"2023-09-08T06:53:19.292709Z","iopub.status.idle":"2023-09-08T06:53:19.309315Z","shell.execute_reply.started":"2023-09-08T06:53:19.292668Z","shell.execute_reply":"2023-09-08T06:53:19.307561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positional_emb = True\nconv_layers = 2\nprojection_dim = 128\n\nnum_heads = 2\ntransformer_units = [\n    projection_dim,\n    projection_dim,\n]\ntransformer_layers = 2\nstochastic_depth_rate = 0.1\n\nlearning_rate = 0.001\nweight_decay = 0.0001\nbatch_size = 128\nnum_epochs = 100\nimage_size = 32\npatch_size = 3\nnum_patches = (image_size // patch_size) ** 2\nprojection_dim = 64\nnum_heads = 4\n","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:19.835746Z","iopub.execute_input":"2023-09-08T06:53:19.836250Z","iopub.status.idle":"2023-09-08T06:53:19.843727Z","shell.execute_reply.started":"2023-09-08T06:53:19.836214Z","shell.execute_reply":"2023-09-08T06:53:19.842681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_patches","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:20.504432Z","iopub.execute_input":"2023-09-08T06:53:20.505466Z","iopub.status.idle":"2023-09-08T06:53:20.513907Z","shell.execute_reply.started":"2023-09-08T06:53:20.505403Z","shell.execute_reply":"2023-09-08T06:53:20.512264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef load_and_preprocess(image_file, target):\n    image = tf.io.read_file(image_file)\n    image = tf.image.decode_png(image)\n    image = tf.image.resize(image, (image_size, image_size))  # Resize images to a common size\n    image = tf.cast(image, tf.uint8)\n    return image, target\n\ntrain_loader = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n    .shuffle(7)\n    .batch(batch_size)\n)\n\ntest_loader = (\n    tf.data.Dataset\n    .from_tensor_slices((x_test, y_test))\n    .map(load_and_preprocess, num_parallel_calls=AUTOTUNE)\n    .shuffle(7)\n    .batch(batch_size)\n)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:27.834796Z","iopub.execute_input":"2023-09-08T06:53:27.835299Z","iopub.status.idle":"2023-09-08T06:53:28.374791Z","shell.execute_reply.started":"2023-09-08T06:53:27.835262Z","shell.execute_reply":"2023-09-08T06:53:28.373598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\n# Define the load function\n@tf.function\ndef load(image_file, target):\n    image = tf.io.read_file(image_file)\n    image = tf.image.decode_png(image)\n    image = tf.cast(image, tf.uint8)\n    return image, target\n\n# Create the train_batch dataset\ntrain_batch = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .map(load, num_parallel_calls=tf.data.AUTOTUNE)\n    .shuffle(7)\n    .batch(x_train.shape[0] - 100)  # Adjust batch size as needed\n)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:28.452392Z","iopub.execute_input":"2023-09-08T06:53:28.452836Z","iopub.status.idle":"2023-09-08T06:53:28.555105Z","shell.execute_reply.started":"2023-09-08T06:53:28.452804Z","shell.execute_reply":"2023-09-08T06:53:28.554045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_batch = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train,y_train))\n    .map(load, num_parallel_calls=AUTOTUNE)\n    .shuffle(7)\n    .batch(x_train.shape[0]-100)#X_train.shape[0]-100\n)\n#next(iter(train_batch))[0].shape","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:28.979608Z","iopub.execute_input":"2023-09-08T06:53:28.980104Z","iopub.status.idle":"2023-09-08T06:53:29.008747Z","shell.execute_reply.started":"2023-09-08T06:53:28.980068Z","shell.execute_reply":"2023-09-08T06:53:29.007399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Patches(layers.Layer):\n    def __init__(self, patch_size):\n        super(Patches, self).__init__()\n        self.patch_size = patch_size\n        \n    def call(self, images):\n        batch_size = tf.shape(images)[0]\n        patches = tf.image.extract_patches(\n            images = images,\n            sizes = [1, self.patch_size, self.patch_size, 1],\n            strides=[1, self.patch_size, self.patch_size, 1],\n            rates=[1, 1, 1, 1],\n            padding=\"VALID\",\n        )\n        patch_dims = patches.shape[-1]\n        #print(patches.shape)\n        patches = tf.reshape(patches, [batch_size, -1, patch_dims])\n        return patches","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:29.795506Z","iopub.execute_input":"2023-09-08T06:53:29.797071Z","iopub.status.idle":"2023-09-08T06:53:29.806691Z","shell.execute_reply.started":"2023-09-08T06:53:29.797013Z","shell.execute_reply":"2023-09-08T06:53:29.805664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8, 8))\nimage = next(iter(train_loader))[0][5]\n\nplt.imshow(image)\nplt.axis(\"off\")\n\nresized_image = tf.image.resize(\n    tf.convert_to_tensor([image]), size=(image_size, image_size)\n)\n\nprint(resized_image.shape)\npatches = Patches(patch_size)(resized_image)\nprint(f\"Image size: {image_size} X {image_size}\")\nprint(f\"Patch size: {patch_size} X {patch_size}\")\nprint(f\"Patches per image: {patches.shape[1]}\")\nprint(f\"Elements per patch: {patches.shape[-1]}\")\n\nn = int(np.sqrt(patches.shape[1]))\n#print(n)\n\nplt.figure(figsize=(8, 8))\nfor i, patch in enumerate(patches[0]):\n    ax = plt.subplot(n, n, i + 1)\n    patch_img = tf.reshape(patch, (patch_size, patch_size, 3))\n    plt.imshow(patch_img.numpy().astype('uint8'))\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:31.512314Z","iopub.execute_input":"2023-09-08T06:53:31.513417Z","iopub.status.idle":"2023-09-08T06:53:43.274433Z","shell.execute_reply.started":"2023-09-08T06:53:31.513343Z","shell.execute_reply":"2023-09-08T06:53:43.273323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CCTTokenizer(layers.Layer):\n    def __init__(\n        self,\n        kernel_size=3,\n        stride=1,\n        padding=1,\n        pooling_kernel_size=3,\n        pooling_stride=2,\n        num_conv_layers=conv_layers,\n        num_output_channels=[64, 128],\n        positional_emb=positional_emb,\n        **kwargs,\n    ):\n        super().__init__(**kwargs)\n\n        # This is our tokenizer.\n        self.conv_model = keras.Sequential()\n        for i in range(num_conv_layers):\n            self.conv_model.add(\n                layers.Conv2D(\n                    num_output_channels[i],\n                    kernel_size,\n                    stride,\n                    padding=\"valid\",\n                    use_bias=False,\n                    activation=\"relu\",\n                    kernel_initializer=\"he_normal\",\n                )\n            )\n            self.conv_model.add(layers.ZeroPadding2D(padding))\n            self.conv_model.add(\n                layers.MaxPool2D(pooling_kernel_size, pooling_stride, \"same\")\n            )\n\n        self.positional_emb = positional_emb\n\n    def call(self, images):\n        outputs = self.conv_model(images)\n        # After passing the images through our mini-network the spatial dimensions\n        # are flattened to form sequences.\n        reshaped = tf.reshape(\n            outputs,\n            (-1, tf.shape(outputs)[1] * tf.shape(outputs)[2], tf.shape(outputs)[-1]),\n        )\n        return reshaped\n\n    def positional_embedding(self, image_size):\n        # Positional embeddings are optional in CCT. Here, we calculate\n        # the number of sequences and initialize an `Embedding` layer to\n        # compute the positional embeddings later.\n        if self.positional_emb:\n            dummy_inputs = tf.ones((1, image_size, image_size, 3))\n            dummy_outputs = self.call(dummy_inputs)\n            sequence_length = tf.shape(dummy_outputs)[1]\n            projection_dim = tf.shape(dummy_outputs)[-1]\n\n            embed_layer = layers.Embedding(\n                input_dim=sequence_length, output_dim=projection_dim\n            )\n            return embed_layer, sequence_length\n        else:\n            return None","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:56.298180Z","iopub.execute_input":"2023-09-08T06:53:56.298643Z","iopub.status.idle":"2023-09-08T06:53:56.314266Z","shell.execute_reply.started":"2023-09-08T06:53:56.298607Z","shell.execute_reply":"2023-09-08T06:53:56.312890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Referred from: github.com:rwightman/pytorch-image-models.\nclass StochasticDepth(layers.Layer):\n    def __init__(self, drop_prop, **kwargs):\n        super().__init__(**kwargs)\n        self.drop_prob = drop_prop\n\n    def call(self, x, training=None):\n        if training:\n            keep_prob = 1 - self.drop_prob\n            shape = (tf.shape(x)[0],) + (1,) * (tf.shape(x).shape[0] - 1)\n            random_tensor = keep_prob + tf.random.uniform(shape, 0, 1)\n            random_tensor = tf.floor(random_tensor)\n            return (x / keep_prob) * random_tensor\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:57.171098Z","iopub.execute_input":"2023-09-08T06:53:57.171594Z","iopub.status.idle":"2023-09-08T06:53:57.181686Z","shell.execute_reply.started":"2023-09-08T06:53:57.171557Z","shell.execute_reply":"2023-09-08T06:53:57.179924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mlp(x, hidden_units, dropout_rate):\n    for units in hidden_units:\n        x = layers.Dense(units, activation=tf.nn.gelu)(x)\n        x = layers.Dropout(dropout_rate)(x)\n    return x","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:53:57.969494Z","iopub.execute_input":"2023-09-08T06:53:57.970829Z","iopub.status.idle":"2023-09-08T06:53:57.976097Z","shell.execute_reply.started":"2023-09-08T06:53:57.970790Z","shell.execute_reply":"2023-09-08T06:53:57.975279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Note the rescaling layer. These layers have pre-defined inference behavior.\nimage_size = 32  # Set the desired image size\ndata_augmentation = keras.Sequential(\n    [\n        layers.Rescaling(scale=1.0 / 255),\n        layers.RandomCrop(image_size, image_size),\n        layers.RandomFlip(\"horizontal\"),\n    ],\n    name=\"data_augmentation\",\n)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-08T14:26:16.212313Z","iopub.execute_input":"2023-09-08T14:26:16.212780Z","iopub.status.idle":"2023-09-08T14:26:16.231245Z","shell.execute_reply.started":"2023-09-08T14:26:16.212745Z","shell.execute_reply":"2023-09-08T14:26:16.230244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example 1: Adding a Convolutional Layer\nfrom tensorflow.keras.layers import Conv2D\n\n# Add a convolutional layer after the data augmentation\nx = Conv2D(64, (3, 3), activation='relu', padding='same')","metadata":{"execution":{"iopub.status.busy":"2023-09-08T14:26:32.565616Z","iopub.execute_input":"2023-09-08T14:26:32.566022Z","iopub.status.idle":"2023-09-08T14:26:32.574423Z","shell.execute_reply.started":"2023-09-08T14:26:32.565992Z","shell.execute_reply":"2023-09-08T14:26:32.573194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_cct_model(\n    image_size=image_size,\n    input_shape=input_shape,\n    num_heads=num_heads,\n    projection_dim=projection_dim,\n    transformer_units=transformer_units,\n):\n    inputs = layers.Input(input_shape)\n\n    # Augment data.\n    augmented = data_augmentation(inputs)\n\n    # Encode patches.\n    cct_tokenizer = CCTTokenizer()\n    encoded_patches = cct_tokenizer(augmented)\n\n    # Apply positional embedding.\n    if positional_emb:\n        pos_embed, seq_length = cct_tokenizer.positional_embedding(image_size)\n        positions = tf.range(start=0, limit=seq_length, delta=1)\n        position_embeddings = pos_embed(positions)\n        encoded_patches += position_embeddings\n\n    # Calculate Stochastic Depth probabilities.\n    dpr = [x for x in np.linspace(0, stochastic_depth_rate, transformer_layers)]\n\n    # Create multiple layers of the Transformer block.\n    for i in range(transformer_layers):\n        # Layer normalization 1.\n        x1 = layers.LayerNormalization(epsilon=1e-5)(encoded_patches)\n\n        # Create a multi-head attention layer.\n        attention_output = layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=projection_dim, dropout=0.1\n        )(x1, x1)\n\n        # Skip connection 1.\n        attention_output = StochasticDepth(dpr[i])(attention_output)\n        x2 = layers.Add()([attention_output, encoded_patches])\n\n        # Layer normalization 2.\n        x3 = layers.LayerNormalization(epsilon=1e-5)(x2)\n\n        # MLP.\n        x3 = mlp(x3, hidden_units=transformer_units, dropout_rate=0.1)\n\n        # Skip connection 2.\n        x3 = StochasticDepth(dpr[i])(x3)\n        encoded_patches = layers.Add()([x3, x2])\n\n    # Apply sequence pooling.\n    representation = layers.LayerNormalization(epsilon=1e-5)(encoded_patches)\n    attention_weights = tf.nn.softmax(layers.Dense(1)(representation), axis=1)\n    weighted_representation = tf.matmul(\n        attention_weights, representation, transpose_a=True\n    )\n    weighted_representation = tf.squeeze(weighted_representation, -2)\n\n    # Classify outputs.\n    logits = layers.Dense(num_classes)(weighted_representation)\n    # Create the Keras model.\n    model = keras.Model(inputs=inputs, outputs=logits)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:54:10.851265Z","iopub.execute_input":"2023-09-08T06:54:10.851717Z","iopub.status.idle":"2023-09-08T06:54:10.867614Z","shell.execute_reply.started":"2023-09-08T06:54:10.851681Z","shell.execute_reply":"2023-09-08T06:54:10.866091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def experiment(model):\n    optimizer = tfa.optimizers.AdamW(\n        learning_rate=learning_rate, weight_decay=weight_decay\n    )\n    \n    model.compile(\n        optimizer=optimizer,\n        loss=keras.losses.CategoricalCrossentropy(from_logits=True),\n        metrics=[\n            keras.metrics.CategoricalAccuracy(name=\"accuracy\"),\n            keras.metrics.AUC( name=\"AUC\"),\n        ],\n     )\n    checkpoint_filepath = \"./tmp/checkpoint\"\n    checkpoint_callback = keras.callbacks.ModelCheckpoint(\n        checkpoint_filepath,\n        monitor=\"val_accuracy\",\n        save_best_only=True,\n        save_weights_only=True,\n    )\n\n    history = model.fit(train_loader ,\n                        batch_size=batch_size,\n                        epochs=num_epochs,\n                        validation_data=test_loader,\n                        callbacks=[checkpoint_callback],)\n    model.load_weights(checkpoint_filepath)\n    _, accuracy, auc = model.evaluate(test_loader)\n    print(f\"Test accuracy: {round(accuracy * 100, 2)}%\")\n    print(f\"Test AUC: {round(auc * 100, 2)}%\")\n\n    return history","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:54:12.823777Z","iopub.execute_input":"2023-09-08T06:54:12.824499Z","iopub.status.idle":"2023-09-08T06:54:12.834949Z","shell.execute_reply.started":"2023-09-08T06:54:12.824462Z","shell.execute_reply":"2023-09-08T06:54:12.833371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cct_classifier=create_cct_model()\ncct_classifier.summary()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T14:26:52.207415Z","iopub.execute_input":"2023-09-08T14:26:52.207966Z","iopub.status.idle":"2023-09-08T14:26:53.093352Z","shell.execute_reply.started":"2023-09-08T14:26:52.207925Z","shell.execute_reply":"2023-09-08T14:26:53.091951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = experiment(cct_classifier)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:54:21.208004Z","iopub.execute_input":"2023-09-08T06:54:21.208486Z","iopub.status.idle":"2023-09-08T12:29:16.981034Z","shell.execute_reply.started":"2023-09-08T06:54:21.208451Z","shell.execute_reply":"2023-09-08T12:29:16.968596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history[\"loss\"], label=\"train_loss\")\nplt.plot(history.history[\"val_loss\"], label=\"val_loss\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Loss\")\nplt.title(\"Train and Validation Losses Over Epochs\", fontsize=14)\nplt.legend()\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T12:29:31.666667Z","iopub.execute_input":"2023-09-08T12:29:31.667241Z","iopub.status.idle":"2023-09-08T12:29:32.234621Z","shell.execute_reply.started":"2023-09-08T12:29:31.667192Z","shell.execute_reply":"2023-09-08T12:29:32.233099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list all data in history\nprint(history.history.keys())\n# summarize history for accuracy\nplt.figure(figsize=(12,10))\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n# summarize history for loss\nplt.figure(figsize=(12,10))\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T12:29:38.076985Z","iopub.execute_input":"2023-09-08T12:29:38.077550Z","iopub.status.idle":"2023-09-08T12:29:38.904623Z","shell.execute_reply.started":"2023-09-08T12:29:38.077507Z","shell.execute_reply":"2023-09-08T12:29:38.903095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cct_classifier.load_weights(\"./tmp/checkpoint\")","metadata":{"execution":{"iopub.status.busy":"2023-09-08T12:32:28.199918Z","iopub.execute_input":"2023-09-08T12:32:28.200406Z","iopub.status.idle":"2023-09-08T12:32:28.378545Z","shell.execute_reply.started":"2023-09-08T12:32:28.200371Z","shell.execute_reply":"2023-09-08T12:32:28.376918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_img_array(img):\n    \n    # `array` is a float32 Numpy array of shape (299, 299, 3)\n    array = keras.preprocessing.image.img_to_array(img)\n    # We add a dimension to transform our array into a \"batch\"\n    # of size (1, 299, 299, 3)\n    array = np.expand_dims(array, axis=0)\n    return array","metadata":{"execution":{"iopub.status.busy":"2023-09-08T12:32:56.880710Z","iopub.execute_input":"2023-09-08T12:32:56.881222Z","iopub.status.idle":"2023-09-08T12:32:56.890047Z","shell.execute_reply.started":"2023-09-08T12:32:56.881182Z","shell.execute_reply":"2023-09-08T12:32:56.888352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gradcam_heatmap(img_array, model, last_conv_layer_name, pred_index=None):\n    # First, we create a model that maps the input image to the activations\n    # of the last conv layer as well as the output predictions\n    grad_model = tf.keras.models.Model(\n        [model.input], [model.get_layer(last_conv_layer_name).output,  model.output]\n    )\n    \n    # Then, we compute the gradient of the top predicted class for our input image\n    # with respect to the activations of the last conv layer\n    with tf.GradientTape() as tape:\n        last_conv_layer_output, preds = grad_model(img_array)\n        if pred_index is None:\n            pred_index = tf.argmax(preds[0])\n        class_channel = preds[:, pred_index]\n        # This is the gradient of the output neuron (top predicted or chosen)\n    # with regard to the output feature map of the last conv layer\n    grads = tape.gradient(class_channel, last_conv_layer_output)\n\n    # This is a vector where each entry is the mean intensity of the gradient\n    # over a specific feature map channel\n    pooled_grads = tf.reduce_mean(grads, axis=(0, 1))\n    # We multiply each channel in the feature map array\n    # by \"how important this channel is\" with regard to the top predicted class\n    # then sum all the channels to obtain the heatmap class activation\n    last_conv_layer_output = last_conv_layer_output\n    heatmap = last_conv_layer_output @ pooled_grads[..., tf.newaxis]\n    heatmap = tf.squeeze(heatmap)\n    \n    # For visualization purpose, we will also normalize the heatmap between 0 & 1\n    heatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)\n    return heatmap.numpy()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T12:33:54.136332Z","iopub.execute_input":"2023-09-08T12:33:54.136827Z","iopub.status.idle":"2023-09-08T12:33:54.151282Z","shell.execute_reply.started":"2023-09-08T12:33:54.136792Z","shell.execute_reply":"2023-09-08T12:33:54.149734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes.values()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T12:34:10.039866Z","iopub.execute_input":"2023-09-08T12:34:10.041893Z","iopub.status.idle":"2023-09-08T12:34:10.052581Z","shell.execute_reply.started":"2023-09-08T12:34:10.041820Z","shell.execute_reply":"2023-09-08T12:34:10.050644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_gradcam(img, heatmap, cam_path=\"cam.jpg\", alpha=0.4, preds=[0,0,0,0,0], plot=None):\n\n    # Rescale heatmap to a range 0-255\n    heatmap = np.uint8(255 * heatmap)\n\n    # Use jet colormap to colorize heatmap\n    jet = cm.get_cmap(\"jet\")\n\n    # Use RGB values of the colormap\n    jet_colors = jet(np.arange(256))[:, :3]\n    jet_heatmap = jet_colors[heatmap]\n\n    # Create an image with RGB colorized heatmap\n    jet_heatmap = keras.preprocessing.image.array_to_img(jet_heatmap)\n    jet_heatmap = jet_heatmap.resize((img.shape[1], img.shape[0]))\n    jet_heatmap = keras.preprocessing.image.img_to_array(jet_heatmap)\n\n    # Superimpose the heatmap on original image\n    superimposed_img = jet_heatmap * alpha + img\n    superimposed_img = keras.preprocessing.image.array_to_img(superimposed_img)\n\n    # Save the superimposed image\n    #superimposed_img.save(cam_path)\n\n    # Display Grad CAM\n    plot.imshow(superimposed_img)\n    plot.set(title =\n        \" No DR: \\\n        {:.3f}\\nMild: \\\n        {:.3f}\\nModerate: \\\n        {:.3f}\\nSevere: \\\n        {:.3f}\\nProliferative: \\\n        {:.3f}\".format(preds[0], \\\n                    preds[1], \\\n                    preds[2], \\\n                    preds[3],\n                    preds[4])\n    )\n    plot.axis('off')","metadata":{"execution":{"iopub.status.busy":"2023-09-08T12:34:46.270693Z","iopub.execute_input":"2023-09-08T12:34:46.271378Z","iopub.status.idle":"2023-09-08T12:34:46.285239Z","shell.execute_reply.started":"2023-09-08T12:34:46.271330Z","shell.execute_reply":"2023-09-08T12:34:46.283172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# As in layer_normalization (LayerNorma (None, 1296, 64) ) \n#the last dim is 1296 so 36x36 for heatmap\nnp.sqrt(1296)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T14:30:57.964387Z","iopub.execute_input":"2023-09-08T14:30:57.965023Z","iopub.status.idle":"2023-09-08T14:30:57.975998Z","shell.execute_reply.started":"2023-09-08T14:30:57.964978Z","shell.execute_reply":"2023-09-08T14:30:57.974502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_image = next(iter(test_loader))[0][5]\n# Prepare image\nimg_array =get_img_array(test_image)\n\nlast_conv_layer_name = 'cct_tokenizer_2'\n# Remove last layer's softmax\ncct_classifier.layers[-1].activation = None\n# Print what the top predicted class is\npreds = cct_classifier.predict(img_array)\nprint(\"Predicted:\\n\" + \"No DR: \\\n    {p1}\\nMild: {p2}\\nModerate: \\\n    {p3}\\nSevere: \\\n    {p4}\\nProliferative: {p5}\".format(p1=preds[0][0], \\\n                                            p2=preds[0][1],p3=preds[0][2],p4=preds[0][3],p5=preds[0][4]))\n# Generate class activation heatmap\nheatmap = gradcam_heatmap(img_array, cct_classifier, last_conv_layer_name)\nheatmap = np.reshape(heatmap, (8, 8))\n\n# Display heatmap\nplt.matshow(heatmap)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T14:34:01.193727Z","iopub.execute_input":"2023-09-08T14:34:01.194194Z","iopub.status.idle":"2023-09-08T14:34:07.729147Z","shell.execute_reply.started":"2023-09-08T14:34:01.194137Z","shell.execute_reply":"2023-09-08T14:34:07.728184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(30, 30))\nfor images, ax in zip(next(iter(test_loader))[0][:6], axis.flat):\n    img_array = get_img_array(images)\n    # Remove last layer's softmax\n    cct_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = cct_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, cct_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (8,8))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T14:36:41.153368Z","iopub.execute_input":"2023-09-08T14:36:41.154769Z","iopub.status.idle":"2023-09-08T14:36:52.026883Z","shell.execute_reply.started":"2023-09-08T14:36:41.154697Z","shell.execute_reply":"2023-09-08T14:36:52.025130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 2, figsize=(30, 30))\nfor images, ax in zip(next(iter(test_loader))[0][6:12], axis.flat):\n    img_array = get_img_array(images)\n    # Remove last layer's softmax\n    cct_classifier.layers[-1].activation = None\n    # Print what the top predicted class is\n    preds = cct_classifier.predict(img_array)\n    heatmap = gradcam_heatmap(img_array, cct_classifier, last_conv_layer_name)\n\n    heatmap = np.reshape(heatmap, (8,8))\n    display_gradcam(images, heatmap, preds=preds[0], plot=ax)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T14:38:23.021520Z","iopub.execute_input":"2023-09-08T14:38:23.022181Z","iopub.status.idle":"2023-09-08T14:38:33.188877Z","shell.execute_reply.started":"2023-09-08T14:38:23.022115Z","shell.execute_reply":"2023-09-08T14:38:33.187193Z"},"trusted":true},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}