{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":9398912,"sourceType":"datasetVersion","datasetId":5704900},{"sourceId":199442909,"sourceType":"kernelVersion"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Flatten, Dense, Dropout, Input\nfrom tensorflow.keras.optimizers import Adam\n\n###\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models, optimizers, callbacks\nfrom tensorflow.keras.applications import DenseNet201\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-05T06:38:00.375708Z","iopub.execute_input":"2024-10-05T06:38:00.375995Z","iopub.status.idle":"2024-10-05T06:38:00.413007Z","shell.execute_reply.started":"2024-10-05T06:38:00.375964Z","shell.execute_reply":"2024-10-05T06:38:00.412258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Distiller(keras.Model):\n    def __init__(self, student, teacher):\n        super(Distiller, self).__init__()\n        self.teacher = teacher\n        self.student = student\n\n    def compile(\n        self,\n        optimizer,\n        metrics,\n        student_loss_fn,\n        distillation_loss_fn,\n        alpha=0.1,\n        temperature=3,\n    ):\n        \"\"\" Configure the distiller.\n\n        Args:\n            optimizer: Keras optimizer for the student weights\n            metrics: Keras metrics for evaluation\n            student_loss_fn: Loss function of difference between student\n                predictions and ground-truth\n            distillation_loss_fn: Loss function of difference between soft\n                student predictions and soft teacher predictions\n            alpha: weight to student_loss_fn and 1-alpha to distillation_loss_fn\n            temperature: Temperature for softening probability distributions.\n                Larger temperature gives softer distributions.\n        \"\"\"\n        super(Distiller, self).compile(optimizer=optimizer, metrics=metrics)\n        self.student_loss_fn = student_loss_fn\n        self.distillation_loss_fn = distillation_loss_fn\n        self.alpha = alpha\n        self.temperature = temperature\n\n    def train_step(self, data):\n        # Unpack data\n        x, y = data\n\n        # Forward pass of teacher\n        teacher_predictions = self.teacher(x, training=False)\n\n        with tf.GradientTape() as tape:\n            # Forward pass of student\n            student_predictions = self.student(x, training=True)\n\n            # Compute losses\n            student_loss = self.student_loss_fn(y, student_predictions)\n\n            # Compute scaled distillation loss from https://arxiv.org/abs/1503.02531\n            # The magnitudes of the gradients produced by the soft targets scale\n            # as 1/T^2, multiply them by T^2 when using both hard and soft targets.\n            distillation_loss = (\n                self.distillation_loss_fn(\n                    tf.nn.softmax(teacher_predictions / self.temperature, axis=1),\n                    tf.nn.softmax(student_predictions / self.temperature, axis=1),\n                )\n                * self.temperature**2\n            )\n\n            loss = self.alpha * student_loss + (1 - self.alpha) * distillation_loss\n\n        # Compute gradients\n        trainable_vars = self.student.trainable_variables\n        gradients = tape.gradient(loss, trainable_vars)\n\n        # Update weights\n        self.optimizer.apply_gradients(zip(gradients, trainable_vars))\n\n        # Update the metrics configured in `compile()`.\n        self.compiled_metrics.update_state(y, student_predictions)\n\n        # Return a dict of performance\n        results = {m.name: m.result() for m in self.metrics}\n        results.update(\n            {\"student_loss\": student_loss, \"distillation_loss\": distillation_loss}\n        )\n        return results\n\n    def test_step(self, data):\n        # Unpack the data\n        x, y = data\n\n        # Compute predictions\n        y_prediction = self.student(x, training=False)\n\n        # Calculate the loss\n        student_loss = self.student_loss_fn(y, y_prediction)\n\n        # Update the metrics.\n        self.compiled_metrics.update_state(y, y_prediction)\n\n        # Return a dict of performance\n        results = {m.name: m.result() for m in self.metrics}\n        results.update({\"student_loss\": student_loss})\n        return results","metadata":{"execution":{"iopub.status.busy":"2024-10-05T06:38:18.445801Z","iopub.execute_input":"2024-10-05T06:38:18.446639Z","iopub.status.idle":"2024-10-05T06:38:18.460322Z","shell.execute_reply.started":"2024-10-05T06:38:18.446595Z","shell.execute_reply":"2024-10-05T06:38:18.459297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Distiller(tf.keras.Model):\n    def __init__(self, student, teacher):\n        super(Distiller, self).__init__()\n        self.teacher = teacher\n        self.student = student\n\n    def compile(\n        self, optimizer, metrics, student_loss_fn, distillation_loss_fn, alpha=0.1, temperature=3\n    ):\n        super(Distiller, self).compile(optimizer=optimizer, metrics=metrics)\n        self.student_loss_fn = student_loss_fn\n        self.distillation_loss_fn = distillation_loss_fn\n        self.alpha = alpha\n        self.temperature = temperature\n\n    def train_step(self, data):\n        x, y = data\n        teacher_predictions = self.teacher(x, training=False)\n\n        with tf.GradientTape() as tape:\n            student_predictions = self.student(x, training=True)\n            student_loss = self.student_loss_fn(y, student_predictions)\n            distillation_loss = self.distillation_loss_fn(\n                tf.nn.softmax(teacher_predictions / self.temperature, axis=1),\n                tf.nn.softmax(student_predictions / self.temperature, axis=1),\n            ) * self.temperature**2\n\n            loss = self.alpha * student_loss + (1 - self.alpha) * distillation_loss\n\n        gradients = tape.gradient(loss, self.student.trainable_variables)\n        self.optimizer.apply_gradients(zip(gradients, self.student.trainable_variables))\n        self.compiled_metrics.update_state(y, student_predictions)\n\n        results = {m.name: m.result() for m in self.metrics}\n        results.update({\"student_loss\": student_loss, \"distillation_loss\": distillation_loss})\n        return results\n\n    def test_step(self, data):\n        x, y = data\n        student_predictions = self.student(x, training=False)\n        student_loss = self.student_loss_fn(y, student_predictions)\n        self.compiled_metrics.update_state(y, student_predictions)\n\n        results = {m.name: m.result() for m in self.metrics}\n        results.update({\"student_loss\": student_loss})\n        return results","metadata":{"execution":{"iopub.status.busy":"2024-10-05T06:38:33.216379Z","iopub.execute_input":"2024-10-05T06:38:33.216774Z","iopub.status.idle":"2024-10-05T06:38:33.229534Z","shell.execute_reply.started":"2024-10-05T06:38:33.216737Z","shell.execute_reply":"2024-10-05T06:38:33.228512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Augmentations\ntrain_datagen_with_aug = ImageDataGenerator(\n    rescale=1./255,\n    shear_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    validation_split=0.2\n)\n\ntrain_generator = train_datagen_with_aug.flow_from_directory(\n    r'/kaggle/input/aptos-training-split-in-5-classes/train_split',\n    target_size=(224, 224),\n    batch_size=32,\n    # color_mode='grayscale',\n    class_mode='sparse',\n    subset='training'\n    ) \n\nvalidation_generator = train_datagen_with_aug.flow_from_directory(\n    r'/kaggle/input/aptos-training-split-in-5-classes/train_split', # same directory as training data\n    target_size=(224, 224),\n    batch_size=32,\n    # color_mode='grayscale',\n    class_mode='sparse',\n    subset='validation'\n    )\n\n#input_img = Input(shape=(224, 224, 3))","metadata":{"execution":{"iopub.status.busy":"2024-10-05T06:38:42.692567Z","iopub.execute_input":"2024-10-05T06:38:42.693209Z","iopub.status.idle":"2024-10-05T06:38:45.564011Z","shell.execute_reply.started":"2024-10-05T06:38:42.693168Z","shell.execute_reply":"2024-10-05T06:38:45.563195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load DenseNet-201 model without the top layers (pre-trained on ImageNet)\ndensenet201_model = DenseNet201(weights='imagenet', include_top=False, input_shape=(224, 224, 3))\n\n# Create the teacher model\nteacher = models.Sequential([\n    densenet201_model,                  # DenseNet base model\n    layers.GlobalAveragePooling2D(),    # Reduces dimensionality (spatial pooling)\n    layers.Dropout(0.2),                # Dropout to prevent overfitting\n    layers.Dense(1024, activation='relu'), # Fully connected layer with ReLU activation\n    layers.Dropout(0.2),                # Dropout\n    layers.Dense(512, activation='relu'),  # Another dense layer with ReLU\n    layers.Dense(5, activation='softmax')  # Final output layer (assuming 5 classes)\n])\n\n# Create the student model\nstudent = models.Sequential([\n    layers.Input(shape=(224, 224, 3)),\n    layers.Conv2D(48, (5, 5), strides=(2, 2), padding=\"same\"),\n    layers.ReLU(),\n    layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding=\"same\"),\n\n    layers.Conv2D(64, (3, 3), strides=(2, 2), padding=\"same\"),\n    layers.ReLU(),\n    layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding=\"same\"),\n\n\n    layers.Conv2D(128, (3, 3), strides=(2, 2), padding=\"same\"),\n    layers.ReLU(),\n    layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding=\"same\"),\n\n    layers.Conv2D(256, (3, 3), strides=(2, 2), padding=\"same\"),\n    layers.ReLU(),\n    layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding=\"same\"),\n\n    layers.Conv2D(256, (3, 3), strides=(1, 1), padding=\"same\"),\n    layers.ReLU(),\n    layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding=\"same\"),\n\n    layers.Conv2D(64, (3, 3), strides=(2, 2), padding=\"same\"),\n    layers.ReLU(),\n    layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding=\"same\"),\n\n    layers.Flatten(),\n    layers.Dense(600, activation='relu'),\n    layers.Dense(300, activation='relu'),\n    layers.Dense(150, activation='relu'),\n    layers.Dense(5, activation='softmax')\n])\n\nstudent.summary()\n\n# Clone the student model for scratch training comparison\nstudent_scratch = models.clone_model(student)\n'''\n# Image data generators\ntrain_datagen_with_aug = ImageDataGenerator(\n    rescale=1./255,\n    shear_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    validation_split=0.2\n)\n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\ntrain_generator = train_datagen_with_aug.flow_from_directory(\n    r'/Users/hardikchhipa/Desktop/DR-Arvind/Aravind eye dataset/aptos2019-blindness-detection/train_split',\n    target_size=(224, 224),\n    batch_size=32,\n    # color_mode='grayscale',\n    class_mode='sparse',\n    subset='training'\n    ) # set as training data\n\nvalidation_generator = train_datagen_with_aug.flow_from_directory(\n    r'/Users/hardikchhipa/Desktop/DR-Arvind/Aravind eye dataset/aptos2019-blindness-detection/train_split', # same directory as training data\n    target_size=(224, 224),\n    batch_size=32,\n    # color_mode='grayscale',\n    class_mode='sparse',\n    subset='validation'\n    ) # set as validation data\n\n#input_img = Input(shape=(224, 224, 3))\n'''","metadata":{"execution":{"iopub.status.busy":"2024-10-05T06:39:03.556911Z","iopub.execute_input":"2024-10-05T06:39:03.557631Z","iopub.status.idle":"2024-10-05T06:39:11.782294Z","shell.execute_reply.started":"2024-10-05T06:39:03.557588Z","shell.execute_reply":"2024-10-05T06:39:11.781394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import Callback\nclass MaxMinMetricsTracker(Callback):\n    def __init__(self):\n        super(MaxMinMetricsTracker, self).__init__()\n        self.max_sparse_categorical_accuracy = 0.0\n        self.min_student_loss = float('inf')\n        self.min_distillation_loss = float('inf')\n        self.max_val_sparse_categorical_accuracy = 0.0\n        self.min_val_student_loss = float('inf')\n    \n    def on_epoch_end(self, epoch, logs=None):\n        if logs.get('sparse_categorical_accuracy') > self.max_sparse_categorical_accuracy:\n            self.max_sparse_categorical_accuracy = logs.get('sparse_categorical_accuracy')\n        \n        if logs.get('student_loss') < self.min_student_loss:\n            self.min_student_loss = logs.get('student_loss')\n        \n        if logs.get('distillation_loss') < self.min_distillation_loss:\n            self.min_distillation_loss = logs.get('distillation_loss')\n        \n        if logs.get('val_sparse_categorical_accuracy') > self.max_val_sparse_categorical_accuracy:\n            self.max_val_sparse_categorical_accuracy = logs.get('val_sparse_categorical_accuracy')\n        \n        if logs.get('val_student_loss') < self.min_val_student_loss:\n            self.min_val_student_loss = logs.get('val_student_loss')\n        \n        print(f\"Epoch {epoch + 1}:\")\n        print(f\"Max sparse_categorical_accuracy: {self.max_sparse_categorical_accuracy:.4f}\")\n        print(f\"Min student_loss: {self.min_student_loss:.4f}\")\n        print(f\"Min distillation_loss: {self.min_distillation_loss:.4f}\")\n        print(f\"Max val_sparse_categorical_accuracy: {self.max_val_sparse_categorical_accuracy:.4f}\")\n        print(f\"Min val_student_loss: {self.min_val_student_loss:.4f}\")\n\n\ndistiller = Distiller(student=student, teacher=teacher)\ndistiller.compile(\n    optimizer=tf.keras.optimizers.Adam(),\n    metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],\n    student_loss_fn=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n    distillation_loss_fn=tf.keras.losses.KLDivergence(),\n    alpha=0.1,\n    temperature=6,\n)\n\nmax_min_tracker = MaxMinMetricsTracker()\n\nhistory = distiller.fit(\n    train_generator,\n    epochs=25,\n    validation_data=validation_generator,\n    callbacks=[max_min_tracker]\n)","metadata":{"execution":{"iopub.status.busy":"2024-10-05T06:39:23.077387Z","iopub.execute_input":"2024-10-05T06:39:23.078085Z","iopub.status.idle":"2024-10-05T06:40:37.807148Z","shell.execute_reply.started":"2024-10-05T06:39:23.078043Z","shell.execute_reply":"2024-10-05T06:40:37.805766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nfig1 = plt.gcf()\n\nplt.plot(history.history['sparse_categorical_accuracy'])\nplt.plot(history.history['val_sparse_categorical_accuracy'])\n\nplt.grid()\n\nplt.title('OSPS-MicroNet model Accuracy to its teacher Densenet201')\nplt.ylabel('Accuracy')\nplt.xlabel('Epochs')\n\nplt.legend(['train', 'validation'])\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_acc = max(history.history['val_sparse_categorical_accuracy'])\nprint(f'The highest accuracy achieved for OSPS-MicroNet model Accuracy to its teacher densenet201 model is {max_acc * 100:.2f}%')\n","metadata":{},"execution_count":null,"outputs":[]}]}