{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-10T06:29:53.992901Z","iopub.execute_input":"2023-06-10T06:29:53.993235Z","iopub.status.idle":"2023-06-10T06:30:01.911134Z","shell.execute_reply.started":"2023-06-10T06:29:53.993207Z","shell.execute_reply":"2023-06-10T06:30:01.910222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def append_ext(fn):\n    return fn+\".png\"\n\ntrain_dataframe = pd.read_csv(\"/kaggle/input/aptos2019-blindness-detection/train.csv\",dtype=str)\ntrain_dataframe['id_code'] = train_dataframe['id_code'].apply(append_ext)\ntrain_dataframe.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:30:06.361949Z","iopub.execute_input":"2023-06-10T06:30:06.362772Z","iopub.status.idle":"2023-06-10T06:30:06.402703Z","shell.execute_reply.started":"2023-06-10T06:30:06.362731Z","shell.execute_reply":"2023-06-10T06:30:06.401953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"diagnosis_counts = train_dataframe.groupby('diagnosis')['id_code'].count()\n\n# Plot the counts\ndiagnosis_counts.plot(kind='bar')\n\n# Add labels and title to the plot\nplt.xlabel('Diagnosis')\nplt.ylabel('Count')\nplt.title('Count of id_code for each diagnosis')\n\n# Display the plot\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:32:14.840331Z","iopub.execute_input":"2023-06-10T06:32:14.841304Z","iopub.status.idle":"2023-06-10T06:32:15.121282Z","shell.execute_reply.started":"2023-06-10T06:32:14.841269Z","shell.execute_reply":"2023-06-10T06:32:15.120414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX = train_dataframe['id_code']\ny = train_dataframe['diagnosis']\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1, random_state=42, shuffle=True)\nX_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.1, random_state=42, shuffle=True)\ntrain_df = pd.DataFrame({'id_code': X_train, 'diagnosis': y_train}).reset_index(drop=True)\nvalid_df = pd.DataFrame({'id_code': X_valid, 'diagnosis': y_valid}).reset_index(drop=True)\ntest_df = pd.DataFrame({'id_code': X_test, 'diagnosis': y_test}).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T16:35:49.922599Z","iopub.execute_input":"2023-06-07T16:35:49.922936Z","iopub.status.idle":"2023-06-07T16:35:50.084292Z","shell.execute_reply.started":"2023-06-07T16:35:49.922901Z","shell.execute_reply":"2023-06-07T16:35:50.083276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datagen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1 / 255.0,\n                                                               rotation_range = 10, \n                                                               zoom_range = 0.30, \n                                                               shear_range = 0.30,\n                                                               fill_mode = \"nearest\"\n                                                               )\n\ntest_datagen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1 / 255.0)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T16:35:50.087193Z","iopub.execute_input":"2023-06-07T16:35:50.087583Z","iopub.status.idle":"2023-06-07T16:35:50.092926Z","shell.execute_reply.started":"2023-06-07T16:35:50.087548Z","shell.execute_reply":"2023-06-07T16:35:50.091915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_directory = \"/kaggle/input/aptos2019-blindness-detection/train_images\"\ntrain_generator = train_datagen.flow_from_dataframe(dataframe = train_df,\n                                                    directory = images_directory,\n                                                    x_col = 'id_code',\n                                                    y_col = 'diagnosis',\n                                                    target_size=(224, 224),\n                                                    color_mode='rgb',\n                                                    class_mode='categorical',\n                                                    batch_size=16)\nvalid_generator = test_datagen.flow_from_dataframe(dataframe = valid_df,\n                                                    directory = images_directory,\n                                                    x_col = 'id_code',\n                                                    y_col = 'diagnosis',\n                                                    target_size=(224, 224),\n                                                    color_mode='rgb',\n                                                    class_mode='categorical',\n                                                    batch_size=16)\ntest_generator = test_datagen.flow_from_dataframe(dataframe = test_df,\n                                                    directory = images_directory,\n                                                    x_col = 'id_code',\n                                                    y_col = 'diagnosis',\n                                                    target_size=(224, 224),\n                                                    color_mode='rgb',\n                                                    class_mode='categorical',\n                                                    batch_size=16)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T16:35:50.094548Z","iopub.execute_input":"2023-06-07T16:35:50.095263Z","iopub.status.idle":"2023-06-07T16:35:57.843643Z","shell.execute_reply.started":"2023-06-07T16:35:50.095226Z","shell.execute_reply":"2023-06-07T16:35:57.842727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass Distiller(keras.Model):\n    def __init__(self, student, teacher):\n        super().__init__()\n        self.teacher = teacher\n        self.student = student\n\n    def compile(\n        self,\n        optimizer,\n        metrics,\n        student_loss_fn,\n        distillation_loss_fn,\n        alpha=0.1,\n        temperature=3,\n    ):\n        \"\"\" Configure the distiller.\n\n        Args:\n            optimizer: Keras optimizer for the student weights\n            metrics: Keras metrics for evaluation\n            student_loss_fn: Loss function of difference between student\n                predictions and ground-truth\n            distillation_loss_fn: Loss function of difference between soft\n                student predictions and soft teacher predictions\n            alpha: weight to student_loss_fn and 1-alpha to distillation_loss_fn\n            temperature: Temperature for softening probability distributions.\n                Larger temperature gives softer distributions.\n        \"\"\"\n        super().compile(optimizer=optimizer, metrics=metrics)\n        self.student_loss_fn = student_loss_fn\n        self.distillation_loss_fn = distillation_loss_fn\n        self.alpha = alpha\n        self.temperature = temperature\n\n    def train_step(self, data):\n        # Unpack data\n        x, y = data\n\n        # Forward pass of teacher\n        teacher_predictions = self.teacher(x, training=False)\n\n        with tf.GradientTape() as tape:\n            # Forward pass of student\n            student_predictions = self.student(x, training=True)\n\n            # Compute losses\n            student_loss = self.student_loss_fn(y, student_predictions)\n\n            # Compute scaled distillation loss from https://arxiv.org/abs/1503.02531\n            # The magnitudes of the gradients produced by the soft targets scale\n            # as 1/T^2, multiply them by T^2 when using both hard and soft targets.\n            distillation_loss = (\n                self.distillation_loss_fn(\n                    tf.nn.softmax(teacher_predictions / self.temperature, axis=1),\n                    tf.nn.softmax(student_predictions / self.temperature, axis=1),\n                )\n                * self.temperature**2\n            )\n\n            loss = self.alpha * student_loss + (1 - self.alpha) * distillation_loss\n\n        # Compute gradients\n        trainable_vars = self.student.trainable_variables\n        gradients = tape.gradient(loss, trainable_vars)\n\n        # Update weights\n        self.optimizer.apply_gradients(zip(gradients, trainable_vars))\n\n        # Update the metrics configured in `compile()`.\n        self.compiled_metrics.update_state(y, student_predictions)\n\n        # Return a dict of performance\n        results = {m.name: m.result() for m in self.metrics}\n        results.update(\n            {\"student_loss\": student_loss, \"distillation_loss\": distillation_loss}\n        )\n        return results\n\n    def test_step(self, data):\n        # Unpack the data\n        x, y = data\n\n        # Compute predictions\n        y_prediction = self.student(x, training=False)\n\n        # Calculate the loss\n        student_loss = self.student_loss_fn(y, y_prediction)\n\n        # Update the metrics.\n        self.compiled_metrics.update_state(y, y_prediction)\n\n        # Return a dict of performance\n        results = {m.name: m.result() for m in self.metrics}\n        results.update({\"student_loss\": student_loss})\n        return results\n","metadata":{"execution":{"iopub.status.busy":"2023-06-07T16:35:57.845159Z","iopub.execute_input":"2023-06-07T16:35:57.845523Z","iopub.status.idle":"2023-06-07T16:35:57.860477Z","shell.execute_reply.started":"2023-06-07T16:35:57.845475Z","shell.execute_reply":"2023-06-07T16:35:57.859163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model(model_name):\n    # Download the pre-trained model\n    pre_trained_model = tf.keras.applications.__dict__[model_name](weights='imagenet', include_top=False, pooling = 'avg')\n    \n    pre_trained_model.trainable = True\n\n    model_output = pre_trained_model.output\n    \n    # Add a dense layer with 10 units\n    dense_layer = tf.keras.layers.Dense(5, activation='softmax')(model_output)\n    \n    # Create a new model\n    model = tf.keras.models.Model(inputs=pre_trained_model.input, outputs=dense_layer)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-06-07T16:35:57.862090Z","iopub.execute_input":"2023-06-07T16:35:57.862829Z","iopub.status.idle":"2023-06-07T16:35:57.876660Z","shell.execute_reply.started":"2023-06-07T16:35:57.862795Z","shell.execute_reply":"2023-06-07T16:35:57.875645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the teacher\nteacher = create_model('DenseNet201')\n\n# Create the student\nstudent = create_model('ResNet50')","metadata":{"execution":{"iopub.status.busy":"2023-06-07T16:35:57.878679Z","iopub.execute_input":"2023-06-07T16:35:57.879339Z","iopub.status.idle":"2023-06-07T16:36:10.482945Z","shell.execute_reply.started":"2023-06-07T16:35:57.879298Z","shell.execute_reply":"2023-06-07T16:36:10.481955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"teacher.compile(\n    optimizer=keras.optimizers.Adam(learning_rate=1e-3),\n    loss=keras.losses.CategoricalCrossentropy(),\n    metrics=[keras.metrics.CategoricalAccuracy()],\n)\n\n# Train and evaluate teacher on data.\nteacher_history = teacher.fit(train_generator,\n                    validation_data = valid_generator,\n                    steps_per_epoch = train_generator.n//train_generator.batch_size,\n                    validation_steps = valid_generator.n//valid_generator.batch_size,epochs=50)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T16:36:10.484375Z","iopub.execute_input":"2023-06-07T16:36:10.484718Z","iopub.status.idle":"2023-06-07T16:46:25.861578Z","shell.execute_reply.started":"2023-06-07T16:36:10.484686Z","shell.execute_reply":"2023-06-07T16:46:25.860543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summarize history for accuracy\nplt.plot(teacher_history.history['categorical_accuracy'])\nplt.plot(teacher_history.history['val_categorical_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-07T16:52:40.816566Z","iopub.execute_input":"2023-06-07T16:52:40.817558Z","iopub.status.idle":"2023-06-07T16:52:41.126585Z","shell.execute_reply.started":"2023-06-07T16:52:40.817517Z","shell.execute_reply":"2023-06-07T16:52:41.125463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = teacher.evaluate(test_generator)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])","metadata":{"execution":{"iopub.status.busy":"2023-06-07T16:55:35.100736Z","iopub.execute_input":"2023-06-07T16:55:35.101783Z","iopub.status.idle":"2023-06-07T16:56:17.548321Z","shell.execute_reply.started":"2023-06-07T16:55:35.101747Z","shell.execute_reply":"2023-06-07T16:56:17.547173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nteacher_predictions = np.argmax(teacher.predict(test_generator),axis=1)\ncnf_matrix = confusion_matrix(test_generator.classes, teacher_predictions)\nax= plt.subplot()\nsns.heatmap(cnf_matrix, annot=True, fmt='g', ax=ax);\n\n# labels, title and ticks\nax.set_xlabel('Predicted labels');\nax.set_ylabel('True labels'); \nax.set_title('Confusion Matrix');\nax.xaxis.set_ticklabels(['No_DR', 'Mild','Moderate','Severe','Proliferative_DR']); \nax.yaxis.set_ticklabels(['No_DR', 'Mild','Moderate','Severe','Proliferative_DR']);","metadata":{"execution":{"iopub.status.busy":"2023-06-07T17:00:11.011850Z","iopub.execute_input":"2023-06-07T17:00:11.012208Z","iopub.status.idle":"2023-06-07T17:00:46.831444Z","shell.execute_reply.started":"2023-06-07T17:00:11.012181Z","shell.execute_reply":"2023-06-07T17:00:46.830549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"distiller = Distiller(student=student, teacher=teacher)\ndistiller.compile(\n    optimizer=keras.optimizers.Adam(),\n    metrics=[keras.metrics.CategoricalAccuracy()],\n    student_loss_fn=keras.losses.CategoricalCrossentropy(),\n    distillation_loss_fn=keras.losses.KLDivergence(),\n    alpha=0.1,\n    temperature=20,\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T17:01:06.847859Z","iopub.execute_input":"2023-06-07T17:01:06.848232Z","iopub.status.idle":"2023-06-07T17:01:06.878303Z","shell.execute_reply.started":"2023-06-07T17:01:06.848203Z","shell.execute_reply":"2023-06-07T17:01:06.877391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distill teacher to student\ndistiller.fit(train_generator,\n                    validation_data = valid_generator,\n                    steps_per_epoch = train_generator.n//train_generator.batch_size,\n                    validation_steps = valid_generator.n//valid_generator.batch_size,epochs=50)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T17:01:11.489457Z","iopub.execute_input":"2023-06-07T17:01:11.490146Z","iopub.status.idle":"2023-06-07T17:10:55.447466Z","shell.execute_reply.started":"2023-06-07T17:01:11.490110Z","shell.execute_reply":"2023-06-07T17:10:55.446285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate student on test dataset\nscore = distiller.evaluate(test_generator)\nprint('Categorical_accuracy:', score[0])\nprint('student_loss:', score[1])","metadata":{"execution":{"iopub.status.busy":"2023-06-07T17:11:00.304245Z","iopub.execute_input":"2023-06-07T17:11:00.304725Z","iopub.status.idle":"2023-06-07T17:11:43.010867Z","shell.execute_reply.started":"2023-06-07T17:11:00.304686Z","shell.execute_reply":"2023-06-07T17:11:43.009884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"student_predictions = np.argmax(distiller.student.predict(test_generator),axis=1)\ncnf_matrix = confusion_matrix(test_generator.classes, student_predictions)\nax= plt.subplot()\nsns.heatmap(cnf_matrix, annot=True, fmt='g', ax=ax);\n\n# labels, title and ticks\nax.set_xlabel('Predicted labels');\nax.set_ylabel('True labels'); \nax.set_title('Confusion Matrix');\nax.xaxis.set_ticklabels(['No_DR', 'Mild','Moderate','Severe','Proliferative_DR']); \nax.yaxis.set_ticklabels(['No_DR', 'Mild','Moderate','Severe','Proliferative_DR']);","metadata":{"execution":{"iopub.status.busy":"2023-06-07T17:19:42.029467Z","iopub.execute_input":"2023-06-07T17:19:42.029849Z","iopub.status.idle":"2023-06-07T17:20:18.266366Z","shell.execute_reply.started":"2023-06-07T17:19:42.029821Z","shell.execute_reply":"2023-06-07T17:20:18.265396Z"},"trusted":true},"execution_count":null,"outputs":[]}]}