{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CLASSIFICATION OF STAGES OF DIABETIC RETINOPATHY USING DEEP LEARNING METHODS","metadata":{}},{"cell_type":"markdown","source":"Images were taken from\n1. https://www.kaggle.com/competitions/diabetic-retinopathy-detection/data, \n2. https://www.kaggle.com/competitions/aptos2019-blindness-detection/data\n3. https://ieee-dataport.org/open-access/indian-diabetic-retinopathy-image-dataset-idrid\n\nImages were preprocessed using preprocessing function and uploaded","metadata":{}},{"cell_type":"code","source":"import warnings\n\nwarnings.simplefilter('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:32:26.236785Z","iopub.execute_input":"2022-04-27T07:32:26.237045Z","iopub.status.idle":"2022-04-27T07:32:26.240893Z","shell.execute_reply.started":"2022-04-27T07:32:26.237017Z","shell.execute_reply":"2022-04-27T07:32:26.239943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import libraries and packages\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport cv2\nimport io\nimport os\nfrom sklearn import metrics\nfrom sklearn.utils import shuffle","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-04-27T07:32:26.709853Z","iopub.execute_input":"2022-04-27T07:32:26.710396Z","iopub.status.idle":"2022-04-27T07:32:26.71463Z","shell.execute_reply.started":"2022-04-27T07:32:26.710358Z","shell.execute_reply":"2022-04-27T07:32:26.713952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow import keras\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.layers import (Add, Input, Dropout, Activation, BatchNormalization, ZeroPadding2D, AveragePooling2D)\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.optimizers import Adam, SGD\nfrom tensorflow.keras.layers import Conv2D\nfrom tensorflow.keras.layers import MaxPooling2D, GlobalAveragePooling2D\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.layers import Flatten\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.applications import ResNet50, InceptionV3, DenseNet121\nfrom tensorflow.keras.callbacks import TensorBoard, ModelCheckpoint, Callback\n\n%matplotlib inline","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-04-27T07:32:27.312711Z","iopub.execute_input":"2022-04-27T07:32:27.313543Z","iopub.status.idle":"2022-04-27T07:32:27.325648Z","shell.execute_reply.started":"2022-04-27T07:32:27.313491Z","shell.execute_reply":"2022-04-27T07:32:27.324779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read Data","metadata":{}},{"cell_type":"code","source":"aptos_gt = pd.read_csv(\"../input/classify-diabeticretinopathy/DR_Datasets/Datasets/APTOS/train.csv\")\neyepacs_gt = pd.read_csv(\"../input/classify-diabeticretinopathy/DR_Datasets/Datasets/EyePACS/trainLabels.csv\")\nIDRiD_gt = pd.read_csv(\"../input/classify-diabeticretinopathy/DR_Datasets/Datasets/IDRiD/Training_Labels.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:32:28.422086Z","iopub.execute_input":"2022-04-27T07:32:28.422334Z","iopub.status.idle":"2022-04-27T07:32:28.461922Z","shell.execute_reply.started":"2022-04-27T07:32:28.422306Z","shell.execute_reply":"2022-04-27T07:32:28.461245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Rename columns\n\naptos_gt = aptos_gt.rename(columns={\"id_code\":\"image\", \"diagnosis\": \"grade\"})\neyepacs_gt = eyepacs_gt.rename(columns = {\"level\":\"grade\"})\nIDRiD_gt = IDRiD_gt.rename(columns = {\"Image name\":\"image\", \"Retinopathy grade\":\"grade\"})","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:32:28.757117Z","iopub.execute_input":"2022-04-27T07:32:28.757767Z","iopub.status.idle":"2022-04-27T07:32:28.765917Z","shell.execute_reply.started":"2022-04-27T07:32:28.757722Z","shell.execute_reply":"2022-04-27T07:32:28.76503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eyepacs_gt['ext'] = \".jpg\"","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:32:29.289232Z","iopub.execute_input":"2022-04-27T07:32:29.289631Z","iopub.status.idle":"2022-04-27T07:32:29.29531Z","shell.execute_reply.started":"2022-04-27T07:32:29.289598Z","shell.execute_reply":"2022-04-27T07:32:29.294561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eyepacs_gt['image'] = eyepacs_gt['image'].astype(str) + eyepacs_gt['ext'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:32:29.748878Z","iopub.execute_input":"2022-04-27T07:32:29.74917Z","iopub.status.idle":"2022-04-27T07:32:29.76909Z","shell.execute_reply.started":"2022-04-27T07:32:29.749137Z","shell.execute_reply":"2022-04-27T07:32:29.76845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eyepacs_path = \"../input/classify-diabeticretinopathy/DR_Datasets/Datasets/EyePACS/train_jpg_eyepacs/\"\ndir_list = os.listdir(eyepacs_path)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:32:30.347779Z","iopub.execute_input":"2022-04-27T07:32:30.348197Z","iopub.status.idle":"2022-04-27T07:32:30.366827Z","shell.execute_reply.started":"2022-04-27T07:32:30.348161Z","shell.execute_reply":"2022-04-27T07:32:30.366162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eyepacs_gt = eyepacs_gt[eyepacs_gt[\"image\"].isin(dir_list)]\nlen(eyepacs_gt)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:32:31.068925Z","iopub.execute_input":"2022-04-27T07:32:31.069457Z","iopub.status.idle":"2022-04-27T07:32:31.096317Z","shell.execute_reply.started":"2022-04-27T07:32:31.069419Z","shell.execute_reply":"2022-04-27T07:32:31.095671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add path to the training set \n \naptos_gt['path'] = \"../input/classify-diabeticretinopathy/DR_Datasets/Datasets/APTOS/train_jpg_aptos/\"\neyepacs_gt['path'] = \"../input/classify-diabeticretinopathy/DR_Datasets/Datasets/EyePACS/train_jpg_eyepacs/\"\nIDRiD_gt['path'] = \"../input/classify-diabeticretinopathy/DR_Datasets/Datasets/IDRiD/train_jpg_idrid/\"\n\n# Add extension column to training set\naptos_gt['ext'] = \".jpg\"\neyepacs_gt['ext'] = \".jpg\"\nIDRiD_gt['ext'] = \".jpg\"\n\n# Add image path column to training set\n\naptos_gt['image_path'] = aptos_gt['path'].astype(str) + aptos_gt['image'].astype(str) + aptos_gt['ext'].astype(str)\neyepacs_gt['image_path'] = eyepacs_gt['path'].astype(str) + eyepacs_gt['image'].astype(str)\nIDRiD_gt['image_path'] = IDRiD_gt['path'].astype(str) + IDRiD_gt['image'].astype(str) + IDRiD_gt['ext'].astype(str)\n\n# Keep only image_path and grade columns\n\naptos_gt = aptos_gt[['image_path', 'grade']]\neyepacs_gt = eyepacs_gt[['image_path', 'grade']]\nIDRiD_gt = IDRiD_gt[['image_path','grade']]","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:32:31.605322Z","iopub.execute_input":"2022-04-27T07:32:31.605585Z","iopub.status.idle":"2022-04-27T07:32:31.633623Z","shell.execute_reply.started":"2022-04-27T07:32:31.605554Z","shell.execute_reply":"2022-04-27T07:32:31.632987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eyepacs_gt.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:32:31.811965Z","iopub.execute_input":"2022-04-27T07:32:31.812174Z","iopub.status.idle":"2022-04-27T07:32:31.820238Z","shell.execute_reply.started":"2022-04-27T07:32:31.812151Z","shell.execute_reply":"2022-04-27T07:32:31.819585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a function to plot images\n\ndef plot_image(images, captions=None, cmap=None):\n    f, axes = plt.subplots(1, len(images), sharey=True)\n    f.set_figwidth(15)\n    \n    for ax,image,caption in zip(axes, images, captions):\n        ax.imshow(image, cmap)\n        ax.set_title(caption)\n        plt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:32:32.702953Z","iopub.execute_input":"2022-04-27T07:32:32.703477Z","iopub.status.idle":"2022-04-27T07:32:32.708869Z","shell.execute_reply.started":"2022-04-27T07:32:32.703439Z","shell.execute_reply":"2022-04-27T07:32:32.7081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Concatenate the 3 datasets\n\ndata_list = [IDRiD_gt, eyepacs_gt]\ndr_df = pd.concat(data_list, axis=0)\ntrain_df = shuffle(dr_df)\ntest_df = shuffle(aptos_gt)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:32:32.774792Z","iopub.execute_input":"2022-04-27T07:32:32.775072Z","iopub.status.idle":"2022-04-27T07:32:32.789142Z","shell.execute_reply.started":"2022-04-27T07:32:32.775048Z","shell.execute_reply":"2022-04-27T07:32:32.788478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Pre-processing","metadata":{}},{"cell_type":"code","source":"# Define a function to preprocess the images. Function will perform Gaussian blur on all images, then use Canny edge detection\n# to detect edges of the images and then crop the images. Then CLAHE will be applied to increase visibility of lesions\n\n\ndef preprocess_img(image):\n    \n# Perform Gaussian blur\n    \n    gaus_img = cv2.GaussianBlur(image, (15,15), 0)    # Perform Gaussian blur on image with 15x15 kernel size filter\n    \n    # CLAHE can be applied on greyscale images or images which have been converted to LAB\n    # (L for lightness and A and B for the color opponents green–red and blue–yellow)\n    \n    clahe = cv2.createCLAHE(clipLimit=8.0,tileGridSize=(20, 20))\n    lab_img = cv2.cvtColor(gaus_img, cv2.COLOR_BGR2LAB)    # Convert image to LAB \n    lab_channels = cv2.split(lab_img)    # Split the image to components\n    lab_channels[0] = clahe.apply(lab_channels[0])   # Apply CLAHE on LAB channels\n    lab_img = cv2.merge(lab_channels)   # Merge LAB channels\n    clahe_img = cv2.cvtColor(lab_img, cv2.COLOR_LAB2RGB)    # Change LAB to RGB \n    \n    # Crop images using Canny edge detection to discover edges of the images\n    \n    edges = cv2.Canny(clahe_img,100, 110)    # Perform Canny edge detection to detect edges\n    indices = np.argwhere(edges != 0)    # Find indices of all points in Canny edge detection\n    y1,x1 = indices.min(axis=0)    # Find minimum value of x and y coordinates\n    y2,x2 = indices.max(axis=0)    # Find maximum value of x and y coordinates\n    crop_img = clahe_img[y1:y2, x1:x2]    # Crop image based on maximum and minimum x and y coordinates   \n\n    # Resized images to 224x224\n    dim = (224, 224)\n    resized_img = cv2.resize(crop_img, dim, interpolation = cv2.INTER_AREA)\n    \n    return resized_img","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:32:42.910036Z","iopub.execute_input":"2022-04-27T07:32:42.910295Z","iopub.status.idle":"2022-04-27T07:32:42.919696Z","shell.execute_reply.started":"2022-04-27T07:32:42.910268Z","shell.execute_reply":"2022-04-27T07:32:42.91901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale=1./255, validation_split=0.1)\n\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\nbatch = 32\n\ntrain_generator = train_datagen.flow_from_dataframe(\n    train_df, x_col='image_path', y_col='grade',\n    target_size=(224, 224),\n    batch_size=32,\n    class_mode='raw',\n    subset='training')\n\ntest_generator = test_datagen.flow_from_dataframe(\n    test_df, x_col='image_path', y_col='grade',\n    target_size=(224, 224),\n    batch_size=32,\n    class_mode='raw')\n\nvalidation_generator = train_datagen.flow_from_dataframe(\n    train_df, x_col='image_path', y_col='grade',\n    target_size=(224, 224),\n    batch_size=32,\n    class_mode='raw',\n    subset='validation')","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:33:15.624201Z","iopub.execute_input":"2022-04-27T07:33:15.624459Z","iopub.status.idle":"2022-04-27T07:34:14.542989Z","shell.execute_reply.started":"2022-04-27T07:33:15.624431Z","shell.execute_reply":"2022-04-27T07:34:14.542163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Building and Training","metadata":{}},{"cell_type":"code","source":"opt = Adam(lr=0.0001)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:34:14.544874Z","iopub.execute_input":"2022-04-27T07:34:14.545336Z","iopub.status.idle":"2022-04-27T07:34:14.550415Z","shell.execute_reply.started":"2022-04-27T07:34:14.545295Z","shell.execute_reply":"2022-04-27T07:34:14.549565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model(base_model, gal_name, pl_name):\n    base_model.trainable = True\n    global_average_layer = GlobalAveragePooling2D(name=gal_name)(base_model.output)\n    prediction_layer = Dense(5, activation='softmax', name = pl_name)(global_average_layer)\n    model = Model(inputs=base_model.input, outputs=prediction_layer)\n    model.compile(optimizer=opt, loss='sparse_categorical_crossentropy', metrics=[\"accuracy\"])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:34:14.55187Z","iopub.execute_input":"2022-04-27T07:34:14.552139Z","iopub.status.idle":"2022-04-27T07:34:14.563022Z","shell.execute_reply.started":"2022-04-27T07:34:14.552099Z","shell.execute_reply":"2022-04-27T07:34:14.562319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 32\nepochs = 20\ndef fit_model(model):\n    history = model.fit_generator(train_generator,\n                      steps_per_epoch=len(train_generator)//batch_size, \n                        epochs=epochs, \n                        validation_data=(validation_generator), \n                        validation_steps=len(validation_generator)//batch_size)\n    return history","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:34:23.277928Z","iopub.execute_input":"2022-04-27T07:34:23.278192Z","iopub.status.idle":"2022-04-27T07:34:23.282975Z","shell.execute_reply.started":"2022-04-27T07:34:23.278164Z","shell.execute_reply":"2022-04-27T07:34:23.282301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(history):\n    acc = history.history['accuracy']\n    val_acc = history.history['val_accuracy']\n\n    loss=history.history['loss']\n    val_loss=history.history['val_loss']\n\n    epochs_range = range(epochs)\n\n    plt.figure(figsize=(12, 6))\n    plt.subplot(1, 2, 1)\n    plt.plot(epochs_range, acc, label='Training Accuracy')\n    plt.plot(epochs_range, val_acc, label='Validation Accuracy')\n    plt.legend(loc='lower right')\n    plt.title('Training and Validation Accuracy')\n\n    plt.subplot(1, 2, 2)\n    plt.plot(epochs_range, loss, label='Training Loss')\n    plt.plot(epochs_range, val_loss, label='Validation Loss')\n    plt.legend(loc='upper right')\n    plt.title('Training and Validation Loss')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:34:24.344849Z","iopub.execute_input":"2022-04-27T07:34:24.345626Z","iopub.status.idle":"2022-04-27T07:34:24.353715Z","shell.execute_reply.started":"2022-04-27T07:34:24.345572Z","shell.execute_reply":"2022-04-27T07:34:24.352897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train ResNet-50, Inception-V3 and DenseNet-121 models","metadata":{}},{"cell_type":"code","source":"IMG_SHAPE = (224, 224, 3)\nresnet_model = ResNet50(input_shape=IMG_SHAPE, include_top=False, weights=\"imagenet\")\nres_model = create_model(resnet_model, gal_name=\"Resnet_gal\", pl_name='Resnet_pl' )\nres_history = fit_model(res_model)\n#res_model.save('res_models/res_model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:34:37.824768Z","iopub.execute_input":"2022-04-27T07:34:37.825636Z","iopub.status.idle":"2022-04-27T07:37:30.583542Z","shell.execute_reply.started":"2022-04-27T07:34:37.825596Z","shell.execute_reply":"2022-04-27T07:37:30.582762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for layer in res_model.layers:\n    layer._name = layer.name + str(\"_res\")","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:37:30.585394Z","iopub.execute_input":"2022-04-27T07:37:30.585659Z","iopub.status.idle":"2022-04-27T07:37:30.594876Z","shell.execute_reply.started":"2022-04-27T07:37:30.585625Z","shell.execute_reply":"2022-04-27T07:37:30.593987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SHAPE = (224, 224, 3)\ninception_model = InceptionV3(input_shape=IMG_SHAPE, include_top=False, weights=\"imagenet\")\ninc_model = create_model(inception_model, gal_name=\"Inception_gal\", pl_name='Inception_pl')\ninc_history = fit_model(inc_model)\n#inc_model.save('inc_models/inc_model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:37:30.596233Z","iopub.execute_input":"2022-04-27T07:37:30.596488Z","iopub.status.idle":"2022-04-27T07:39:24.828505Z","shell.execute_reply.started":"2022-04-27T07:37:30.596453Z","shell.execute_reply":"2022-04-27T07:39:24.827709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for layer in inc_model.layers:\n    layer._name = layer.name + str(\"_inc\")","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:39:24.831052Z","iopub.execute_input":"2022-04-27T07:39:24.831306Z","iopub.status.idle":"2022-04-27T07:39:24.845222Z","shell.execute_reply.started":"2022-04-27T07:39:24.831271Z","shell.execute_reply":"2022-04-27T07:39:24.844439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SHAPE = (224, 224, 3)\ndensenet_model = DenseNet121(input_shape=IMG_SHAPE, include_top=False, weights=\"imagenet\")\ndense_model = create_model(densenet_model, gal_name=\"DenseNet_gal\", pl_name='DenseNet_pl')\ndense_history = fit_model(dense_model)\n#dense_model.save('dense_models/dense_model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:39:24.84656Z","iopub.execute_input":"2022-04-27T07:39:24.847034Z","iopub.status.idle":"2022-04-27T07:42:22.599292Z","shell.execute_reply.started":"2022-04-27T07:39:24.846994Z","shell.execute_reply":"2022-04-27T07:42:22.598513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for layer in dense_model.layers:\n    layer._name = layer.name + str(\"_dense\")","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:42:22.600748Z","iopub.execute_input":"2022-04-27T07:42:22.60268Z","iopub.status.idle":"2022-04-27T07:42:22.61745Z","shell.execute_reply.started":"2022-04-27T07:42:22.602639Z","shell.execute_reply":"2022-04-27T07:42:22.616777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble all models","metadata":{}},{"cell_type":"code","source":"models = [res_model, inc_model, dense_model]","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:42:22.618768Z","iopub.execute_input":"2022-04-27T07:42:22.619253Z","iopub.status.idle":"2022-04-27T07:42:22.628631Z","shell.execute_reply.started":"2022-04-27T07:42:22.619216Z","shell.execute_reply":"2022-04-27T07:42:22.627921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ensemble_model(all_models):\n    for i, model in enumerate(all_models):\n        for layer in model.layers:\n            layer.trainable = False\n    model_input = tf.keras.Input(shape=(224, 224, 3))\n    model_outputs = [model(model_input) for model in all_models]\n    ensemble_output = tf.keras.layers.Average()(model_outputs)\n    model = tf.keras.Model(inputs=model_input, outputs=ensemble_output) \n    model.compile(optimizer=Adam(lr=0.001), loss='sparse_categorical_crossentropy', metrics=[\"accuracy\"])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:42:22.629824Z","iopub.execute_input":"2022-04-27T07:42:22.630082Z","iopub.status.idle":"2022-04-27T07:42:22.638457Z","shell.execute_reply.started":"2022-04-27T07:42:22.630049Z","shell.execute_reply":"2022-04-27T07:42:22.637734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ens_model = ensemble_model(models)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:42:22.639704Z","iopub.execute_input":"2022-04-27T07:42:22.640079Z","iopub.status.idle":"2022-04-27T07:42:24.427377Z","shell.execute_reply.started":"2022-04-27T07:42:22.640041Z","shell.execute_reply":"2022-04-27T07:42:24.426573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 48\nepochs = 20\nhistory = ens_model.fit_generator(train_generator,\n                    steps_per_epoch=len(train_generator) // batch_size,\n                    epochs=epochs,\n                    validation_data=(validation_generator),\n                    validation_steps=len(validation_generator) // batch_size\n                    )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:45:05.070243Z","iopub.execute_input":"2022-04-27T07:45:05.070512Z","iopub.status.idle":"2022-04-27T07:45:05.410742Z","shell.execute_reply.started":"2022-04-27T07:45:05.070477Z","shell.execute_reply":"2022-04-27T07:45:05.410054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('ResNet50 acc:', res_history.history['val_accuracy'][-1])\nprint('InceptionV3 acc:', inc_history.history['val_accuracy'][-1])\nprint('DenseNet121 acc:', dense_history.history['val_accuracy'][-1])\nprint('Ensemble acc:', history.history['val_accuracy'][-1])","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:45:05.752096Z","iopub.execute_input":"2022-04-27T07:45:05.75246Z","iopub.status.idle":"2022-04-27T07:45:05.759039Z","shell.execute_reply.started":"2022-04-27T07:45:05.752425Z","shell.execute_reply":"2022-04-27T07:45:05.758329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model performance","metadata":{}},{"cell_type":"code","source":"ens_model.evaluate_generator(train_generator)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:45:13.018976Z","iopub.execute_input":"2022-04-27T07:45:13.019231Z","iopub.status.idle":"2022-04-27T07:47:49.686159Z","shell.execute_reply.started":"2022-04-27T07:45:13.019203Z","shell.execute_reply":"2022-04-27T07:47:49.685456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ens_model.evaluate_generator(validation_generator)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:47:49.687807Z","iopub.execute_input":"2022-04-27T07:47:49.68806Z","iopub.status.idle":"2022-04-27T07:48:06.816314Z","shell.execute_reply.started":"2022-04-27T07:47:49.688018Z","shell.execute_reply":"2022-04-27T07:48:06.815578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ens_model.evaluate_generator(test_generator)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T07:48:06.817723Z","iopub.execute_input":"2022-04-27T07:48:06.818547Z","iopub.status.idle":"2022-04-27T07:48:24.315923Z","shell.execute_reply.started":"2022-04-27T07:48:06.818489Z","shell.execute_reply":"2022-04-27T07:48:24.315227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_pred = ens_model.predict_generator(train_generator)\ny_train_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"y_train_pred_df = pd.DataFrame({'Prob': y_train_pred})\ny_train_pred_df['Actual'] = y_train.values\ny_train_pred_df['Predicted'] = y_train_pred_df.Prob.map(lambda x: 1 if x > 0.5 else 0)\ny_train_pred_df = y_train_pred_df.set_index(y_train.index)\ny_train_pred_df.head()","metadata":{}},{"cell_type":"markdown","source":"# Confusion matrix\nconfusion = metrics.confusion_matrix(y_train_pred_df.Actual, y_train_pred_df.Predicted)\nprint(confusion)","metadata":{}},{"cell_type":"markdown","source":"TP = confusion[1,1]\nTN = confusion[0,0]\nFP = confusion[0,1]\nFN = confusion[1,0]","metadata":{}},{"cell_type":"markdown","source":"# Accuracy\nacc = metrics.accuracy_score(y_train_pred_df.Actual, y_train_pred_df.Predicted)\nprint(acc)","metadata":{}},{"cell_type":"markdown","source":"# Sensitivity\nsen = TP / float(TP+FN)\nprint(sen)","metadata":{}},{"cell_type":"markdown","source":"# Specificity\nspec = TN / float(TN+FP)\nprint(spec)","metadata":{}},{"cell_type":"markdown","source":"# F1 score\nf1score = metrics.f1_score(y_train_pred_df.Actual, y_train_pred_df.Predicted)\nprint(f1score)","metadata":{}},{"cell_type":"markdown","source":"# AUC\ntrain_pred_prob = model.predict_proba(X_train)\naucroc = metrics.roc_auc_score(y_train_pred_df.Actual, train_pred_prob[:, 1], average='weighted')\nprint(aucroc)","metadata":{}},{"cell_type":"markdown","source":"# Function to draw AUC-ROC\ndef draw_roc(actual, probs):\n    fpr, tpr, thresholds = metrics.roc_curve(actual, probs, drop_intermediate = False )\n    auc_score = metrics.roc_auc_score(actual, probs)\n    plt.figure(figsize = (5, 5))\n    plt.plot(fpr, tpr, label = 'ROC curve (area = %0.2f)' % auc_score )\n    plt.plot([0, 1], [0, 1], 'k--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver Operating Characteristic')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n\n    return None","metadata":{}},{"cell_type":"markdown","source":"# Metrics for AUC-ROC\nfpr, tpr, thresholds = metrics.roc_curve(y_train_pred_df.Actual, y_train_pred_df.Prob, drop_intermediate = False)","metadata":{}},{"cell_type":"markdown","source":"# Plot ROC curve\ndraw_roc(y_train_pred_df.Actual, y_train_pred_df.Prob)","metadata":{}},{"cell_type":"markdown","source":"# Find optimal cutoff\np, r, thresholds = precision_recall_curve(y_train_pred_df.Actual, y_train_pred_df.Prob)\nplt.plot(thresholds, p[:-1], \"g-\")\nplt.plot(thresholds, r[:-1], \"r-\")\nplt.show()","metadata":{}},{"cell_type":"markdown","source":"## Evaluate model performance on test set","metadata":{}},{"cell_type":"markdown","source":"ens_model.evaluate_generator(test_generator)","metadata":{"execution":{"iopub.status.busy":"2022-04-13T05:59:39.448961Z","iopub.execute_input":"2022-04-13T05:59:39.449561Z","iopub.status.idle":"2022-04-13T06:00:03.58749Z","shell.execute_reply.started":"2022-04-13T05:59:39.449524Z","shell.execute_reply":"2022-04-13T06:00:03.58678Z"}}},{"cell_type":"markdown","source":"y_test_pred = ens_model.predict_generator(test_generator)\ny_test_pred","metadata":{"execution":{"iopub.status.busy":"2022-04-13T06:02:19.33511Z","iopub.execute_input":"2022-04-13T06:02:19.335393Z","iopub.status.idle":"2022-04-13T06:02:35.696511Z","shell.execute_reply.started":"2022-04-13T06:02:19.33536Z","shell.execute_reply":"2022-04-13T06:02:35.695691Z"}}},{"cell_type":"markdown","source":"print(metrics.confusion_matrix(validation_generator.classes, y_test_pred))","metadata":{"execution":{"iopub.status.busy":"2022-04-13T06:19:54.839208Z","iopub.execute_input":"2022-04-13T06:19:54.839996Z","iopub.status.idle":"2022-04-13T06:19:54.862024Z","shell.execute_reply.started":"2022-04-13T06:19:54.839957Z","shell.execute_reply":"2022-04-13T06:19:54.86051Z"}}},{"cell_type":"markdown","source":"#aucroc = metrics.roc_auc_score(test_generator, y_test_pred)\nfpr, tpr, thresholds = metrics.roc_curve(y_test_pred, test_generator.classes)","metadata":{"execution":{"iopub.status.busy":"2022-04-13T06:17:02.425672Z","iopub.execute_input":"2022-04-13T06:17:02.426389Z","iopub.status.idle":"2022-04-13T06:17:02.449765Z","shell.execute_reply.started":"2022-04-13T06:17:02.426349Z","shell.execute_reply":"2022-04-13T06:17:02.448854Z"}}},{"cell_type":"markdown","source":"y_test_pred_df = pd.DataFrame({'Prob': y_test_pred})\ny_test_pred_df['Actual'] = y_test.values\ny_test_pred_df['Predicted'] = y_test_pred_df.Prob.map(lambda x: 1 if x > 0.5 else 0)\ny_test_pred_df = y_test_pred_df.set_index(y_test.index)\ny_test_pred_df.head()","metadata":{}},{"cell_type":"markdown","source":"y_test_pred_df['Predicted'] = np.argmax(y_test_pred,axis=1)\ny_test_pred_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T13:23:47.326645Z","iopub.execute_input":"2021-12-21T13:23:47.327127Z","iopub.status.idle":"2021-12-21T13:23:47.346471Z","shell.execute_reply.started":"2021-12-21T13:23:47.327079Z","shell.execute_reply":"2021-12-21T13:23:47.345839Z"}}},{"cell_type":"markdown","source":"# Confusion matrix\nconfusion = metrics.confusion_matrix(y_test_pred_df.Actual, y_test_pred_df.Predicted)\nprint(confusion)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T13:24:07.809878Z","iopub.execute_input":"2021-12-21T13:24:07.810146Z","iopub.status.idle":"2021-12-21T13:24:07.82159Z","shell.execute_reply.started":"2021-12-21T13:24:07.810115Z","shell.execute_reply":"2021-12-21T13:24:07.820221Z"}}},{"cell_type":"markdown","source":"TP = confusion[1,1]\nTN = confusion[0,0]\nFP = confusion[0,1]\nFN = confusion[1,0]","metadata":{"execution":{"iopub.status.busy":"2021-12-21T13:24:12.294273Z","iopub.execute_input":"2021-12-21T13:24:12.294608Z","iopub.status.idle":"2021-12-21T13:24:12.300567Z","shell.execute_reply.started":"2021-12-21T13:24:12.294576Z","shell.execute_reply":"2021-12-21T13:24:12.297862Z"}}},{"cell_type":"markdown","source":"# Accuracy\nacc = metrics.accuracy_score(y_test, vd_preds)\nprint(acc)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T13:24:22.670816Z","iopub.execute_input":"2021-12-21T13:24:22.671084Z","iopub.status.idle":"2021-12-21T13:24:22.678798Z","shell.execute_reply.started":"2021-12-21T13:24:22.671054Z","shell.execute_reply":"2021-12-21T13:24:22.678045Z"}}},{"cell_type":"markdown","source":"# Sensitivity\nsen = TP / float(TP+FN)\nprint(sen)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T13:24:24.755474Z","iopub.execute_input":"2021-12-21T13:24:24.756333Z","iopub.status.idle":"2021-12-21T13:24:24.761551Z","shell.execute_reply.started":"2021-12-21T13:24:24.756298Z","shell.execute_reply":"2021-12-21T13:24:24.760804Z"}}},{"cell_type":"markdown","source":"# Specificity\nspec = TN / float(TN+FP)\nprint(spec)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T13:24:26.086086Z","iopub.execute_input":"2021-12-21T13:24:26.08667Z","iopub.status.idle":"2021-12-21T13:24:26.091582Z","shell.execute_reply.started":"2021-12-21T13:24:26.086629Z","shell.execute_reply":"2021-12-21T13:24:26.090845Z"}}},{"cell_type":"markdown","source":"# F1 score\nf1score = metrics.f1_score(y_test_pred_df.Actual, y_test_pred_df.Predicted, average = 'weighted')\nprint(f1score)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T13:25:08.76786Z","iopub.execute_input":"2021-12-21T13:25:08.768143Z","iopub.status.idle":"2021-12-21T13:25:08.77775Z","shell.execute_reply.started":"2021-12-21T13:25:08.768113Z","shell.execute_reply":"2021-12-21T13:25:08.776837Z"}}},{"cell_type":"markdown","source":"# AUC\n\naucroc = metrics.roc_auc_score(y_test_pred_df.Actual, y_test_pred, average='weighted')\nprint(aucroc)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T13:26:54.943774Z","iopub.execute_input":"2021-12-21T13:26:54.944062Z","iopub.status.idle":"2021-12-21T13:26:54.971107Z","shell.execute_reply.started":"2021-12-21T13:26:54.94403Z","shell.execute_reply":"2021-12-21T13:26:54.970129Z"}}},{"cell_type":"markdown","source":"# Function to draw AUC-ROC\ndef draw_roc(actual, probs):\n    fpr, tpr, thresholds = metrics.roc_curve(actual, probs, drop_intermediate = False )\n    auc_score = metrics.roc_auc_score(actual, probs)\n    plt.figure(figsize = (5, 5))\n    plt.plot(fpr, tpr, label = 'ROC curve (area = %0.2f)' % auc_score )\n    plt.plot([0, 1], [0, 1], 'k--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver Operating Characteristic')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n\n    return None","metadata":{}},{"cell_type":"markdown","source":"# Metrics for AUC-ROC\nfpr, tpr, thresholds = metrics.roc_curve(y_test_pred_df.Actual, y_test_pred_df.Prob, drop_intermediate = False)","metadata":{}},{"cell_type":"markdown","source":"# Plot ROC curve\ndraw_roc(y_test_pred_df.Actual, y_test_pred_df.Prob)","metadata":{}},{"cell_type":"markdown","source":"# Find optimal cutoff\np, r, thresholds = precision_recall_curve(y_test_pred_df.Actual, y_test_pred_df.Prob)\nplt.plot(thresholds, p[:-1], \"g-\")\nplt.plot(thresholds, r[:-1], \"r-\")\nplt.show()","metadata":{}}]}