{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Why bother?\nAs one of the main purpose of any competition is making a better score so why shold we even bother about this in the first Place? I think model building, training and evaluating is like attending an exam. For exam, we prepare , we attend exam and then we get results. After result we get to see our mistakes so that we can correct them. So, after training and evaluating our model shouldn't we need to find our model's mistakes??? Otherwise how we can correct them?","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# What we expecting from this notebook:\n* Grad-CAM visualization\n* Saliency Map\n* Some useful tool that might come handy","execution_count":null},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\nimport os\nfrom glob import glob\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set(style = 'white')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\ntest_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'\ntest_dir = '/kaggle/input/siim-isic-melanoma-classification/jpeg/test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['image_path'] = train_dir + train_df.image_name +'.jpg'\ntest_df['image_path'] = test_dir + test_df.image_name +'.jpg'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['y'] = train_df.target.astype(str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.y.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_0 = train_df[train_df.target ==0].iloc[:584,:]\ntrain_df_1 = train_df[train_df.target==1]\ntrain_df_short = pd.concat((train_df_0 , train_df_1),axis = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_short.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X, y = train_df_short[train_df_short.columns[:-1]], train_df_short.y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val , y_train, y_val = train_test_split(X, y, random_state = 10, test_size = .2, stratify = y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_size = (256, 256)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Converting images to array for better speed","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nimport gc\n\ntrain_images = np.zeros(shape = (X_train.shape[0], *image_size, 3))\ntrain_labels = np.zeros(X_train.shape[0])\n\nfor idx in tqdm(range(X_train.shape[0])):\n    img = cv2.imread(X_train.image_path.iloc[idx])\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, dsize = image_size)\n    \n    \n    train_images[idx,:,:] = img\n    train_labels[idx] = int(y_train.iloc[idx])\n    \n    del img\n    gc.collect()\n    \n# train_images = np.array(train_images)\n# train_labels = np.array(train_labels)\ntrain_images.shape, train_labels.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Validation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nval_images = np.zeros(shape = (X_val.shape[0], *image_size, 3))\nval_labels = np.zeros(X_val.shape[0])\n\nfor idx in tqdm(range(X_val.shape[0])):\n    \n    img = cv2.imread(X_val.image_path.iloc[idx])\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, dsize = image_size)\n    \n    \n    val_images[idx,:,:] = img\n    val_labels[idx] = int(y_val.iloc[idx])\n    \n    del img\n    gc.collect()\n\nval_images.shape, val_labels.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(val_images[2].astype('uint8'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_short = pd.concat((X_train, y_train), axis = 1)\nval_df = pd.concat((X_val, y_val), axis = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ImageDataGenerator","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.applications.vgg16 import VGG16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_gen = ImageDataGenerator(\n                                rotation_range=30,\n                                width_shift_range=0.2,\n                                height_shift_range=0.2,\n#                                 brightness_range=[0.7, 1.3],\n                                shear_range=0.0,\n                                zoom_range=0.2,\n                                fill_mode=\"nearest\",\n                                horizontal_flip=True,\n                                vertical_flip=True,\n                                rescale=1/255.0)\n\ntest_gen = ImageDataGenerator(rescale=1/255.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator = train_gen.flow(\n    x = train_images,\n    y= train_labels,\n    batch_size=32,\n    shuffle=True,\n    seed=1001\n)\n\nval_generator = test_gen.flow(\n    x = val_images,\n    y= val_labels,\n    batch_size=32,\n    shuffle=False,\n    seed=1001\n)\n\n# Loading all test 10K images as array will consume all the memory so it's wise to use flow_from_dataframe()\ntest_generator = test_gen.flow_from_dataframe(\n                                                test_df,\n                                                x_col=\"image_path\",\n                                                target_size=image_size,\n                                                class_mode=None,\n                                                batch_size=32,\n                                                shuffle=False,\n                                                seed=10\n                                            )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Why just AUC when can get so much more?\nHere we'll be using a callback which will give almost every classification report possible with AUC and Confusion Matrix Plot","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Plot Confusion Matrix","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import itertools\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix\nimport pandas.util.testing as tm\nfrom sklearn import metrics\nimport seaborn as sns\nsns.set()\n\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=90)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.grid(b=False)\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Function to get all the score","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate_model(model, test_generator, y_test, class_labels, cm_normalize=True, \\\n                 print_cm=True):\n    \n    from datetime import datetime\n\n    results = dict()\n\n    print('Predicting test data')\n    test_start_time = datetime.now()\n    y_pred_original = model.predict(test_generator,verbose=1)\n    # y_pred = (y_pred_original>0.5).astype('int')\n\n    y_pred = (y_pred_original>0.5).astype('int')\n    #y_test = np.argmax(testy, axis=-1)\n    \n    test_end_time = datetime.now()\n    print('Done \\n \\n')\n    results['testing_time'] = test_end_time - test_start_time\n    print('testing time(HH:MM:SS:ms) - {}\\n\\n'.format(results['testing_time']))\n    results['predicted'] = y_pred\n    \n    \n    # balanced_accuracy\n    from sklearn.metrics import balanced_accuracy_score\n    balanced_accuracy = balanced_accuracy_score(y_true=y_test, y_pred=y_pred)\n    print('---------------------')\n    print('| Balanced Accuracy  |')\n    print('---------------------')\n    print('\\n    {}\\n\\n'.format(balanced_accuracy))\n\n    # calculate overall accuracty of the model\n    accuracy = metrics.accuracy_score(y_true=y_test, y_pred=y_pred)\n    # store accuracy in results\n    results['accuracy'] = accuracy\n    print('---------------------')\n    print('|      Accuracy      |')\n    print('---------------------')\n    print('\\n    {}\\n\\n'.format(accuracy))\n    \n\n    # calculate Cohen Kappa Score of the model\n    kappa = metrics.cohen_kappa_score(y_test, y_pred)\n    print('---------------------')\n    print('|      Cohen Kappa Score      |')\n    print('---------------------')\n    print('\\n    {}\\n\\n'.format(kappa))\n\n    \n    print('---------------------')\n    print('|      ROC Plot      |')\n    print('---------------------')\n\n    fpr, tpr, thr = metrics.roc_curve(y_test, y_pred_original)\n    auc = metrics.auc(fpr, tpr)\n\n    plt.figure(figsize=(4,4))\n    lw = 2\n    plt.plot(fpr, tpr, color='darkorange',\n         lw=lw, label='ROC curve (area = %0.3f)' % auc)\n    plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver operating characteristic (ROC) Curve')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n\n    \n    # calculate auc score of the model\n    auc = metrics.auc(fpr, tpr)\n    # store accuracy in results\n    print('---------------------')\n    print('|      ROC AUC Score      |')\n    print('---------------------')\n    print('\\n    {}\\n\\n'.format(auc))\n    \n\n    # confusion matrix\n    cm = metrics.confusion_matrix(y_test, y_pred)\n    results['confusion_matrix'] = cm\n    if print_cm: \n        print('--------------------')\n        print('| Confusion Matrix |')\n        print('--------------------')\n        print('\\n {}'.format(cm))\n        \n    # plot confusin matrix\n    plt.figure(figsize=(6,4))\n    plt.grid(b=False)\n    plot_confusion_matrix(cm, classes=class_labels, normalize=True, title='Normalized confusion matrix')\n    plt.show()\n    \n    # get classification report\n    print('-------------------------')\n    print('| Classifiction Report |')\n    print('-------------------------')\n    classification_report = metrics.classification_report(y_test, y_pred)\n    # store report in results\n    results['classification_report'] = classification_report\n    print(classification_report)\n    \n    # add the trained  model to the results\n    results['model'] = model\n    \n    return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import Callback\nclass Scores(Callback):\n  \n  def __init__(self, test_generator, y_test, class_labels):\n    super(Scores, self).__init__()\n    self.test_generator = test_generator\n    self.y_test = y_test\n    self.class_labels = class_labels\n        \n  def on_epoch_end(self, epoch, logs=None):\n\n    evaluate_model(self.model, self.test_generator, self.y_test, self.class_labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EpochPlot Callback\n`This callback is for those who are really impatient like me to wait for the whole time until the model training is done...`\n\nThis callback will create EpochPlot after certain epoch. In that way we'll have better understanding if our model is overfitting or not..","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nsns.set()\n\nfrom keras.callbacks import Callback\n\nclass EpochPlot(Callback):\n    \n    def __init__(self, freq): # Frequency of EpochPlot\n        super(EpochPlot, self).__init__()\n        self.freq = freq\n\n    def on_train_begin(self, logs={}):\n        self.i = 0\n        self.x = []\n        self.losses = []\n        self.val_losses = []\n        self.acc = []\n        self.val_acc = []\n        self.fig = plt.figure()\n        \n        self.logs = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        \n        self.logs.append(logs)\n        self.x.append(self.i)\n        self.losses.append(logs.get('loss'))\n        self.val_losses.append(logs.get('val_loss'))\n        self.acc.append(logs.get('acc'))\n        self.val_acc.append(logs.get('val_acc'))\n        self.i += 1\n\n        if (self.i % self.freq)==0:\n\n          f, (ax1, ax2) = plt.subplots(1, 2, figsize =  (24,6), sharex=True, constrained_layout = True)\n          \n          # ax1.set_yscale('log')\n          ax2.plot(self.x, self.losses, label=\"loss\", marker = 'o')\n          ax2.plot(self.x, self.val_losses, label=\"val_loss\", marker = 'o')\n          ax2.legend()\n          \n          ax1.plot(self.x, self.acc, label=\"acc\", marker = 'o')\n          ax1.plot(self.x, self.val_acc, label=\"val_acc\", marker = 'o')\n          ax1.legend()\n          \n          plt.show();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# As our main focus is on what our model is learning, I won't be bulding any fancy model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import *\nfrom keras.models import *\nfrom keras.utils import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications.vgg16 import VGG16\nbase_model = VGG16(input_shape = (*image_size, 3), weights = 'imagenet', include_top = False)\nbase_model.summary()\n# for layer in base_model.layers[:-4]:\n#     layer.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.optimizers import Adam\ninputs = base_model.input\noutputs = base_model.output\n\noutputs = GlobalAveragePooling2D()(outputs)\noutputs = Dense(256, activation = 'relu')(outputs)\noutputs = Dropout(0.5)(outputs)\noutputs = Dense(128, activation = 'relu')(outputs)\noutputs = Dropout(0.3)(outputs)\noutputs = Dense(64, activation = 'relu')(outputs)\noutputs = Dropout(0.1)(outputs)\noutputs = Dense(1, activation = 'sigmoid')(outputs)\n\nmodel = Model(inputs = inputs, outputs = outputs)\nmodel.compile(optimizer = Adam(lr = 1e-5), loss = 'binary_crossentropy', metrics = ['acc'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint\n\nfilepath = 'best_model.h5'\ncallback1 = ModelCheckpoint(\n    filepath, monitor='val_acc', verbose=1, save_best_only=True,\n    save_weights_only=False, mode='max'\n)\n\ncallback2 = Scores(val_generator, y_val.values.astype(int), ['benign', 'malignant'])\ncallback3 = EpochPlot(5)\ncallbacks = [callback1, callback2, callback3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_generator, \n       validation_data = val_generator,\n       steps_per_epoch = train_generator.n//train_generator.batch_size,\n       epochs = 50,\n#        class_weight = class_weight,\n       callbacks = callbacks,\n       verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading our best Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = load_model('/kaggle/working/best_model.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Before visualization let's get our model prediction so that we can plot class wise TP, FP , TN or FN images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def TP(row):\n    if (row.y_pred ==1)& (row.y_true==1):\n        return True\n    else:\n        return False\ndef TN(row):\n    if (row.y_pred ==0)& (row.y_true==0):\n        return True\n    else:\n        return False\ndef FP(row):\n    if (row.y_pred ==1)& (row.y_true==0):\n        return True\n    else:\n        return False\ndef FN(row):\n    if (row.y_pred ==0)& (row.y_true==1):\n        return True\n    else:\n        return False\n    \n\npreds = model.predict(val_generator, verbose = 1)    \ny_df = pd.DataFrame({'image_path':X_val.image_path.values,\n                     'y_true':y_val.values.astype(int), \n                     'y_pred': preds.squeeze().round().astype(int)})\n\ny_df['TP'] = y_df.apply(TP, axis = 1).values\ny_df['TN'] = y_df.apply(TN, axis = 1).values\ny_df['FP'] = y_df.apply(FP, axis = 1).values\ny_df['FN'] = y_df.apply(FN, axis = 1).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Let's just clear some memory before Plot","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_images\ndel val_images\ndel train_labels\ndel val_labels\n\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Grad-CAM\n![Grad-CAM](https://media.arxiv-vanity.com/render-output/2950516/figures/gcam_ablation_gap_gmp.png)\n![Grad-CAM](http://gradcam.cloudcv.org/static/images/network.png)\nCheck this paper [here](https://arxiv.org/abs/1610.02391)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Saliency Map\n![](https://miro.medium.com/max/1400/1*vmOVSK7KplO77BSWhzzJkA.gif)\n**There is a already a package for this kind of visualization: [keras-vis](https://github.com/raghakot/keras-vis)**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# This Function generates both Grad-CAM and Saliency Map","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize(img_path , model , last_conv_layer_name,  image_size = (256, 256), alpha = 0.4):\n\n  import os\n  import numpy as np\n  import pandas as pd\n  import cv2\n  #   from google.colab.patches import cv2_imshow\n  from PIL import Image\n  from matplotlib import pyplot as plt\n\n  from keras.applications.inception_resnet_v2 import preprocess_input, decode_predictions\n  from keras.preprocessing.image import load_img, img_to_array\n  from keras.models import Model, load_model\n  from keras import backend as K\n\n\n\n  # ==================================\n  #   1. Test images prediction\n  # ==================================\n  img = load_img(img_path, target_size= (int(model.input.shape[1]), int(model.input.shape[2])))\n  # msk = load_img(mask_path, target_size=image_size, color_mode= 'grayscale')\n  img = img_to_array(img)\n  img = img/255.0\n  pred_img = np.expand_dims(img, axis=0)\n  # pred_img = preprocess_input(img)\n  pred = model.predict(pred_img)\n\n  # ==============================\n  #   2. Heatmap visualization \n  # ==============================\n  # Item of prediction vector\n  pred_output = model.output[:, np.argmax(pred)]\n\n  # Feature map of 'conv_7b_ac' layer, which is the last convolution layer\n  last_conv_layer = model.get_layer(last_conv_layer_name)\n\n  # Gradient of class for feature map output of 'conv_7b_ac'\n  grads = K.gradients(pred_output, last_conv_layer.output)[0]\n\n  # Feature map vector with gradient average value per channel\n  pooled_grads = K.mean(grads, axis=(0, 1, 2))\n\n  # Given a test image, get the feature map output of the previously defined 'pooled_grads' and 'conv_7b_ac'\n  iterate = K.function([model.input], [pooled_grads, last_conv_layer.output[0]])\n\n  # Put a test image and get two numpy arrays\n  pooled_grads_value, conv_layer_output_value = iterate([pred_img])\n\n  # print(pooled_grads.shape[0])\n  # Multiply the importance of a channel for a class by the channels in a feature map array\n  for i in range(int(pooled_grads.shape[0])):\n      conv_layer_output_value[:, :, i] *= pooled_grads_value[i]\n\n  # The averaged value along the channel axis in the created feature map is the heatmap of the class activation\n  heatmap = np.mean(conv_layer_output_value, axis=-1)\n\n  # Normalize the heatmap between 0 and 1 for visualization\n  heatmap = np.maximum(heatmap, 0)\n  heatmap /= np.max(heatmap)\n  heatmap_img = heatmap\n\n\n  # =======================\n  #   3. Apply Grad-CAM\n  # =======================\n  ori_img = load_img(img_path, target_size=image_size)\n\n  heatmap = cv2.resize(heatmap, image_size)\n  heatmap = np.uint8(255 * heatmap)\n  heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_INFERNO)\n  heatmap = cv2.cvtColor(heatmap, cv2.COLOR_RGB2BGR)\n\n  superimposed_img = heatmap * alpha + ori_img\n  cv2.imwrite('grad_cam_result.png', superimposed_img) \n  grad_img = cv2.imread('grad_cam_result.png')\n#   grad_img = cv2.cvtColor(grad_img, cv2.COLOR_BGR2RGB)\n\n  # =======================\n  #   4. Saliency Map\n  # =======================\n\n  import tensorflow as tf\n  pred_output = model.output[:, np.argmax(pred)]\n  sal = K.gradients(tf.reduce_sum(pred_output), model.input)[0]\n\n  # Keras function returning the saliency map given an image input\n  sal_fn = K.function([model.input], [sal])\n    \n  # Generating the saliency map and normalizing it\n  img_sal = sal_fn([np.resize(pred_img, (1, int(model.input.shape[1]), int(model.input.shape[2]), 3))])[0]\n  img_sal = np.abs(img_sal)\n  img_sal /= img_sal.max()\n  img_sal = cv2.resize(img_sal[0,:,:,0], dsize = image_size)\n\n\n  return ori_img, pred, heatmap_img, grad_img, img_sal","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Let's see how our model performed..","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## True Positive(TP)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_img = 10\nlast_conv_layer_name = 'block5_conv3'\ndf = y_df[(y_df.TP==True) & (y_df.y_true == 1)]\n\n\nif df.shape[0]<n_img:\n    print('Number of Image exceeds')\n\nelse:\n    fig, ax = plt.subplots(n_img, 3, figsize = (12,n_img*4), constrained_layout = True)\n\n    for idx in tqdm(range(n_img)):\n\n        img, pred, map_img, grad_img, sal = visualize(df.image_path.iloc[idx],\n                                                  model ,\n                                                  last_conv_layer_name,\n                                                  image_size = (256, 256),\n                                                  alpha = 0.6)\n\n        ax[idx][0].imshow(img)\n        ax[idx][0].grid(b = False)\n        ax[idx][0].set_xticks([])\n        ax[idx][0].set_yticks([])\n        ax[idx][0].set_title('Original Image:', fontsize = 15)\n\n        ax[idx][1].imshow(grad_img)\n        ax[idx][1].grid(b = False)\n        ax[idx][1].set_xticks([])\n        ax[idx][1].set_yticks([])\n        ax[idx][1].set_title(f'Grad-CAM: {df.y_true.iloc[idx]} | {pred[0][0]:.2f}', fontsize = 15)\n\n        ax[idx][2].imshow(sal)\n        ax[idx][2].grid(b = False)\n        ax[idx][2].set_xticks([])\n        ax[idx][2].set_yticks([])\n        ax[idx][2].set_title(f'Saliency-MAP: {df.y_true.iloc[idx]} | {pred[0][0]:.2f}', fontsize = 15)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## True Negative(TN)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_img = 10\nlast_conv_layer_name = 'block5_conv3'\ndf = y_df[y_df.TN==True]\n\nif df.shape[0]<n_img:\n    print('Number of Image exceeds !!!')\n\nelse:\n    fig, ax = plt.subplots(n_img, 3, figsize = (12,n_img*4), constrained_layout = True)\n\n    for idx in tqdm(range(n_img)):\n\n        img, pred, map_img, grad_img, sal = visualize(df.image_path.iloc[idx],\n                                                  model ,\n                                                  last_conv_layer_name,\n                                                  image_size = (256, 256),\n                                                  alpha = 0.6)\n\n        ax[idx][0].imshow(img)\n        ax[idx][0].grid(b = False)\n        ax[idx][0].set_xticks([])\n        ax[idx][0].set_yticks([])\n        ax[idx][0].set_title('Original Image:', fontsize = 15)\n\n        ax[idx][1].imshow(grad_img)\n        ax[idx][1].grid(b = False)\n        ax[idx][1].set_xticks([])\n        ax[idx][1].set_yticks([])\n        ax[idx][1].set_title(f'Grad-CAM: {df.y_true.iloc[idx]} | {pred[0][0]:.2f}', fontsize = 15)\n\n        ax[idx][2].imshow(sal)\n        ax[idx][2].grid(b = False)\n        ax[idx][2].set_xticks([])\n        ax[idx][2].set_yticks([])\n        ax[idx][2].set_title(f'Saliency-MAP: {df.y_true.iloc[idx]} | {pred[0][0]:.2f}', fontsize = 15)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## False Positive(FP)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_img = 10\nlast_conv_layer_name = 'block5_conv3'\ndf = y_df[y_df.FP==True]\n\nif df.shape[0]<n_img:\n    print('Number of Image exceeds !!!')\n\nelse:\n    fig, ax = plt.subplots(n_img, 3, figsize = (12,n_img*4), constrained_layout = True)\n\n    for idx in tqdm(range(n_img)):\n\n        img, pred, map_img, grad_img, sal = visualize(df.image_path.iloc[idx],\n                                                  model ,\n                                                  last_conv_layer_name,\n                                                  image_size = (256, 256),\n                                                  alpha = 0.6)\n\n        ax[idx][0].imshow(img)\n        ax[idx][0].grid(b = False)\n        ax[idx][0].set_xticks([])\n        ax[idx][0].set_yticks([])\n        ax[idx][0].set_title('Original Image:', fontsize = 15)\n\n        ax[idx][1].imshow(grad_img)\n        ax[idx][1].grid(b = False)\n        ax[idx][1].set_xticks([])\n        ax[idx][1].set_yticks([])\n        ax[idx][1].set_title(f'Grad-CAM: {df.y_true.iloc[idx]} | {pred[0][0]:.2f}', fontsize = 15)\n\n        ax[idx][2].imshow(sal)\n        ax[idx][2].grid(b = False)\n        ax[idx][2].set_xticks([])\n        ax[idx][2].set_yticks([])\n        ax[idx][2].set_title(f'Saliency-MAP: {df.y_true.iloc[idx]} | {pred[0][0]:.2f}', fontsize = 15)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## False Negative(FN)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_img = 10\nlast_conv_layer_name = 'block5_conv3'\ndf = y_df[y_df.FN==True]\n\nif df.shape[0]<n_img:\n    print('Number of Image exceeds !!!')\n\nelse:\n    fig, ax = plt.subplots(n_img, 3, figsize = (12,n_img*4), constrained_layout = True)\n\n    for idx in tqdm(range(n_img)):\n\n        img, pred, map_img, grad_img, sal = visualize(df.image_path.iloc[idx],\n                                                  model ,\n                                                  last_conv_layer_name,\n                                                  image_size = (256, 256),\n                                                  alpha = 0.6)\n\n        ax[idx][0].imshow(img)\n        ax[idx][0].grid(b = False)\n        ax[idx][0].set_xticks([])\n        ax[idx][0].set_yticks([])\n        ax[idx][0].set_title('Original Image:', fontsize = 15)\n\n        ax[idx][1].imshow(grad_img)\n        ax[idx][1].grid(b = False)\n        ax[idx][1].set_xticks([])\n        ax[idx][1].set_yticks([])\n        ax[idx][1].set_title(f'Grad-CAM: {df.y_true.iloc[idx]} | {pred[0][0]:.2f}', fontsize = 15)\n\n        ax[idx][2].imshow(sal)\n        ax[idx][2].grid(b = False)\n        ax[idx][2].set_xticks([])\n        ax[idx][2].set_yticks([])\n        ax[idx][2].set_title(f'Saliency-MAP: {df.y_true.iloc[idx]} | {pred[0][0]:.2f}', fontsize = 15)\n        \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Thanks for reading this kernel\nPlease let me know if anything else could have been added...","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}