{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"\nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom tqdm import tqdm\nimport cv2 as cv\nimport random\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow.keras \nfrom tensorflow.keras.models import Sequential, load_model, Model\nfrom tensorflow.keras import layers \nfrom tensorflow.keras.optimizers import Adam, SGD, RMSprop\nfrom tensorflow.keras.metrics import Precision, Recall, AUC\nfrom tensorflow.keras.callbacks import ModelCheckpoint, Callback, EarlyStopping\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import the data\ntrain_df = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntest_df = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json \n\ndiseases_file = open('../input/cassava-leaf-disease-classification/label_num_to_disease_map.json')\ndiseases = json.load(diseases_file)\ndiseases","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Quick Data Exploration"},{"metadata":{"trusted":true},"cell_type":"code","source":"num_instances = train_df.groupby('label').size()\n\nplt.figure(figsize = (10,5))\nplt.bar(np.unique(train_df.label),num_instances)\nplt.title('Number of labels within the training set', fontweight = 'bold')\nplt.xlabel('labels')\nplt.ylabel('instances')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"raw","source":"We can see that label 3 overweighs the remaining labels in terms of instances/counts. Therefore, we are dealing with a class imabalance dataset, so we expect a vanilla neural network pipeline will NOT be sufficient. "},{"metadata":{},"cell_type":"markdown","source":"## Prepare The Data:"},{"metadata":{},"cell_type":"markdown","source":"Convert and resize the jpg images into npy files with size 224x224. Important to note, 224x224 is not assumed to be the optimal size; however, this size is a decent place to start because we are implementing imagenet pretrained models like Vgg16 and Resnet50 as cheap baseline models."},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.makedirs('/kaggle/working/train_images', exist_ok=True)\n\ntrain_path = '../input/cassava-leaf-disease-classification/train_images/'\nfor index, filename in tqdm(enumerate(train_df.image_id)):\n    image = cv.imread(train_path + filename).astype('float32')\n    image = cv.resize(image, (224,224), interpolation = cv.INTER_CUBIC)\n    np.save('/kaggle/working/train_images/' + filename.replace('.jpg', ''), image)\n    \nos.makedirs('/kaggle/working/test_images', exist_ok=True)\n\ntest_path = '../input/cassava-leaf-disease-classification/test_images/'\nfor index, filename in tqdm(enumerate(test_df.image_id)):\n    image = cv.imread(test_path + filename).astype('float32')\n    image = cv.resize(image, (224,224), interpolation = cv.INTER_CUBIC)\n    np.save('/kaggle/working/test_images/' + filename.replace('.jpg', ''), image)\n\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils import to_categorical\n\ndef onehot_encoded(data):\n    return to_categorical(data)\n\n\ntrain_df['image_path'] = '../input/cassava-leaf-disease-classification/train_images/' + train_df.image_id\ntrain_df.head()\nplt.imshow(cv.imread(train_df.image_path[0]))\nplt.show()\n\nencoded_df = pd.DataFrame(onehot_encoded(train_df.label))\nencoded_df.columns = ['label_0', 'label_1', 'label_2', 'label_3', 'label_4']\ntrain_df = pd.concat([train_df, encoded_df], axis = 1)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"hat tip [Shervine Amidi](https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly) for the general data generator implementation in Keras. "},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\n\nclass DataGenerator(keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, list_IDs, labels, batch_size=32, dim=(224, 224), n_channels=3,\n                 n_classes=5, shuffle=True):\n        'Initialization'\n        self.dim = dim\n        self.batch_size = batch_size\n        self.labels = labels\n        self.list_IDs = list_IDs\n        self.n_channels = n_channels\n        self.n_classes = n_classes\n        self.shuffle = shuffle\n        self.on_epoch_end()\n\n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        return int(np.floor(len(self.list_IDs) / self.batch_size))\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        # Generate indexes of the batch\n        self.indexes = np.arange(len(self.list_IDs))\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        # Find list of IDs\n        list_IDs_temp = [self.list_IDs.iloc[k] for k in indexes]\n        # Generate data\n        X, y = self.__data_generation(list_IDs_temp)\n        return X, y\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange(len(self.list_IDs))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n            \n    def standard_norm(self,image):\n        image[:,:,0] = (image[:,:,0] - np.mean(image[:,:,0]))/np.std(image[:,:,0])\n        image[:,:,1] = (image[:,:,1] - np.mean(image[:,:,1]))/np.std(image[:,:,1])\n        image[:,:,2] = (image[:,:,2] - np.mean(image[:,:,2]))/np.std(image[:,:,2])\n        return image\n    \n    def __data_generation(self, list_IDs_temp):\n        'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)\n        # Initialization\n        X = np.empty((self.batch_size, *self.dim, self.n_channels))\n        y = np.empty((self.batch_size), dtype=int)\n\n        # Generate data\n        for i, ID in enumerate(list_IDs_temp):\n            # Store sample\n            X[i,:,:,:] = np.load('/kaggle/working/train_images/' + ID.replace('.jpg', '.npy')).astype('float32')\n            X[i,] = self.standard_norm(X[i,])\n            \n            y[i] = self.labels[self.list_IDs == str(ID)]\n\n        return X, keras.utils.to_categorical(y, num_classes=self.n_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# The function for creating the data generators needed for training ...\n\ndef create_gen(train_df, train_index, test_index, params):\n    # IDs\n    tr_partition, ts_partition = train_df.image_id.iloc[train_index], train_df.image_id.iloc[test_index]\n    #labels\n    tr_labels,ts_labels = train_df.label.iloc[train_index], train_df.label.iloc[test_index]\n\n    #labels = train_df.iloc[train_index,-5:], train_df.iloc[test_index, -5:]\n\n    # Generators\n    training_generator = DataGenerator(tr_partition, tr_labels, **params)\n    validation_generator = DataGenerator(ts_partition, ts_labels, **params)\n    return training_generator, validation_generator\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Model Pipeline:"},{"metadata":{},"cell_type":"markdown","source":"Hat tip to [Chengwei](https://www.dlology.com/blog/multi-class-classification-with-focal-loss-for-imbalanced-datasets/) for sharing the quick and easy focal cross entropy function for multi-classification. Focal cross entropy is introduced as an effective and heurisitic loss to over class imbalances; however, in future models, I plan to drop focal cross entropy and balance the data set with other methods. "},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\n\ndef focal_loss(gamma=2., alpha=4.):\n\n    gamma = float(gamma)\n    alpha = float(alpha)\n\n    def focal_loss_fixed(y_true, y_pred):\n        \"\"\"Focal loss for multi-classification\n        FL(p_t)=-alpha(1-p_t)^{gamma}ln(p_t)\n        Notice: y_pred is probability after softmax\n        gradient is d(Fl)/d(p_t) not d(Fl)/d(x) as described in paper\n        d(Fl)/d(p_t) * [p_t(1-p_t)] = d(Fl)/d(x)\n        Focal Loss for Dense Object Detection\n        https://arxiv.org/abs/1708.02002\n\n        Arguments:\n            y_true {tensor} -- ground truth labels, shape of [batch_size, num_cls]\n            y_pred {tensor} -- model's output, shape of [batch_size, num_cls]\n\n        Keyword Arguments:\n            gamma {float} -- (default: {2.0})\n            alpha {float} -- (default: {4.0})\n\n        Returns:\n            [tensor] -- loss.\n        \"\"\"\n        epsilon = 1.e-9\n        y_true = tf.convert_to_tensor(y_true, tf.float32)\n        y_pred = tf.convert_to_tensor(y_pred, tf.float32)\n\n        model_out = tf.add(y_pred, epsilon)\n        ce = tf.multiply(y_true, -tf.math.log(model_out))\n        weight = tf.multiply(y_true, tf.pow(tf.subtract(1., model_out), gamma))\n        fl = tf.multiply(alpha, tf.multiply(weight, ce))\n        reduced_fl = tf.reduce_max(fl, axis=1)\n        return tf.reduce_mean(reduced_fl)\n    return focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nNeural network garage ... models are created so that the back bone is \na pretrained imagenet model and a final global average pool is used before\nthe final dense layer. This way, we can easy implement the class activation map\nanalysis shown at the end of this notebook\n\"\"\"\n\nfrom tensorflow.keras.applications.xception import Xception\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.applications.vgg16 import VGG16\n\nfrom tensorflow.keras.models import Sequential, Model, load_model\nfrom tensorflow.keras.layers import Input, Dense, GlobalAveragePooling2D\nimport tensorflow as tf\n\n\ndef Resnet50_model():\n    return ResNet50(include_top = False, weights = \"imagenet\", input_shape = (224, 224, 3))\n\ndef Vgg16_model():\n    return VGG16(include_top = False, weights = \"imagenet\", input_shape = (224, 224, 3))\n\ndef Xception_model():\n    return Xception(include_top = False, weights = \"imagenet\", input_shape = (224,224,3))\n\ndef classification_model(bottom_model, lr, num_class = 5, target_size = (224,224)):\n    model = Sequential()\n #   model.add(Input(shape = (600, 800, 3)))\n  #  model.add(tf.keras.layers.Lambda(lambda image: tf.image.resize(image, target_size)))\n    model.add(bottom_model)\n    model.add(GlobalAveragePooling2D(name = \"Global_avg\"))\n    model.add(Dense(num_class, activation = 'softmax', name = 'classes'))\n    model.compile(optimizer = Adam(lr=lr), metrics = ['accuracy' ,'Precision', 'Recall', 'AUC'],\n                  loss = focal_loss())\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# all possible models ... \n\n'example code: showing how to load an individual model'\n\nbottom_model = Xception_model()\nmodel_Xception = classification_model(bottom_model, 0.0001)\nbottom_model = Resnet50_model()\nmodel_Resnet50 = classification_model(bottom_model, 0.0001)\nbottom_model = Vgg16_model()\nmodel_Vgg16 = classification_model(bottom_model, 0.0001)\n\nmodel_Vgg16.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train model"},{"metadata":{"trusted":true},"cell_type":"code","source":"'fucntion to train the model with a data generator'\ndef train_model(model,tr_gen, ts_gen, epochs):\n    train_history = model.fit_generator(\n        generator = tr_gen,\n        epochs =  epochs,\n        validation_data = ts_gen,\n        use_multiprocessing=True,\n        workers=8,\n        verbose = 1)\n    return train_history, model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom sklearn.model_selection import KFold\n\n\"\"\"\nHere, we are using cross-validation so that we can create an ensemble model to the measure the uncertainity\nin the test prediction. With GPU, roughly takes ~1hr to train for VGG16... \n\"\"\"\n\n# Parameters\nparams = {'dim': (224,224),\n          'batch_size': 128,\n          'n_classes': 5,\n          'n_channels': 3,\n          'shuffle': True}\n\ntrain_dic = dict(hist=list(), train_df=list(), valid_df=list())\n\nkf = KFold(n_splits = 5)\nKFold(n_splits = 2, random_state = 42, shuffle = True)\n\nmodel_ensemble = []\nkk=0\n\nfor train_index, test_index in kf.split(train_df):\n    \n    # creating the training and testing data generators\n    column_names = ['image_path', 'label_0', 'label_1', 'label_2', 'label_3', 'label_4']\n    height_img, width_img = 224, 224\n    \n    training_generator, validation_generator = create_gen(train_df, train_index, test_index, params)\n    \n    bot_model = Vgg16_model()\n    model_final = classification_model(bot_model, 0.0001)\n    print(f'========== Training Fold {kk+1} ==========' )\n    train_history, model = train_model(model_final, training_generator, validation_generator, 5)\n    \n    model_ensemble.append(model)\n    \n    kk+=1\n    train_dic['hist'].append(train_history)\n    train_dic['train_df'].append(train_df.iloc[train_index])\n    train_dic['valid_df'].append(train_df.iloc[test_index])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Predict on the single test image: "},{"metadata":{"trusted":true},"cell_type":"code","source":"'Compute test predictions with the trained ensemble model ...'\n\ndef standard_norm(image):\n    image[:,:,0] = (image[:,:,0] - np.mean(image[:,:,0]))/np.std(image[:,:,0])\n    image[:,:,1] = (image[:,:,1] - np.mean(image[:,:,1]))/np.std(image[:,:,1])\n    image[:,:,2] = (image[:,:,2] - np.mean(image[:,:,2]))/np.std(image[:,:,2])\n    return image\n\ntest_img = np.load('/kaggle/working/test_images/'+test_df.image_id[0].replace('.jpg','.npy'))\ntest_img = standard_norm(cv.resize(test_img, (224,224), interpolation = cv.INTER_CUBIC))\n\ny_pred = np.zeros((5,5))\n\nfor index, model in enumerate(model_ensemble):\n    y_pred[index, :] = model.predict(test_img.reshape(1,224,224,3))\n    plt.scatter(column_names[1:], y_pred[index,:], color = 'b', alpha = 0.3)\n\nplt.scatter(column_names[1:], np.mean(y_pred, axis = 0), color = 'r', label = 'ensemble mean')\nplt.title('Red --> mean prediction | Blue --> k-fold prediction')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Above plot**: The blue scatter points correspond to each k-fold neural network prediction, so we should see 5 blue points for each column because we conducted k=5 cross-validation. The red points is the ensemble model's mean prediction. We can see that each k-fold network disagrees with each other, while also, it seems like the ensemble model cannot differentiate between label 2 and 4. "},{"metadata":{},"cell_type":"markdown","source":"**Summary on the ensemble model's peformanceo on the test sample:** We can see that our ensemble model is very poor because the prediction on the test sample has high entropy, and the two highest proability predictions (label 1 and 4) are difficult to differentiate. However, it is safe to say that the ensemble model indicates high confidence in predicting label 0 and 3 because each individual k-fold network predicted 0 probability. Unsuprisingly, albeit the ensemble model seems to be very confident in predicting label 3, this class label is highly abudant; in other words, it seems that this baseline ensemble model has overfitted to this over imbalanced. However, which is somewhate suprising, ensemble model predict label 0 quite confidently, indicating that this class is quite easy to differentiate. Overall, this will perforamnce is okay considering that this Vgg16 ensemble model is only a cheap baseline. "},{"metadata":{},"cell_type":"markdown","source":"## Class Activation Maps:\n\nBecause our model implements a Global Average Pooling layer before the final dense layer, we can implement class activation maps to see the overall model performance. \n"},{"metadata":{},"cell_type":"markdown","source":"The below code can be found in my other notebook, implementing class activation maps on the MNIST dataset: [MNIST Activation Maps](https://www.kaggle.com/niksapraljak/mnist-activation-maps)"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\n    Implementing class activation maps for architectures with Global Average Pooling 2D before the final dense layer \n\"\"\"\nclass Leaf_CAM:\n    \n    def __init__(self, img):\n        self.resize_width, self.resize_height, _ = img.shape    \n    \n    # zero-center normalization \n    def standard_norm(self, img):\n        return ((img - np.mean(img))/np.std(img))\n    \n    # final layer should be (7,7,2048)\n    def feature_model(self, model):  \n        return Model(inputs = model.layers[0].input, outputs = model.layers[-3].output)\n    \n    # final weight tensor before classification layer is 3*2048\n    def weight_tensor(self, model):\n        final_outputs = model.layers[-1]\n        return final_outputs.get_weights()[0]\n    \n    # output prediction class of the image of interest\n    def predict_class(self, model, X):\n        prob_vec = model.predict(X)\n        return np.argmax(prob_vec[0])\n        \n    # generate class activation maps (CAMs)    \n    def generate_CAM(self, model, img):\n        norm_img = self.standard_norm(img)\n        Fmap_model = self.feature_model(model)\n        Wtensor = self.weight_tensor(model)\n        feature_map = Fmap_model.predict(norm_img.reshape(1,224,224,3))\n        label = self.predict_class(model, norm_img.reshape(1,224,224,3))\n        CAM = feature_map.dot(Wtensor[:,label])[0,:,:]\n        return cv.resize(CAM, \n                         (self.resize_width, self.resize_height),\n                         interpolation = cv.INTER_CUBIC), label\n    \n    # generate probability vector \n    def generate_probvec(self, model, img):\n        X = self.standard_norm(img)\n        prob_vec = model.predict(X.reshape(1,224,224,3))\n        return prob_vec","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'Example code: quickly compute a class activation map for a random image sample ...'\n\ndef read_img(file_path):\n    image = cv.imread(file_path).astype('float32')\n    image = cv.resize(image, (224,224), interpolation = cv.INTER_CUBIC)\n    # normalize image \n    norm_image = standard_norm(image.copy())\n    return norm_image, image\n\n\n# generate a random activation map\n\nrand_val = random.randint(0, 20000)\n# example image \nfile_path = train_df.image_path[rand_val]\nnorm_img, img = read_img(file_path)\nCAM_generator = Leaf_CAM(img)\nplt.subplot(1,2,1)\nplt.imshow(img.astype('uint'))\nplt.subplot(1,2,2)\nplt.imshow(img.astype('uint'))\nactivation_map, label = CAM_generator.generate_CAM(model_ensemble[0], img)\nplt.imshow(activation_map,'jet', alpha = 0.5)\nplt.title(\"Predicted Class: \" + str(label))\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Activation Map Analysis:"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Plot Cassava Bacterial (CBB) Blight Activation Maps: "},{"metadata":{"trusted":true},"cell_type":"code","source":"CBB_df = train_df[train_df.label == 0]\n\nfor index in range(5):\n\n    file_path = CBB_df.image_path.iloc[index]\n    norm_img, img = read_img(file_path)\n    CAM_generator = Leaf_CAM(img)\n    plt.figure(figsize = (10,5))\n    plt.subplot(1,2,1)\n    plt.imshow(img.astype('uint'))\n    plt.subplot(1,2,2)\n    plt.imshow(img.astype('uint'))\n    activation_map, label = CAM_generator.generate_CAM(model_ensemble[0], img)\n    plt.imshow(activation_map,'jet', alpha = 0.5)\n    plt.title(\"Predicted Class: \" + str(label))\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot Cassava Brown Streak Disease (CBSD) Activation Maps:\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"CBSD_df = train_df[train_df.label == 1]\n\nfor index in range(5):\n\n    file_path = CBSD_df.image_path.iloc[index]\n    norm_img, img = read_img(file_path)\n    CAM_generator = Leaf_CAM(img)\n    plt.figure(figsize = (10,5))\n    plt.subplot(1,2,1)\n    plt.imshow(img.astype('uint'))\n    plt.subplot(1,2,2)\n    plt.imshow(img.astype('uint'))\n    activation_map, label = CAM_generator.generate_CAM(model_ensemble[0], img)\n    plt.imshow(activation_map,'jet', alpha = 0.5)\n    plt.title(\"Predicted Class: \" + str(label))\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot Cassava Green Mottle (CGM) Acativation Maps: "},{"metadata":{"trusted":true},"cell_type":"code","source":"CGM_df = train_df[train_df.label == 2]\n\nfor index in range(5):\n\n    file_path = CGM_df.image_path.iloc[index]\n    norm_img, img = read_img(file_path)\n    CAM_generator = Leaf_CAM(img)\n    plt.figure(figsize = (10,5))\n    plt.subplot(1,2,1)\n    plt.imshow(img.astype('uint'))\n    plt.subplot(1,2,2)\n    plt.imshow(img.astype('uint'))\n    activation_map, label = CAM_generator.generate_CAM(model_ensemble[0], img)\n    plt.imshow(activation_map,'jet', alpha = 0.5)\n    plt.title(\"Predicted Class: \" + str(label))\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##  Plot Cassava Mosaic Disease (CMD) Activation Maps:\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"CMD_df = train_df[train_df.label == 3]\n\nfor index in range(5):\n\n    file_path = CMD_df.image_path.iloc[index]\n    norm_img, img = read_img(file_path)\n    CAM_generator = Leaf_CAM(img)\n    plt.figure(figsize = (10,5))\n    plt.subplot(1,2,1)\n    plt.imshow(img.astype('uint'))\n    plt.subplot(1,2,2)\n    plt.imshow(img.astype('uint'))\n    activation_map, label = CAM_generator.generate_CAM(model_ensemble[0], img)\n    plt.imshow(activation_map,'jet', alpha = 0.5)\n    plt.title(\"Predicted Class: \" + str(label))\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##  Plot Healthly Activation Maps:\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"He_df = train_df[train_df.label == 4]\n\nfor index in range(5):\n\n    file_path = He_df.image_path.iloc[index]\n    norm_img, img = read_img(file_path)\n    CAM_generator = Leaf_CAM(img)\n    plt.figure(figsize = (10,5))\n    plt.subplot(1,2,1)\n    plt.imshow(img.astype('uint'))\n    plt.subplot(1,2,2)\n    plt.imshow(img.astype('uint'))\n    activation_map, label = CAM_generator.generate_CAM(model_ensemble[0], img)\n    plt.imshow(activation_map,'jet', alpha = 0.5)\n    plt.title(\"Predicted Class: \" + str(label))\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Save ensemble model\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfor kfold, model in enumerate(model_ensemble):\n    # serialize model to JSON\n    model_json = model.to_json()\n    with open(f\"model-{kfold}-vgg16.json\", \"w\") as json_file:\n        json_file.write(model_json)\n    # serialize weights to HDF5\n    model.save_weights(f\"model-{kfold}-vgg16.h5\")\n    print(\"Saved model to disk\")\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}