{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## I would like to give credit to the authors found below. I learned a great deal from all of your works!  (https://www.kaggle.com/c/aptos2019-blindness-detection/discussion/105305)\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nimport numpy as np\nimport os\nimport shutil\nfrom tqdm import tqdm\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport keras\nfrom keras.preprocessing import image\nfrom keras import layers\nfrom keras.models import Sequential\nfrom keras.applications import VGG16, ResNet50, Xception\nfrom keras import optimizers\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D\nfrom keras.optimizers import RMSprop,SGD,Adagrad,Adadelta,Adam,Adamax,Nadam\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.layers import BatchNormalization\nfrom keras import utils\nfrom keras.callbacks import Callback, ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom keras.preprocessing.image import ImageDataGenerator\nimport h5py\nfrom keras.models import load_model\nfrom PIL import Image\nimport cv2\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import class_weight, shuffle\nfrom sklearn.metrics import accuracy_score\n\nsns.set(style='white', context='notebook', palette='dark')\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntest_df = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nprint(train_df.shape)\nprint(test_df.shape)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"g = sns.countplot(train_df['diagnosis'])\nsns.despine()\ntrain_df['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_samples(df, columns=2, rows=2):\n    fig=plt.figure(figsize=(5*columns, 4*rows))\n\n    for i in range(columns*rows):\n        image_path = df.loc[i,'id_code']\n        image_id = df.loc[i,'diagnosis']\n        img = cv2.imread(f'../input/aptos2019-blindness-detection/train_images/{image_path}.png')\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        fig.add_subplot(rows, columns, i+1)\n        plt.title(image_id)\n        plt.imshow(img)\n    \n    plt.tight_layout()\n\ndisplay_samples(train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualizing augmented data"},{"metadata":{"trusted":true},"cell_type":"code","source":"orig_img = np.array(Image.open('../input/aptos2019-blindness-detection/test_images/351aba543dc8.png'))\nplt.imshow(orig_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfrom keras.preprocessing.image import ImageDataGenerator, img_to_array, load_img\ndef generate_plot_pics(datagen,orig_img):\n    dir_augmented_data = \"/kaggle/working/aug_preview\"\n    try:\n        ## if the preview folder does not exist, create\n        os.mkdir(dir_augmented_data)\n    except:\n        ## if the preview folder exists, then remove\n        ## the contents (pictures) in the folder\n        for item in os.listdir(dir_augmented_data):\n            os.remove(dir_augmented_data + \"/\" + item)\n    ## convert the original image to array\n    x = img_to_array(orig_img)\n    ## reshape (Sampke, Nrow, Ncol, 3) 3 = R, G or B\n    x = x.reshape((1,) + x.shape)\n    ## -------------------------- ##\n    ## randomly generate pictures\n    ## -------------------------- ##\n    i = 0\n    Nplot = 8\n    for batch in datagen.flow(x,batch_size=1,\n                        save_to_dir=dir_augmented_data,\n                        save_prefix=\"pic\",\n                        save_format='png'):\n        i += 1\n        if i > Nplot - 1: ## generate 8 pictures\n            break\n    ## -------------------------- ##\n    ## plot the generated data\n    ## -------------------------- ##\n    fig = plt.figure(figsize=(8, 6))\n    fig.subplots_adjust(hspace=0.02,wspace=0.01,\n    left=0,right=1,bottom=0, top=1)\n    ## original picture\n    ax = fig.add_subplot(3, 3, 1,xticks=[],yticks=[])\n    ax.imshow(orig_img)\n    ax.set_title(\"original\")\n    i = 2\n    for imgnm in os.listdir(dir_augmented_data):\n        ax = fig.add_subplot(3, 3, i,xticks=[],yticks=[])\n        img = load_img(dir_augmented_data + \"/\" + imgnm)\n        ax.imshow(img)\n        i += 1\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## rotation_range: Int. Degree range for random rotations.\ndatagen = ImageDataGenerator(rotation_range=30)\ngenerate_plot_pics(datagen,orig_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## zoom_range: Float or [lower, upper]. Range for random zoom.\n## If a float, [lower, upper] = [1-zoom_range, 1+zoom_range].\ndatagen = ImageDataGenerator(zoom_range=0.1)\ngenerate_plot_pics(datagen,orig_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(vertical_flip=True)\ngenerate_plot_pics(datagen,orig_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_pad_width(im, new_shape, is_rgb=True):\n    pad_diff = new_shape - im.shape[0], new_shape - im.shape[1]\n    t, b = math.floor(pad_diff[0]/2), math.ceil(pad_diff[0]/2) # padding the top (t) and bottom (b) of images\n    l, r = math.floor(pad_diff[1]/2), math.ceil(pad_diff[1]/2) # padding the left (l) and right (r) of images\n    if is_rgb:\n        pad_width = ((t,b), (l,r), (0, 0))\n    else:\n        pad_width = ((t,b), (l,r))\n    return pad_width\n\ndef preprocess_image(image_path, desired_size=224): #choosing image size\n    im = Image.open(image_path)\n    im = im.resize((desired_size, )*2, resample=Image.LANCZOS)\n    \n    return im","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N = train_df.shape[0]\nx_train = np.empty((N, 224, 224, 3), dtype=np.uint8) #converting data into a numpy array\n\nfor i, image_id in enumerate(tqdm(train_df['id_code'])):\n    x_train[i, :, :, :] = preprocess_image(\n        f'../input/aptos2019-blindness-detection/train_images/{image_id}.png'\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N = test_df.shape[0]\nx_test = np.empty((N, 224, 224, 3), dtype=np.uint8)\n\nfor i, image_id in enumerate(tqdm(test_df['id_code'])):\n    x_test[i, :, :, :] = preprocess_image(\n        f'../input/aptos2019-blindness-detection/test_images/{image_id}.png'\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#y_train = pd.get_dummies(train_df['diagnosis']).values\ny_train = keras.utils.to_categorical(train_df['diagnosis'], num_classes=5, dtype='float32')\nprint(x_train.shape)\nprint(y_train.shape) \n#print(x_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #Creating multilabels (see https://arxiv.org/pdf/0704.1028.pdf)\n# y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)\n# y_train_multi[:, 4] = y_train[:, 4]\n\n# for i in range(3, -1, -1):\n#    y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])\n\n# print(\"Original y_train:\", y_train.sum(axis=0))\n# print(\"Multilabel version:\", y_train_multi.sum(axis=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#splitting data for train and validation \nx_train, x_val, y_train, y_val = train_test_split(\n    x_train, y_train, \n    test_size=0.2, \n    random_state=8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#normalizing data\nx_train = x_train / 255\nx_val = x_val / 255\n#x_test = x_test / 255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_datagen():\n    return ImageDataGenerator(\n        rotation_range=30, # set range for random rotation in degrees\n        zoom_range=0.10,  # set range for random zoom \n        # set mode for filling points outside the input boundaries\n        fill_mode='constant',\n        cval=0.,  # value used for fill_mode = \"constant\"\n        horizontal_flip=True,  # randomly flip images\n        vertical_flip=True,  # randomly flip images\n    )\ndata_generator = create_datagen().flow(x_train, y_train, batch_size=30, seed=8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 30\nnum_classes = 5\nepochs = 20\ninput_shape = (224, 224, 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\nfrom keras.models import Sequential\nfrom keras import optimizers\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D\nfrom keras.optimizers import RMSprop,SGD,Adagrad,Adadelta,Adam,Adamax,Nadam\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.layers import BatchNormalization\nimport h5py\nfrom keras.models import load_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3 convolution layers (conv-pool-conv-pool)"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_a = Sequential()\n\nmodel_a.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_a.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_a.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_a.add(Flatten())\nmodel_a.add(Dense(512, activation='relu'))\nmodel_a.add(Dense(5, activation='softmax'))\n\n\nmodel_a.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_a.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_a.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance with 3 convolution layers, Adam optimizer, and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3 convolution layers (conv-conv-pool-conv-conv-pool)"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_b = Sequential()\n\nmodel_b.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_b.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_b.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_b.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_b.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_b.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_b.add(Flatten())\nmodel_b.add(Dense(512, activation='relu'))\nmodel_b.add(Dense(5, activation='softmax'))\n\n\nmodel_b.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_b.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_b.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance with 3 convolution layers, Adam optimizer, and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Determining optimal number of layers\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_b = Sequential()\n\nmodel_b.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_b.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_b.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_b.add(Conv2D(filters = 128, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_b.add(Flatten())\nmodel_b.add(Dense(512, activation='relu'))\nmodel_b.add(Dense(5, activation='softmax'))\n\n\nmodel_b.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_b.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_b.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance with 4 convolution layers, Adam optimizer, and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_c = Sequential()\n\nmodel_c.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_c.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_c.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_c.add(Conv2D(filters = 128, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_c.add(Conv2D(filters = 256, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_c.add(Flatten())\nmodel_c.add(Dense(512, activation='relu'))\nmodel_c.add(Dense(5, activation='softmax'))\n\n\nmodel_c.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_c.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_c.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance with 5 convolution layers, Adam optimizer, and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_d = Sequential()\n\nmodel_d.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_d.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_d.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_d.add(Conv2D(filters = 128, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_d.add(Conv2D(filters = 256, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_d.add(Conv2D(filters = 512, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_d.add(Flatten())\nmodel_d.add(Dense(512, activation='relu'))\nmodel_d.add(Dense(5, activation='softmax'))\n\n\nmodel_d.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_d.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_d.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance with 6 convolution layers, Adam optimizer, and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_d = Sequential()\n\n# model_d.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n#                  input_shape=input_shape))\n# model_d.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n#                  input_shape=input_shape))\n# model_d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_d.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_d.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_d.add(Conv2D(filters = 128, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(Conv2D(filters = 128, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_d.add(Conv2D(filters = 256, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(Conv2D(filters = 256, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_d.add(Conv2D(filters = 512, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(Conv2D(filters = 512, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_d.add(Flatten())\nmodel_d.add(Dense(512, activation='relu'))\nmodel_d.add(Dense(5, activation='softmax'))\n\n\nmodel_d.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_d.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_d.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Selecting best optimizer"},{"metadata":{},"cell_type":"markdown","source":"# 1a. Simple model using 1 convolutional layer and RMSprop optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_1a = Sequential()\n\nmodel_1a.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_1a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_1a.add(Flatten())\nmodel_1a.add(Dense(512, activation='relu'))\nmodel_1a.add(Dense(5, activation='relu'))\n\n\nmodel_1a.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.RMSprop(),\n              metrics=['accuracy'])\n\nmodel_1a.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_1a.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Single-layer CNN Performance with RMSprop optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1b. Simple model using 1 convolutional layer and Adam optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_1b = Sequential()\n\nmodel_1b.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_1b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_1b.add(Flatten())\nmodel_1b.add(Dense(512, activation='relu'))\nmodel_1b.add(Dense(5, activation='softmax'))\n\nmodel_1b.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_1b.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_1b.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Single-layer CNN Performance with Adam optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1c. Simple model using 1 convolutional layer and Adamax optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_1c = Sequential()\n\nmodel_1c.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_1c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_1c.add(Flatten())\nmodel_1c.add(Dense(512, activation='relu'))\n#model_1c.add(Dense(5, activation='relu'))\nmodel_1c.add(Dense(5, activation='softmax'))\n\nmodel_1c.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adamax(),\n              metrics=['accuracy'])\n\nmodel_1c.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_1c.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Single-layer CNN Performance with Adamax optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1d. Simple model using 1 convolutional layer and Nadam optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_1d = Sequential()\n\nmodel_1d.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_1d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_1d.add(Flatten())\nmodel_1d.add(Dense(512, activation='relu'))\n#model_1d.add(Dense(5, activation='relu'))\nmodel_1d.add(Dense(5, activation='softmax'))\n\nmodel_1d.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Nadam(),\n              metrics=['accuracy'])\n\nmodel_1d.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_1d.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Single-layer CNN Performance with Nadam optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2a. Simple model using 2 convolutional layers and RMSprop optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_2a = Sequential()\n\nmodel_2a.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2a.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2a.add(Flatten())\nmodel_2a.add(Dense(512, activation='relu'))\nmodel_2a.add(Dense(5, activation='relu'))\n\n\nmodel_2a.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.RMSprop(),\n              metrics=['accuracy'])\n\nmodel_2a.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_2a.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Single-layer CNN Performance with RMSprop optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_2a2 = Sequential()\n\nmodel_2a2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2a2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2a2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2a2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2a2.add(Flatten())\nmodel_2a2.add(Dense(512, activation='relu'))\nmodel_2a2.add(Dense(5, activation='softmax'))\n\nmodel_2a2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.RMSprop(),\n              metrics=['accuracy'])\n\nmodel_2a2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_2a2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Two-layer CNN Performance with RMSprop optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2b. Simple model using 2 convolutional layers and Adam optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_2b = Sequential()\n\nmodel_2b.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2b.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2b.add(Flatten())\nmodel_2b.add(Dense(512, activation='relu'))\nmodel_2b.add(Dense(5, activation='relu'))\n\n\nmodel_2b.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_2b.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_2b.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Two-layer CNN Performance with Adam optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_2b2 = Sequential()\n\nmodel_2b2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2b2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2b2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2b2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2b2.add(Flatten())\nmodel_2b2.add(Dense(512, activation='relu'))\nmodel_2b2.add(Dense(5, activation='softmax'))\n\nmodel_2b2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_2b2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_2b2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Two-layer CNN Performance with Adam optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2c. Simple model using 2 convolutional layers and Adamax optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_2c = Sequential()\n\nmodel_2c.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2c.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2c.add(Flatten())\nmodel_2c.add(Dense(512, activation='relu'))\nmodel_2c.add(Dense(5, activation='relu'))\n\n\nmodel_2c.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adamax(),\n              metrics=['accuracy'])\n\nmodel_2c.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_2c.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Two-layer CNN Performance with Adamax optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_2c2 = Sequential()\n\nmodel_2c2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2c2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2c2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2c2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2c2.add(Flatten())\nmodel_2c2.add(Dense(512, activation='relu'))\nmodel_2c2.add(Dense(5, activation='softmax'))\n\n\nmodel_2c2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adamax(),\n              metrics=['accuracy'])\n\nmodel_2c2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_2c2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Two-layer CNN Performance with Adamax optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2d. Simple model using 2 convolutional layers and Nadam optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_2d = Sequential()\n\nmodel_2d.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2d.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2d.add(Flatten())\nmodel_2d.add(Dense(512, activation='relu'))\nmodel_2d.add(Dense(5, activation='relu'))\n\n\nmodel_2d.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Nadam(),\n              metrics=['accuracy'])\n\nmodel_2d.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_2d.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Two-layer CNN Performance with Nadam optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_2d2 = Sequential()\n\nmodel_2d2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2d2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2d2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_2d2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2d2.add(Flatten())\nmodel_2d2.add(Dense(512, activation='relu'))\nmodel_2d2.add(Dense(5, activation='softmax'))\n\nmodel_2d2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Nadam(),\n              metrics=['accuracy'])\n\nmodel_2d2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_2d2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Two-layer CNN Performance with Nadam optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3a. Simple model using 3 convolutional layers and RMSprop optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_3a = Sequential()\n\nmodel_3a.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3a.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3a.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3a.add(Flatten())\nmodel_3a.add(Dense(512, activation='relu'))\nmodel_3a.add(Dense(5, activation='relu'))\n\n\nmodel_3a.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.RMSprop(),\n              metrics=['accuracy'])\n\nmodel_3a.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_3a.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with RMSprop optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_3a2 = Sequential()\n\nmodel_3a2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3a2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3a2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3a2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3a2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3a2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3a2.add(Flatten())\nmodel_3a2.add(Dense(512, activation='relu'))\nmodel_3a2.add(Dense(5, activation='softmax'))\n\nmodel_3a2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.RMSprop(),\n              metrics=['accuracy'])\n\nmodel_3a2.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_3a2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with RMSprop optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3b. Simple model using 3 convolutional layers and Adam optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_3b = Sequential()\n\nmodel_3b.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3b.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3b.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3b.add(Flatten())\nmodel_3b.add(Dense(512, activation='relu'))\nmodel_3b.add(Dense(5, activation='relu'))\n\n\nmodel_3b.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_3b.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_3b.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with Adam optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_3b2 = Sequential()\n\nmodel_3b2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3b2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3b2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3b2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3b2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3b2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3b2.add(Flatten())\nmodel_3b2.add(Dense(512, activation='relu'))\nmodel_3b2.add(Dense(5, activation='softmax'))\n\nmodel_3b2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_3b2.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_3b2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with Adam optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3c. Simple model using 3 convolutional layers and Adamax optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_3c = Sequential()\n\nmodel_3c.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3c.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3c.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3c.add(Flatten())\nmodel_3c.add(Dense(512, activation='relu'))\nmodel_3c.add(Dense(5, activation='relu'))\n\n\nmodel_3c.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adamax(),\n              metrics=['accuracy'])\n\nmodel_3c.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_3c.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with Adamax optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_3c2 = Sequential()\n\nmodel_3c2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3c2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3c2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3c2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3c2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3c2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3c2.add(Flatten())\nmodel_3c2.add(Dense(512, activation='relu'))\nmodel_3c2.add(Dense(5, activation='softmax'))\n\nmodel_3c2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adamax(),\n              metrics=['accuracy'])\n\nmodel_3c2.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_3c2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with Adamax optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3d. Simple model using 3 convolutional layers and Nadam optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_3d = Sequential()\n\nmodel_3d.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3d.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3d.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3d.add(Flatten())\nmodel_3d.add(Dense(512, activation='relu'))\nmodel_3d.add(Dense(5, activation='relu'))\n\n\nmodel_3d.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Nadam(),\n              metrics=['accuracy'])\n\nmodel_3d.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_3d.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with Nadam optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_3d2 = Sequential()\n\nmodel_3d2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3d2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3d2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3d2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3d2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_3d2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_3d2.add(Flatten())\nmodel_3d2.add(Dense(512, activation='relu'))\nmodel_3d2.add(Dense(5, activation='softmax'))\n\nmodel_3d2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Nadam(),\n              metrics=['accuracy'])\n\nmodel_3d2.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_3d2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with Nadam optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4a. CNN model using 3 convolutional layers with regularization using dropout (0.1) and RMSprop optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_4a = Sequential()\n\nmodel_4a.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4a.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4a.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4a.add(Flatten())\nmodel_4a.add(Dense(512, activation='relu'))\nmodel_4a.add(Dropout(0.1))\nmodel_4a.add(Dense(512, activation='relu'))\nmodel_4a.add(Dropout(0.1))\nmodel_4a.add(Dense(5, activation='relu'))\n\n\nmodel_4a.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.RMSprop(),\n              metrics=['accuracy'])\n\nmodel_4a.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_4a.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.1 dropout with RMSprop optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_4a2 = Sequential()\n\nmodel_4a2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4a2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4a2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4a2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4a2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4a2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4a2.add(Flatten())\nmodel_4a2.add(Dense(512, activation='relu'))\nmodel_4a2.add(Dropout(0.1))\nmodel_4a2.add(Dense(512, activation='relu'))\nmodel_4a2.add(Dropout(0.1))\nmodel_4a2.add(Dense(5, activation='softmax'))\n\nmodel_4a2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.RMSprop(),\n              metrics=['accuracy'])\n\nmodel_4a2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_4a2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.1 dropout with RMSprop optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4b. CNN model using 3 convolutional layers with regularization using dropout (0.1) and Adam optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_4b = Sequential()\n\nmodel_4b.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4b.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4b.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4b.add(Flatten())\nmodel_4b.add(Dense(512, activation='relu'))\nmodel_4b.add(Dropout(0.1))\nmodel_4b.add(Dense(512, activation='relu'))\nmodel_4b.add(Dropout(0.1))\nmodel_4b.add(Dense(5, activation='relu'))\n\n\nmodel_4b.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_4b.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_4b.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.1 dropout with Adam optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_4b2 = Sequential()\n\nmodel_4b2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4b2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4b2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4b2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4b2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4b2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4b2.add(Flatten())\nmodel_4b2.add(Dense(512, activation='relu'))\nmodel_4b2.add(Dropout(0.1))\nmodel_4b2.add(Dense(512, activation='relu'))\nmodel_4b2.add(Dropout(0.1))\nmodel_4b2.add(Dense(5, activation='softmax'))\n\nmodel_4b2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_4b2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_4b2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.1 dropout with Adam optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4c. CNN model using 3 convolutional layers with regularization using dropout (0.1) and Adamax optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_4c = Sequential()\n\nmodel_4c.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4c.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4c.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4c.add(Flatten())\nmodel_4c.add(Dense(512, activation='relu'))\nmodel_4c.add(Dropout(0.1))\nmodel_4c.add(Dense(512, activation='relu'))\nmodel_4c.add(Dropout(0.1))\nmodel_4c.add(Dense(5, activation='relu'))\n\n\nmodel_4c.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adamax(),\n              metrics=['accuracy'])\n\nmodel_4c.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_4c.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.1 dropout with Adamax optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_4c2 = Sequential()\n\nmodel_4c2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4c2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4c2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4c2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4c2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4c2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4c2.add(Flatten())\nmodel_4c2.add(Dense(512, activation='relu'))\nmodel_4c2.add(Dropout(0.1))\nmodel_4c2.add(Dense(512, activation='relu'))\nmodel_4c2.add(Dropout(0.1))\nmodel_4c2.add(Dense(5, activation='softmax'))\n\nmodel_4c2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adamax(),\n              metrics=['accuracy'])\n\nmodel_4c2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_4c2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.1 dropout with Adamax optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4d. CNN model using 3 convolutional layers with regularization using dropout (0.1) and Nadam optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_4d = Sequential()\n\nmodel_4d.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4d.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4d.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4d.add(Flatten())\nmodel_4d.add(Dense(512, activation='relu'))\nmodel_4d.add(Dropout(0.1))\nmodel_4d.add(Dense(512, activation='relu'))\nmodel_4d.add(Dropout(0.1))\nmodel_4d.add(Dense(5, activation='relu'))\n\n\n\nmodel_4d.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Nadam(),\n              metrics=['accuracy'])\n\nmodel_4d.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_4d.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.1 dropout with Nadam optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_4d2 = Sequential()\n\nmodel_4d2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4d2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4d2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4d2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4d2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_4d2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_4d2.add(Flatten())\nmodel_4d2.add(Dense(512, activation='relu'))\nmodel_4d2.add(Dropout(0.1))\nmodel_4d2.add(Dense(512, activation='relu'))\nmodel_4d2.add(Dropout(0.1))\nmodel_4d2.add(Dense(5, activation='softmax'))\n\n\nmodel_4d2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Nadam(),\n              metrics=['accuracy'])\n\nmodel_4d2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_4d2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.1 dropout with Nadam optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5a. CNN model using 3 convolutional layers with regularization using dropout (0.3) and RMSprop optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_5a = Sequential()\n\nmodel_5a.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5a.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5a.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5a.add(Flatten())\nmodel_5a.add(Dense(512, activation='relu'))\nmodel_5a.add(Dropout(0.3))\nmodel_5a.add(Dense(512, activation='relu'))\nmodel_5a.add(Dropout(0.3))\nmodel_5a.add(Dense(5, activation='relu'))\n\n\nmodel_5a.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.RMSprop(),\n              metrics=['accuracy'])\n\nmodel_5a.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_5a.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.3 dropout with RMSprop optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_5a2 = Sequential()\n\nmodel_5a2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5a2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5a2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5a2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5a2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5a2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5a2.add(Flatten())\nmodel_5a2.add(Dense(512, activation='relu'))\nmodel_5a2.add(Dropout(0.3))\nmodel_5a2.add(Dense(512, activation='relu'))\nmodel_5a2.add(Dropout(0.3))\nmodel_5a2.add(Dense(5, activation='softmax'))\n\nmodel_5a2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.RMSprop(),\n              metrics=['accuracy'])\n\nmodel_5a2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_5a2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.3 dropout with RMSprop optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5b. CNN model using 3 convolutional layers with regularization using dropout (0.3) and Adam optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_5b = Sequential()\n\nmodel_5b.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5b.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5b.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5b.add(Flatten())\nmodel_5b.add(Dense(512, activation='relu'))\nmodel_5b.add(Dropout(0.3))\nmodel_5b.add(Dense(512, activation='relu'))\nmodel_5b.add(Dropout(0.3))\nmodel_5b.add(Dense(5, activation='relu'))\n\n\nmodel_5b.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_5b.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_5b.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.3 dropout with Adam optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_5b2 = Sequential()\n\nmodel_5b2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5b2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5b2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5b2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5b2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5b2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5b2.add(Flatten())\nmodel_5b2.add(Dense(512, activation='relu'))\nmodel_5b2.add(Dropout(0.3))\nmodel_5b2.add(Dense(512, activation='relu'))\nmodel_5b2.add(Dropout(0.3))\nmodel_5b2.add(Dense(5, activation='softmax'))\n\nmodel_5b2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_5b2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_5b2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.3 dropout with Adam optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5c. CNN model using 3 convolutional layers with regularization using dropout (0.3) and Adamax optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_5c = Sequential()\n\nmodel_5c.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5c.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5c.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5c.add(Flatten())\nmodel_5c.add(Dense(512, activation='relu'))\nmodel_5c.add(Dropout(0.3))\nmodel_5c.add(Dense(512, activation='relu'))\nmodel_5c.add(Dropout(0.3))\nmodel_5c.add(Dense(5, activation='relu'))\n\n\nmodel_5c.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adamax(),\n              metrics=['accuracy'])\n\nmodel_5c.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_5c.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.3 dropout with Adamax optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_5c2 = Sequential()\n\nmodel_5c2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5c2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5c2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5c2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5c2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5c2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5c2.add(Flatten())\nmodel_5c2.add(Dense(512, activation='relu'))\nmodel_5c2.add(Dropout(0.3))\nmodel_5c2.add(Dense(512, activation='relu'))\nmodel_5c2.add(Dropout(0.3))\nmodel_5c2.add(Dense(5, activation='softmax'))\n\nmodel_5c2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adamax(),\n              metrics=['accuracy'])\n\nmodel_5c2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_5c2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.3 dropout with Adamax optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5d. CNN model using 3 convolutional layers with regularization using dropout (0.3) and Nadam optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_5d = Sequential()\n\nmodel_5d.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5d.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5d.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5d.add(Flatten())\nmodel_5d.add(Dense(512, activation='relu'))\nmodel_5d.add(Dropout(0.3))\nmodel_5d.add(Dense(512, activation='relu'))\nmodel_5d.add(Dropout(0.3))\nmodel_5d.add(Dense(5, activation='relu'))\n\n\nmodel_5d.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Nadam(),\n              metrics=['accuracy'])\n\nmodel_5d.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_5d.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.3 dropout with Nadam optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_5d2 = Sequential()\n\nmodel_5d2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5d2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5d2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5d2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5d2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_5d2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_5d2.add(Flatten())\nmodel_5d2.add(Dense(512, activation='relu'))\nmodel_5d2.add(Dropout(0.3))\nmodel_5d2.add(Dense(512, activation='relu'))\nmodel_5d2.add(Dropout(0.3))\nmodel_5d2.add(Dense(5, activation='softmax'))\n\nmodel_5d2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Nadam(),\n              metrics=['accuracy'])\n\nmodel_5d2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_5d2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.3 dropout with Nadam optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6a. CNN model using 3 convolutional layers with regularization using dropout (0.5) and RMSprop optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_6a = Sequential()\n\nmodel_6a.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6a.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6a.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6a.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6a.add(Flatten())\nmodel_6a.add(Dense(512, activation='relu'))\nmodel_6a.add(Dropout(0.5))\nmodel_6a.add(Dense(512, activation='relu'))\nmodel_6a.add(Dropout(0.5))\nmodel_6a.add(Dense(5, activation='relu'))\n\n\nmodel_6a.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.RMSprop(),\n              metrics=['accuracy'])\n\nmodel_6a.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_6a.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.5 dropout with RMSprop optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_6a2 = Sequential()\n\nmodel_6a2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6a2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6a2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6a2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6a2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6a2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6a2.add(Flatten())\nmodel_6a2.add(Dense(512, activation='relu'))\nmodel_6a2.add(Dropout(0.5))\nmodel_6a2.add(Dense(512, activation='relu'))\nmodel_6a2.add(Dropout(0.5))\nmodel_6a2.add(Dense(5, activation='softmax'))\n\nmodel_6a2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.RMSprop(),\n              metrics=['accuracy'])\n\nmodel_6a2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_6a2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.5 dropout with RMSprop optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6b. CNN model using 3 convolutional layers with regularization using dropout (0.5) and Adam optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_6b = Sequential()\n\nmodel_6b.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6b.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6b.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6b.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6b.add(Flatten())\n# model_6b.add(Dense(512, activation='relu'))\n# model_6b.add(Dropout(0.5))\nmodel_6b.add(Dense(512, activation='relu'))\nmodel_6b.add(Dropout(0.5))\n#model_6b.add(Dense(5, activation='relu'))\nmodel_6b.add(Dense(5, activation='softmax'))\n\n\nmodel_6b.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_6b.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_6b.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.5 dropout with Adam optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_6b2 = Sequential()\n\nmodel_6b2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6b2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6b2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6b2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6b2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6b2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6b2.add(Flatten())\nmodel_6b2.add(Dense(512, activation='relu'))\nmodel_6b2.add(Dropout(0.5))\nmodel_6b2.add(Dense(512, activation='relu'))\nmodel_6b2.add(Dropout(0.5))\nmodel_6b2.add(Dense(5, activation='softmax'))\n\nmodel_6b2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_6b2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_6b2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.5 dropout with Adam optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6c. CNN model using 3 convolutional layers with regularization using dropout (0.5) and Adamax optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_6c = Sequential()\n\nmodel_6c.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6c.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6c.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6c.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6c.add(Flatten())\nmodel_6c.add(Dense(512, activation='relu'))\nmodel_6c.add(Dropout(0.5))\nmodel_6c.add(Dense(512, activation='relu'))\nmodel_6c.add(Dropout(0.5))\nmodel_6c.add(Dense(5, activation='relu'))\n\n\nmodel_6c.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adamax(),\n              metrics=['accuracy'])\n\nmodel_6c.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_6c.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.5 dropout with Adamax optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_6c2 = Sequential()\n\nmodel_6c2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6c2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6c2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6c2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6c2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6c2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6c2.add(Flatten())\nmodel_6c2.add(Dense(512, activation='relu'))\nmodel_6c2.add(Dropout(0.5))\nmodel_6c2.add(Dense(512, activation='relu'))\nmodel_6c2.add(Dropout(0.5))\nmodel_6c2.add(Dense(5, activation='softmax'))\n\nmodel_6c2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adamax(),\n              metrics=['accuracy'])\n\nmodel_6c2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_6c2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.5 dropout with Adamax optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6d. CNN model using 3 convolutional layers with regularization using dropout (0.5) and Nadam optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_6d = Sequential()\n\nmodel_6d.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6d.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6d.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6d.add(Flatten())\nmodel_6d.add(Dense(512, activation='relu'))\nmodel_6d.add(Dropout(0.5))\nmodel_6d.add(Dense(512, activation='relu'))\nmodel_6d.add(Dropout(0.5))\nmodel_6d.add(Dense(5, activation='relu'))\n\n\nmodel_6d.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Nadam(),\n              metrics=['accuracy'])\n\nmodel_6d.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_6d.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.5 dropout with Nadam optimizer and relu activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_6d2 = Sequential()\n\nmodel_6d2.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6d2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6d2.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6d2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6d2.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_6d2.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_6d2.add(Flatten())\nmodel_6d2.add(Dense(512, activation='relu'))\nmodel_6d2.add(Dropout(0.5))\nmodel_6d2.add(Dense(512, activation='relu'))\nmodel_6d2.add(Dropout(0.5))\nmodel_6d2.add(Dense(5, activation='softmax'))\n\nmodel_6d2.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Nadam(),\n              metrics=['accuracy'])\n\nmodel_6d2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_6d2.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('Three-layer CNN Performance with 0.5 dropout with Nadam optimizer and softmax activation', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fine tuning best model"},{"metadata":{},"cell_type":"markdown","source":"## Modifying kernel size"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_k33 = Sequential()\n\nmodel_k33.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_k33.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_k33.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_k33.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_k33.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_k33.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_k33.add(Conv2D(filters = 128, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_k33.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_k33.add(Flatten())\nmodel_k33.add(Dense(512, activation='relu'))\nmodel_k33.add(Dense(5, activation='softmax'))\n\n\nmodel_k33.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_k33.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_k33.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance with 4 convolution layers, 3x3 kernel size', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_k55 = Sequential()\n\nmodel_k55.add(Conv2D(filters = 16, kernel_size=(5, 5), activation='relu', \n                 input_shape=input_shape))\nmodel_k55.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_k55.add(Conv2D(filters = 32, kernel_size=(5, 5), activation='relu', \n                 input_shape=input_shape))\nmodel_k55.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_k55.add(Conv2D(filters = 64, kernel_size=(5, 5), activation='relu', \n                 input_shape=input_shape))\nmodel_k55.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_k55.add(Conv2D(filters = 128, kernel_size=(5, 5), activation='relu', \n                 input_shape=input_shape))\nmodel_k55.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_k55.add(Flatten())\nmodel_k55.add(Dense(512, activation='relu'))\nmodel_k55.add(Dense(5, activation='softmax'))\n\n\nmodel_k55.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_k55.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_k55.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance with 4 convolution layers, 5x5 kernel size', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_k77 = Sequential()\n\nmodel_k77.add(Conv2D(filters = 16, kernel_size=(7, 7), activation='relu', \n                 input_shape=input_shape))\nmodel_k77.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_k77.add(Conv2D(filters = 32, kernel_size=(7, 7), activation='relu', \n                 input_shape=input_shape))\nmodel_k77.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_k77.add(Conv2D(filters = 64, kernel_size=(7, 7), activation='relu', \n                 input_shape=input_shape))\nmodel_k77.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_k77.add(Conv2D(filters = 128, kernel_size=(7, 7), activation='relu', \n                 input_shape=input_shape))\nmodel_k77.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_k77.add(Flatten())\nmodel_k77.add(Dense(512, activation='relu'))\nmodel_k77.add(Dense(5, activation='softmax'))\n\n\nmodel_k77.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_k77.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_k77.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance with 4 convolution layers, 7x7 kernel size', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Using additional epochs"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 30\nnum_classes = 5\nepochs = 50\ninput_shape = (224, 224, 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_e50 = Sequential()\n\nmodel_e50.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_e50.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_e50.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_e50.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_e50.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_e50.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_e50.add(Conv2D(filters = 128, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_e50.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_e50.add(Flatten())\nmodel_e50.add(Dense(512, activation='relu'))\nmodel_e50.add(Dense(5, activation='softmax'))\n\n\nmodel_e50.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_e50.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_e50.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance with 4 convolution layers, 3x3 kernel size, 50 epochs', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,50))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 50, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 50, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Using batch normalization"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_bn = Sequential()\n\nmodel_bn.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_bn.add(BatchNormalization())\nmodel_bn.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_bn.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_bn.add(BatchNormalization())\nmodel_bn.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_bn.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_bn.add(BatchNormalization())\nmodel_bn.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_bn.add(Conv2D(filters = 128, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_bn.add(BatchNormalization())\nmodel_bn.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_bn.add(Flatten())\nmodel_bn.add(Dense(512, activation='relu'))\nmodel_bn.add(BatchNormalization())\nmodel_bn.add(Dense(5, activation='softmax'))\n\n\nmodel_bn.compile(loss='categorical_crossentropy',\n              #optimizer=optimizers.Adam(),\n              #optimizer=optimizers.Adam(learning_rate=0.01, beta_1=0.9, beta_2=0.999, amsgrad=False),\n              optimizer=optimizers.Adam(learning_rate=0.01, beta_1=0.9, beta_2=0.999, amsgrad=True),\n              metrics=['accuracy'])\n\nmodel_bn.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_bn.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance with 4 convolution layers with batch normalization and learning rate = 0.01, amsgrad=True', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Optimizing dropout"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_drop = Sequential()\n\nmodel_drop.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_drop.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_drop.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_drop.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_drop.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_drop.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_drop.add(Conv2D(filters = 128, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_drop.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_drop.add(Flatten())\nmodel_drop.add(Dense(512, activation='relu'))\nmodel_drop.add(Dropout(0.1))\n#model_drop.add(Dropout(0.3))\n#model_drop.add(Dropout(0.5))\nmodel_drop.add(Dense(5, activation='softmax'))\n\n\nmodel_drop.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\n#model_drop.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_drop.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance with 4 convolution layers with dropout 0.1', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Batch normalization and dropout 0.5"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_bdrop = Sequential()\n\nmodel_bdrop.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_bdrop.add(BatchNormalization())\nmodel_bdrop.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_bdrop.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_bdrop.add(BatchNormalization())\nmodel_bdrop.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_bdrop.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_bdrop.add(BatchNormalization())\nmodel_bdrop.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_bdrop.add(Conv2D(filters = 128, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_bdrop.add(BatchNormalization())\nmodel_bdrop.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_bdrop.add(Flatten())\nmodel_bdrop.add(Dense(512, activation='relu'))\nmodel_bdrop.add(Dropout(0.5))\nmodel_bdrop.add(Dense(5, activation='softmax'))\n\n\nmodel_bdrop.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_bdrop.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_bdrop.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance with 4 convolution layers with batch normalization and dropout 0.5', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Top performing model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_base = Sequential()\n\nmodel_base.add(Conv2D(filters = 16, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_base.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_base.add(Conv2D(filters = 32, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_base.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_base.add(Conv2D(filters = 64, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_base.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_base.add(Conv2D(filters = 128, kernel_size=(3, 3), activation='relu', \n                 input_shape=input_shape))\nmodel_base.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_base.add(Flatten())\nmodel_base.add(Dense(512, activation='relu'))\nmodel_base.add(Dropout(0.5))\nmodel_base.add(Dense(5, activation='softmax'))\n\n\nmodel_base.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(),\n              metrics=['accuracy'])\n\nmodel_base.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_base.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 30\nnum_classes = 5\nepochs = 20\ninput_shape = (224, 224, 3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transfer learning "},{"metadata":{},"cell_type":"markdown","source":"## VGG16"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications import VGG16\n\nvgg16_base = VGG16(weights = 'imagenet', include_top = False, input_shape = input_shape)\n\nvgg16_base.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import models\nfrom keras import layers\nmodel = models.Sequential()\nmodel.add(vgg16_base)\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(512, activation='relu'))\n#model.add(BatchNormalization())\nmodel.add(Dropout(0.5))\nmodel.add(layers.Dense(5, activation='softmax'))\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import models\nfrom keras import layers\nvgg16_model = models.Sequential()\nvgg16_model.add(vgg16_base)\nvgg16_model.add(layers.GlobalAveragePooling2D())\nvgg16_model.add(Dropout(0.5))\nvgg16_model.add(layers.Dense(5, activation='softmax'))\n\nvgg16_model.summary()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('This is the number of trainable weights '\n'before freezing the conv base:', len(vgg16_model.trainable_weights))\n\nvgg16_base.trainable = False\nprint('This is the number of trainable weights '\n'after freezing the conv base:', len(vgg16_model.trainable_weights))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vgg16_model.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(learning_rate=0.00005),\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = vgg16_model.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance VGG16 pretrained weights plus global average pooling', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vgg16_base.trainable = True\nset_trainable = False\nfor layer in vgg16_base.layers:\n    if layer.name == 'block5_conv1':\n        set_trainable = True\n    if set_trainable:\n        layer.trainable = True\n    else:\n        layer.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vgg16_base.trainable = True\nset_trainable = False\nfor layer in vgg16_base.layers:\n    if layer.name == 'block4_conv1':\n        set_trainable = True\n    if set_trainable:\n        layer.trainable = True\n    else:\n        layer.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vgg16_model.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(learning_rate=0.00005),\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 30\nnum_classes = 5\nepochs = 20\ninput_shape = (224, 224, 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = vgg16_model.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance VGG16 pretrained weights with fine-tuning (2nd block) plus global average pooling', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,50))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 50, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 50, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## ResNet50"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 30\nnum_classes = 5\nepochs = 30\ninput_shape = (224, 224, 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications import ResNet50\n\nresnet50_base = ResNet50(weights = 'imagenet', include_top = False, input_shape = input_shape)\n\nresnet50_base.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import models\nfrom keras import layers\nresnet50_60_model = models.Sequential()\nresnet50_60_model.add(resnet50_base)\nresnet50_60_model.add(layers.Flatten())\nresnet50_60_model.add(layers.Dense(512, activation='relu'))\n#model.add(BatchNormalization())\nresnet50_60_model.add(layers.Dropout(0.6)) \nresnet50_60_model.add(layers.Dense(5, activation='softmax'))\n\nresnet50_60_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import models\nfrom keras import layers\nresnet50_model = models.Sequential()\nresnet50_model.add(resnet50_base)\nresnet50_model.add(layers.GlobalAveragePooling2D())\nresnet50_model.add(Dropout(0.5))\nresnet50_model.add(layers.Dense(5, activation='softmax'))\n\nresnet50_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('This is the number of trainable weights '\n'before freezing the conv base:', len(resnet50_60_model.trainable_weights))\n\nresnet50_base.trainable = False\nprint('This is the number of trainable weights '\n'after freezing the conv base:', len(resnet50_60_model.trainable_weights))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50_60_model.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(learning_rate=0.00005),\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = resnet50_60_model.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance ResNet50 pretrained weights plus global average pooling', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"conv_base.trainable = True\nset_trainable = False\nfor layer in conv_base.layers:\n    if layer.name == 'res5a_branch2a':\n        set_trainable = True\n    if set_trainable:\n        layer.trainable = True\n    else:\n        layer.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50_base.trainable = True\nset_trainable = False\nfor layer in resnet50_base.layers:\n    if layer.name == 'res4a_branch2a':\n        set_trainable = True\n    if set_trainable:\n        layer.trainable = True\n    else:\n        layer.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50_60_model.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(learning_rate=0.00005),\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 30\nnum_classes = 5\nepochs = 50\ninput_shape = (224, 224, 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = resnet50_60_model.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance ResNet50 pretrained weights with fine-tuning (2nd block) plus dropout 0.6', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,50))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 50, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 50, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import load_model\n\nresnet50_60_model.save('/kaggle/working/resnet50_model.h5') ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Xception"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 30\nnum_classes = 5\nepochs = 20\ninput_shape = (224, 224, 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications import Xception\n\nxception_base = Xception(weights = 'imagenet', include_top = False, input_shape = input_shape)\n\nxception_base.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import models\nfrom keras import layers\nmodel = models.Sequential()\nmodel.add(conv_base)\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(512, activation='relu'))\n#model.add(BatchNormalization())\nmodel.add(layers.Dropout(0.5)) \nmodel.add(layers.Dense(5, activation='softmax'))\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import models\nfrom keras import layers\nxception_model = models.Sequential()\nxception_model.add(xception_base)\nxception_model.add(layers.GlobalAveragePooling2D())\nxception_model.add(Dropout(0.5))\nxception_model.add(layers.Dense(5, activation='softmax'))\n\nxception_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('This is the number of trainable weights '\n'before freezing the conv base:', len(xception_model.trainable_weights))\n\nxception_base.trainable = False\nprint('This is the number of trainable weights '\n'after freezing the conv base:', len(xception_model.trainable_weights))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xception_model.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(learning_rate=0.00005),\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = xception_model.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance Xception pretrained weights plus global average pooling', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,20))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 20, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 20, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"conv_base.trainable = True\nset_trainable = False\nfor layer in conv_base.layers:\n    if layer.name == 'block14_sepconv1':\n        set_trainable = True\n    if set_trainable:\n        layer.trainable = True\n    else:\n        layer.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xception_base.trainable = True\nset_trainable = False\nfor layer in xception_base.layers:\n    if layer.name == 'block13_sepconv1':\n        set_trainable = True\n    if set_trainable:\n        layer.trainable = True\n    else:\n        layer.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xception_model.compile(loss='categorical_crossentropy',\n              optimizer=optimizers.Adam(learning_rate=0.00005),\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 30\nnum_classes = 5\nepochs = 50\ninput_shape = (224, 224, 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = xception_model.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / batch_size,\n    epochs=epochs,\n    validation_data=(x_val, y_val),\n    callbacks=[#EarlyStopping(monitor = 'accuracy', patience=5, restore_best_weights=True),\n              ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4, verbose=0, mode='auto', min_delta=0.001, cooldown=0, min_lr=0)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('CNN Performance Xception pretrained weights and fine-tuning (2nd block) plus global average pooling', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nepoch_list = list(range(0,50))\nax1.plot(epoch_list, history.history['accuracy'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_accuracy'], label='Validation Accuracy')\nax1.set_xticks(np.arange(0, 50, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(0, 50, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Saved models"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import load_model\n\nvgg16_model.save('/kaggle/working/vgg16_model.h5')  \nresnet50_model.save('/kaggle/working/resnet50_model.h5')\nxception_model.save('/kaggle/working/xception_model.h5')\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualizing Filters & Feature Maps"},{"metadata":{},"cell_type":"markdown","source":"## Filters"},{"metadata":{"trusted":true},"cell_type":"code","source":"# cannot easily visualize filters lower down\nfrom keras.applications.vgg16 import VGG16\nfrom matplotlib import pyplot\n# load the model\nmodel = VGG16()\n# retrieve weights from the second hidden layer\nfilters, biases = model.layers[1].get_weights()\n# normalize filter values to 0-1 so we can visualize them\nf_min, f_max = filters.min(), filters.max()\nfilters = (filters - f_min) / (f_max - f_min)\n# plot first few filters\nn_filters, ix = 6, 1\nfor i in range(n_filters):\n    # get the filter\n    f = filters[:, :, :, i]\n    # plot each channel separately\n    for j in range(3):\n        # specify subplot and turn of axis\n        ax = pyplot.subplot(n_filters, 3, ix)\n        ax.set_xticks([])\n        ax.set_yticks([])\n        # plot filter channel in grayscale\n        pyplot.imshow(f[:, :, j], cmap='gray')\n        ix += 1\n# show the figure\npyplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"/kaggle/input/aptos2019-blindness-detection/test_images/351aba543dc8.png","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_array = np.array(Image.open('../input/aptos2019-blindness-detection/test_images/351aba543dc8.png'))\nplt.imshow(img_array)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# visualize feature maps output from each block in the vgg model\nfrom keras.applications.vgg16 import VGG16\nfrom keras.applications.vgg16 import preprocess_input\nfrom keras.preprocessing.image import load_img\nfrom keras.preprocessing.image import img_to_array\nfrom keras.models import Model\nfrom matplotlib import pyplot\nfrom numpy import expand_dims\n# load the model\nmodel = VGG16()\n# redefine model to output right after the first hidden layer\nixs = [2, 5, 9, 13, 17]\noutputs = [model.layers[i].output for i in ixs]\nmodel = Model(inputs=model.inputs, outputs=outputs)\n# load the image with the required shape\nimg = load_img('../input/aptos2019-blindness-detection/test_images/351aba543dc8.png', target_size=(224, 224))\n# convert the image to an array\nimg = img_to_array(img)\n# expand dimensions so that it represents a single 'sample'\nimg = expand_dims(img, axis=0)\n# prepare the image (e.g. scale pixel values for the vgg)\nimg = preprocess_input(img)\n# get feature map for first hidden layer\nfeature_maps = model.predict(img)\n# plot the output from each block\nsquare = 4\nfor fmap in feature_maps:\n    # plot all 16 maps in an 4x4 squares\n    ix = 1\n    for _ in range(square):\n        for _ in range(square):\n            # specify subplot and turn of axis\n            # fig, ax = plt.subplots(figsize=(30, 30)) #make images larger for presentation\n            ax = pyplot.subplot(square, square, ix)\n            ax.set_xticks([])\n            ax.set_yticks([])\n            # plot filter channel in grayscale\n            pyplot.imshow(fmap[0, :, :, ix-1], cmap='bone')\n            ix += 1\n    # show the figure\n    pyplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport itertools\n\n# Plot the confusion matrix \n\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n\n# Predict the values from the validation dataset\nY_pred = model_base.predict(x_val)\n# Convert predictions classes to one hot vectors \nY_pred_classes = np.argmax(Y_pred,axis = 1) \n# Convert validation observations to one hot vectors\nY_true = np.argmax(y_val,axis = 1) \n# compute the confusion matrix\nconfusion_mtx = confusion_matrix(Y_true, Y_pred_classes) \n# plot the confusion matrix\nplot_confusion_matrix(confusion_mtx, classes = range(5)) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display a sample of error results \n\n# Errors are difference between predicted labels and true labels\nerrors = (Y_pred_classes - Y_true != 0)\n\nY_pred_classes_errors = Y_pred_classes[errors]\nY_pred_errors = Y_pred[errors]\nY_true_errors = Y_true[errors]\nX_val_errors = x_val[errors]\n\ndef display_errors(errors_index,img_errors,pred_errors, obs_errors):\n    \"\"\" This function shows 10 images (2 rows x 5 columns) with their predicted and real labels\"\"\"\n    n = 0\n    nrows = 2\n    ncols = 5\n    fig, ax = plt.subplots(nrows,ncols,sharex=True,sharey=True)\n    fig.set_figheight(15)\n    fig.set_figwidth(15)\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[n]\n            ax[row,col].imshow((img_errors[error]).reshape((224,224,3)))\n            ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(pred_errors[error],obs_errors[error]))\n            n += 1\n\n# Probabilities of the wrong predicted numbers\nY_pred_errors_prob = np.max(Y_pred_errors,axis = 1)\n\n# Predicted probabilities of the true values in the error set\ntrue_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))\n\n# Difference between the probability of the predicted label and the true label\ndelta_pred_true_errors = Y_pred_errors_prob - true_prob_errors\n\n# Sorted list of the delta prob errors\nsorted_delta_errors = np.argsort(delta_pred_true_errors)\n\n# Top 10 errors \nmost_important_errors = sorted_delta_errors[-10:]\n\n# Show the top 10 errors\ndisplay_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}