{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport cv2 as cv\nfrom numpy.random import seed\nseed(45)\nimport pickle\n\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom glob import glob \nfrom tensorflow.keras import layers\n\n%matplotlib inline","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-11-25T16:57:49.326095Z","iopub.execute_input":"2021-11-25T16:57:49.326394Z","iopub.status.idle":"2021-11-25T16:57:49.337989Z","shell.execute_reply.started":"2021-11-25T16:57:49.326346Z","shell.execute_reply":"2021-11-25T16:57:49.33723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Helper Functions\nThe functions below are used for merging the contents of Keras history objects, and for displaying training curves. We will use these after each training run.","metadata":{}},{"cell_type":"code","source":"def merge_history(hlist):\n    history = {}\n    for k in hlist[0].history.keys():\n        history[k] = sum([h.history[k] for h in hlist], [])\n    return history\n\ndef vis_training(h, start=1):\n    epoch_range = range(start, len(h['loss'])+1)\n    s = slice(start-1, None)\n\n    plt.figure(figsize=[14,4])\n\n    n = int(len(h.keys()) / 2)\n\n    for i in range(n):\n        k = list(h.keys())[i]\n        plt.subplot(1,n,i+1)\n        plt.plot(epoch_range, h[k][s], label='Training')\n        plt.plot(epoch_range, h['val_' + k][s], label='Validation')\n        plt.xlabel('Epoch'); plt.ylabel(k); plt.title(k)\n        plt.grid()\n        plt.legend()\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:50.158354Z","iopub.execute_input":"2021-11-25T16:52:50.158634Z","iopub.status.idle":"2021-11-25T16:52:50.169668Z","shell.execute_reply.started":"2021-11-25T16:52:50.158598Z","shell.execute_reply":"2021-11-25T16:52:50.168858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dirname = '/kaggle/input/histopathologic-cancer-detection/train'","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:50.171206Z","iopub.execute_input":"2021-11-25T16:52:50.171709Z","iopub.status.idle":"2021-11-25T16:52:50.178542Z","shell.execute_reply.started":"2021-11-25T16:52:50.17165Z","shell.execute_reply":"2021-11-25T16:52:50.177688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset exploration","metadata":{}},{"cell_type":"code","source":"train_labels = pd.read_csv('/kaggle/input/histopathologic-cancer-detection/train_labels.csv')\ntrain_labels.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:50.181889Z","iopub.execute_input":"2021-11-25T16:52:50.182106Z","iopub.status.idle":"2021-11-25T16:52:50.475199Z","shell.execute_reply.started":"2021-11-25T16:52:50.182075Z","shell.execute_reply":"2021-11-25T16:52:50.474547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Label distribution","metadata":{}},{"cell_type":"code","source":"train_labels['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:50.478435Z","iopub.execute_input":"2021-11-25T16:52:50.478783Z","iopub.status.idle":"2021-11-25T16:52:50.490783Z","shell.execute_reply.started":"2021-11-25T16:52:50.478754Z","shell.execute_reply":"2021-11-25T16:52:50.489947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display a DataFrame showing the proportion of observations with each \n# possible of the target variable (which is label). \n(train_labels.label.value_counts() / len(train_labels)).to_frame()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:50.494045Z","iopub.execute_input":"2021-11-25T16:52:50.49455Z","iopub.status.idle":"2021-11-25T16:52:50.604396Z","shell.execute_reply.started":"2021-11-25T16:52:50.494517Z","shell.execute_reply":"2021-11-25T16:52:50.603671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:50.605764Z","iopub.execute_input":"2021-11-25T16:52:50.60623Z","iopub.status.idle":"2021-11-25T16:52:50.63534Z","shell.execute_reply.started":"2021-11-25T16:52:50.606196Z","shell.execute_reply":"2021-11-25T16:52:50.634633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data is not entirely balanced, there is more negative samples than positive, by about 30 percent","metadata":{}},{"cell_type":"markdown","source":"# View Sample Images","metadata":{}},{"cell_type":"code","source":"positive_samples = train_labels.loc[train_labels['label'] == 1].sample(4)\nnegative_samples = train_labels.loc[train_labels['label'] == 0].sample(4)\npositive_images = []\nnegative_images = []\nfor sample in positive_samples['id']:\n    path = os.path.join(train_dirname, sample+'.tif')\n    img = cv.imread(path)\n    positive_images.append(img)\n        \nfor sample in negative_samples['id']:\n    path = os.path.join(train_dirname, sample+'.tif')\n    img = cv.imread(path)\n    negative_images.append(img)\n\nfig,axis = plt.subplots(2,4,figsize=(20,8))\nfig.suptitle('Dataset samples presentation plot',fontsize=20)\nfor i,img in enumerate(positive_images):\n    axis[0,i].imshow(img)\n    rect = patches.Rectangle((32,32),32,32,linewidth=4,edgecolor='g',facecolor='none', linestyle=':', capstyle='round')\n    axis[0,i].add_patch(rect)\naxis[0,0].set_ylabel('Positive samples', size='large')\nfor i,img in enumerate(negative_images):\n    axis[1,i].imshow(img)\n    rect = patches.Rectangle((32,32),32,32,linewidth=4,edgecolor='r',facecolor='none', linestyle=':', capstyle='round')\n    axis[1,i].add_patch(rect)\naxis[1,0].set_ylabel('Negative samples', size='large')\n    ","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:50.63675Z","iopub.execute_input":"2021-11-25T16:52:50.637222Z","iopub.status.idle":"2021-11-25T16:52:51.788092Z","shell.execute_reply.started":"2021-11-25T16:52:50.637189Z","shell.execute_reply":"2021-11-25T16:52:51.787163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Splitting dataset","metadata":{}},{"cell_type":"markdown","source":"# Setting up learning constants","metadata":{}},{"cell_type":"code","source":"IMG_SIZE = 96\nIMG_CHANNELS = 3\n#TRAIN_SIZE=40000\nTRAIN_SIZE = 10000\nBATCH_SIZE = 64\nEPOCHS = 30","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:51.789272Z","iopub.execute_input":"2021-11-25T16:52:51.789523Z","iopub.status.idle":"2021-11-25T16:52:51.79399Z","shell.execute_reply.started":"2021-11-25T16:52:51.789494Z","shell.execute_reply":"2021-11-25T16:52:51.793283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Balancing the dataset","metadata":{}},{"cell_type":"code","source":"train_neg = train_labels[train_labels['label']==0].sample(TRAIN_SIZE,random_state=45)\ntrain_pos = train_labels[train_labels['label']==1].sample(TRAIN_SIZE,random_state=45)\n\ntrain_data = pd.concat([train_neg, train_pos], axis=0).reset_index(drop=True)\n\ntrain_data = shuffle(train_data)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:51.795435Z","iopub.execute_input":"2021-11-25T16:52:51.79588Z","iopub.status.idle":"2021-11-25T16:52:51.847739Z","shell.execute_reply.started":"2021-11-25T16:52:51.795847Z","shell.execute_reply":"2021-11-25T16:52:51.846986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:51.849329Z","iopub.execute_input":"2021-11-25T16:52:51.849769Z","iopub.status.idle":"2021-11-25T16:52:51.860251Z","shell.execute_reply.started":"2021-11-25T16:52:51.849735Z","shell.execute_reply":"2021-11-25T16:52:51.859687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def append_ext(fn):\n    return fn+\".tif\"","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:51.862977Z","iopub.execute_input":"2021-11-25T16:52:51.863177Z","iopub.status.idle":"2021-11-25T16:52:51.867653Z","shell.execute_reply.started":"2021-11-25T16:52:51.863155Z","shell.execute_reply":"2021-11-25T16:52:51.866782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Splitting the dataset","metadata":{}},{"cell_type":"code","source":"#y = train_data['label']\n#train_df, val_df = train_test_split(train_data, test_size=0.3, random_state=45, stratify=y)\n#y = val_df['label']\n#val_df, test_df = train_test_split(val_df, test_size=0.5, random_state=45, stratify=y)\n#print(train_df.shape)\n#print(val_df.shape)\n#print(test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:51.868968Z","iopub.execute_input":"2021-11-25T16:52:51.869513Z","iopub.status.idle":"2021-11-25T16:52:51.874914Z","shell.execute_reply.started":"2021-11-25T16:52:51.869478Z","shell.execute_reply":"2021-11-25T16:52:51.874217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train_data['label']\ntrain_df, valid_df = train_test_split(train_data, test_size=0.2, random_state=45, stratify=y)\n\nprint(train_df.shape)\nprint(valid_df.shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:51.876453Z","iopub.execute_input":"2021-11-25T16:52:51.876767Z","iopub.status.idle":"2021-11-25T16:52:51.942194Z","shell.execute_reply.started":"2021-11-25T16:52:51.876731Z","shell.execute_reply":"2021-11-25T16:52:51.941415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['id'] = train_df['id'].apply(append_ext)\nvalid_df['id'] = valid_df['id'].apply(append_ext)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:51.943767Z","iopub.execute_input":"2021-11-25T16:52:51.94403Z","iopub.status.idle":"2021-11-25T16:52:51.994731Z","shell.execute_reply.started":"2021-11-25T16:52:51.943998Z","shell.execute_reply":"2021-11-25T16:52:51.994122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image generators for the simple CNN model","metadata":{}},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale=1/255)\nvalid_datagen = ImageDataGenerator(rescale=1/255)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:51.996082Z","iopub.execute_input":"2021-11-25T16:52:51.996629Z","iopub.status.idle":"2021-11-25T16:52:52.001861Z","shell.execute_reply.started":"2021-11-25T16:52:51.996592Z","shell.execute_reply":"2021-11-25T16:52:52.000906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 64\ntrain_path = '../input/histopathologic-cancer-detection/train'\ntrain_df['label'] = train_df['label'].astype(str)\nvalid_df['label'] = valid_df['label'].astype(str)\n\ntrain_loader = train_datagen.flow_from_dataframe(\n    dataframe = train_df,\n    directory = train_path,\n    x_col = 'id',\n    y_col = 'label',\n    batch_size = BATCH_SIZE,\n    seed = 1,\n    shuffle = True,\n    class_mode = 'categorical',\n    target_size = (64,64)\n)\n\nvalid_loader = valid_datagen.flow_from_dataframe(\n    dataframe = valid_df,\n    directory = train_path,\n    x_col = 'id',\n    y_col = 'label',\n    batch_size = BATCH_SIZE,\n    seed = 1,\n    shuffle = True,\n    class_mode = 'categorical',\n    target_size = (64,64)\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:52:52.003272Z","iopub.execute_input":"2021-11-25T16:52:52.003803Z","iopub.status.idle":"2021-11-25T16:53:55.975703Z","shell.execute_reply.started":"2021-11-25T16:52:52.003768Z","shell.execute_reply":"2021-11-25T16:53:55.974906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TR_STEPS = len(train_loader)\nVA_STEPS = len(valid_loader)\n\nprint(TR_STEPS)\nprint(VA_STEPS)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:53:55.978006Z","iopub.execute_input":"2021-11-25T16:53:55.978291Z","iopub.status.idle":"2021-11-25T16:53:55.983208Z","shell.execute_reply.started":"2021-11-25T16:53:55.978257Z","shell.execute_reply":"2021-11-25T16:53:55.982475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Network","metadata":{}},{"cell_type":"markdown","source":"In this section, we will construct our neural network. For feature extraction, we will use the VGG16 model, as trained on the ImageNet dataset.\n\nIn the cell below, we will load the pretrained VGG16 model into a variable named base_model. We will set include_top=False to indicate that we only wish to use the convolutional blocks that appear before the Flatten() layer. We will not include the dense layers composing the classifier at the top of the network. Instead, we will design and train our own classifier.\n\nWe set the input_shape parameter to indicate the shape of the images that we will be feeding into the network.\n\nFinally, we set the trainable parameter of the model to False. This tells Keras that we do not wish to update the weights in the base layer during training. We only wish to train the new classifier that we will design.","metadata":{}},{"cell_type":"code","source":"base_model = tf.keras.applications.VGG16(input_shape=(64,64,3),\n                                         include_top=False,\n                                         weights='imagenet')\n\nbase_model.trainable = False","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:53:55.984489Z","iopub.execute_input":"2021-11-25T16:53:55.984979Z","iopub.status.idle":"2021-11-25T16:54:00.524919Z","shell.execute_reply.started":"2021-11-25T16:53:55.984943Z","shell.execute_reply":"2021-11-25T16:54:00.524217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Before moving forward, let's take a look at the structure of our base model. Notice that it consists of 5 convolutional blocks, some of which contain 2 convolutional layers, and some of which contain 3. Also note that none of the weights in the model are trainable (since we have set them to not be).","metadata":{}},{"cell_type":"code","source":"base_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:54:00.528257Z","iopub.execute_input":"2021-11-25T16:54:00.528505Z","iopub.status.idle":"2021-11-25T16:54:00.543935Z","shell.execute_reply.started":"2021-11-25T16:54:00.528479Z","shell.execute_reply":"2021-11-25T16:54:00.543377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"VGG16 is one of many pretrained models that we could have used. Common choices include VGG16, VGG19, ResNet50, and InceptionV3. A full list of the pretrained models provided by Keras can be found here: Keras Applications\n\nWe are now ready to build a classifier for our neural network. In the cell below, we include base_model in the network as if were a single layer.","metadata":{}},{"cell_type":"code","source":"cnn = Sequential([\n    base_model,\n    \n    Flatten(),\n   \n    \n    Dense(256, activation='relu'),\n    BatchNormalization(),\n    Dropout(0.5),\n    #Dense(128, activation='relu'),\n    #Dropout(0.5),\n    #Dense(32, activation='relu'),\n    #Dropout(0.3),\n    #Dense(16, activation='relu'),\n    #Dropout(0.25),\n    #Dense(8, activation='relu'),\n    #Dropout(0.25),\n    #BatchNormalization(),\n    Dense(2, activation='softmax')\n])\n\ncnn.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:58:47.396133Z","iopub.execute_input":"2021-11-25T16:58:47.39646Z","iopub.status.idle":"2021-11-25T16:58:47.426186Z","shell.execute_reply.started":"2021-11-25T16:58:47.39643Z","shell.execute_reply":"2021-11-25T16:58:47.425152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Network","metadata":{}},{"cell_type":"code","source":"opt = tf.keras.optimizers.Adam(0.001)\ncnn.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', tf.keras.metrics.AUC()])","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:54:01.350849Z","iopub.status.idle":"2021-11-25T16:54:01.35144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training Run 1","metadata":{}},{"cell_type":"code","source":"%%time \n\nh1 = cnn.fit(\n    x = train_loader, \n    steps_per_epoch = TR_STEPS, \n    epochs = 30,\n    validation_data = valid_loader, \n    validation_steps = VA_STEPS, \n    verbose = 1\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:54:01.352551Z","iopub.status.idle":"2021-11-25T16:54:01.353114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = h1.history\nprint(history.keys())","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:54:01.354148Z","iopub.status.idle":"2021-11-25T16:54:01.35472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epoch_range = range(1, len(history['loss'])+1)\n\nplt.figure(figsize=[14,4])\nplt.subplot(1,3,1)\nplt.plot(epoch_range, history['loss'], label='Training')\nplt.plot(epoch_range, history['val_loss'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Loss'); plt.title('Loss')\nplt.legend()\nplt.subplot(1,3,2)\nplt.plot(epoch_range, history['accuracy'], label='Training')\nplt.plot(epoch_range, history['val_accuracy'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Accuracy'); plt.title('Accuracy')\nplt.legend()\nplt.subplot(1,3,3)\nplt.plot(epoch_range, history['auc'], label='Training')\nplt.plot(epoch_range, history['val_auc'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('AUC'); plt.title('AUC')\nplt.legend()\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:54:01.355784Z","iopub.status.idle":"2021-11-25T16:54:01.356334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Training Run 2\ntf.keras.backend.set_value(cnn.optimizer.learning_rate, 0.0001)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:54:01.357391Z","iopub.status.idle":"2021-11-25T16:54:01.35795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\nh2 = cnn.fit(\n    x = train_loader, \n    steps_per_epoch = TR_STEPS, \n    epochs = 30,\n    validation_data = valid_loader, \n    validation_steps = VA_STEPS, \n    verbose = 1\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:54:01.359Z","iopub.status.idle":"2021-11-25T16:54:01.359566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in history.keys():\n    history[k] += h2.history[k]\n\nepoch_range = range(1, len(history['loss'])+1)\n\nplt.figure(figsize=[14,4])\nplt.subplot(1,3,1)\nplt.plot(epoch_range, history['loss'], label='Training')\nplt.plot(epoch_range, history['val_loss'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Loss'); plt.title('Loss')\nplt.legend()\nplt.subplot(1,3,2)\nplt.plot(epoch_range, history['accuracy'], label='Training')\nplt.plot(epoch_range, history['val_accuracy'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Accuracy'); plt.title('Accuracy')\nplt.legend()\nplt.subplot(1,3,3)\nplt.plot(epoch_range, history['auc'], label='Training')\nplt.plot(epoch_range, history['val_auc'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('AUC'); plt.title('AUC')\nplt.legend()\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:54:01.360613Z","iopub.status.idle":"2021-11-25T16:54:01.361188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Training Run 3\ntf.keras.backend.set_value(cnn.optimizer.learning_rate, 0.00001)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:54:01.362223Z","iopub.status.idle":"2021-11-25T16:54:01.362806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\nh3 = cnn.fit(\n    x = train_loader, \n    steps_per_epoch = TR_STEPS, \n    epochs = 30,\n    validation_data = valid_loader, \n    validation_steps = VA_STEPS, \n    verbose = 1\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:54:01.363838Z","iopub.status.idle":"2021-11-25T16:54:01.364401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in history.keys():\n    history[k] += h3.history[k]\n\nepoch_range = range(1, len(history['loss'])+1)\n\nplt.figure(figsize=[14,4])\nplt.subplot(1,3,1)\nplt.plot(epoch_range, history['loss'], label='Training')\nplt.plot(epoch_range, history['val_loss'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Loss'); plt.title('Loss')\nplt.legend()\nplt.subplot(1,3,2)\nplt.plot(epoch_range, history['accuracy'], label='Training')\nplt.plot(epoch_range, history['val_accuracy'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Accuracy'); plt.title('Accuracy')\nplt.legend()\nplt.subplot(1,3,3)\nplt.plot(epoch_range, history['auc'], label='Training')\nplt.plot(epoch_range, history['val_auc'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('AUC'); plt.title('AUC')\nplt.legend()\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:54:01.365473Z","iopub.status.idle":"2021-11-25T16:54:01.366031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Save Model and History","metadata":{}},{"cell_type":"code","source":"cnn.save('cancer_detection_model_v10.h5')\npickle.dump(history, open(f'cancer_detection_history_v10.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:54:01.367065Z","iopub.status.idle":"2021-11-25T16:54:01.36763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"### Create image generators for MobileNetV2","metadata":{}},{"cell_type":"markdown","source":"## ","metadata":{}}]}