{"cells":[{"metadata":{"_uuid":"cf507007618542bdc0f4aca885e5855d0dec7e18"},"cell_type":"markdown","source":"# **Histopathologic Cancer Detection**\n"},{"metadata":{"_uuid":"9891a3f0b3dc9d4e53042330e57f44eeae6b1685"},"cell_type":"markdown","source":"**The images are labeled as 0 or 1, where 0 = No Tumor Tissue and 1 = Has Tumor Tissue(s)**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport cv2\nimport seaborn as sns\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\nimport itertools\nimport shutil\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e37caaed3fcf7ce6022845347d7ff01eb7550f3f"},"cell_type":"markdown","source":"# **Exploratory Data Analysis**"},{"metadata":{"_uuid":"907872e78e7c40c8ee288e3879e42f1f2ed5f8e4"},"cell_type":"markdown","source":"### Total Samples Available"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Total Samples Available\nprint('Train Images = ',len(os.listdir('../input/histopathologic-cancer-detection/train')))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8ea02846b55222461342cec080b44f1261804db4"},"cell_type":"markdown","source":"### Create a DataFrame of all Train Image Labels"},{"metadata":{"trusted":true,"_uuid":"b862721d2db8b1aa633e19ae38cc30f8dc62d1d7"},"cell_type":"code","source":"df = pd.read_csv('../input/histopathologic-cancer-detection/train_labels.csv')\nprint('Shape of DataFrame',df.shape)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"88fdfddefa4d6061df1dd24c6b37ae369e0f81b5"},"cell_type":"markdown","source":"### **Visualize some Train Images**"},{"metadata":{"trusted":true,"_uuid":"890ed0a67736ea238d6a5ed1b46becfba43fd178"},"cell_type":"code","source":"TRAIN_DIR = '../input/histopathologic-cancer-detection/train/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6017cae09d2df8655b67d96d816b8f60db3854a8"},"cell_type":"code","source":"fig = plt.figure(figsize = (20,8))\nindex = 1\nfor i in np.random.randint(low = 0, high = df.shape[0], size = 10):\n    file = TRAIN_DIR + df.iloc[i]['id'] + '.tif'\n    img = cv2.imread(file)\n    ax = fig.add_subplot(2, 5, index)\n    ax.imshow(img, cmap = 'gray')\n    index = index + 1\n    color = ['green' if df.iloc[i].label == 1 else 'red'][0]\n    ax.set_title(df.iloc[i].label, fontsize = 18, color = color)\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ddfbe3be86859a68acb71b0cc631fe5702552c1"},"cell_type":"markdown","source":"### See the distribution of Train Labels"},{"metadata":{"trusted":true,"_uuid":"adab435e9f6481bcd2009f847f64e89eab0b44fd"},"cell_type":"code","source":"# removing this image because it caused a training error previously\ndf[df['id'] != 'dd6dfed324f9fcb6f93f46f32fc800f2ec196be2']\n\n# removing this image because it's black\ndf[df['id'] != '9369c7278ec8bcc6c880d99194de09fc2bd4efbe']\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"406180d03beea5afc5675e72150b0395db4b0829"},"cell_type":"code","source":"fig = plt.figure(figsize = (6,6)) \nax = sns.countplot(df.label).set_title('Label Counts', fontsize = 18)\nplt.annotate(df.label.value_counts()[0],\n            xy = (0,df.label.value_counts()[0] + 2000),\n            va = 'bottom',\n            ha = 'center',\n            fontsize = 12)\nplt.annotate(df.label.value_counts()[1],\n            xy = (1,df.label.value_counts()[1] + 2000),\n            va = 'bottom',\n            ha = 'center',\n            fontsize = 12)\nplt.ylim(0,150000)\nplt.ylabel('Count', fontsize = 16)\nplt.xlabel('Labels', fontsize = 16)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf872d7e100c23dc7256c88eca0353b754d56e8a"},"cell_type":"markdown","source":"Here the **Label-1** is **60%** and **Label-0** is **40%** of the whole train images. "},{"metadata":{"_uuid":"a28db6e854cf3345e28393ca4a42738178207555"},"cell_type":"markdown","source":"# **Feature Engineering**"},{"metadata":{"trusted":true,"_uuid":"b6cbe264f9225f0613a6587c3d8adcd9e3854cda"},"cell_type":"markdown","source":"### **Take 88K images from both categories**"},{"metadata":{"trusted":true,"_uuid":"67cb329de44b201e5190b81bc914669c80ac303a"},"cell_type":"code","source":"SAMPLE_SIZE = 88000\n# take a random sample of class 0 with size equal to num samples in class 1\ndf_0 = df[df['label'] == 0].sample(SAMPLE_SIZE, random_state = 0)\n# filter out class 1\ndf_1 = df[df['label'] == 1].sample(SAMPLE_SIZE, random_state = 0)\n\n# concat the dataframes\ndf_train = pd.concat([df_0, df_1], axis = 0).reset_index(drop = True)\n# shuffle\ndf_train = shuffle(df_train)\n\ndf_train['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"89d02968ae213d657d0fffcf548248932a8e023e"},"cell_type":"markdown","source":"### **Split into Train, Validation+Test Sets**"},{"metadata":{"trusted":true,"_uuid":"e303dcd33773b33e43e553ca831da31f6f7be263"},"cell_type":"code","source":"# train_test_split\n# stratify=y creates a balanced validation set.\ny = df_train['label']\n\ndf_train, df_val = train_test_split(df_train, test_size = (2/11), random_state = 0, stratify = y)\nprint(df_train.head())\nprint(df_val.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **Split into Validation and Test Sets**"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_val_test = df_val['label']\n\ndf_val, df_test = train_test_split(df_val, test_size = 0.5, random_state = 0, stratify = y_val_test)\nprint(df_val.head())\nprint(df_test.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_train['label'].value_counts())\nprint(df_val['label'].value_counts())\nprint(df_test['label'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ab3dea5e5f64c78e7670ad162f31ea591fd96a46"},"cell_type":"markdown","source":"### **Put the two types of images into two folder to help Keras ImageGenerator**"},{"metadata":{"_uuid":"69bbae64fa38dcc9961b0a545eb4cc9d3dbcbc9f"},"cell_type":"markdown","source":"**Creating Directory Structure**"},{"metadata":{"trusted":true,"_uuid":"6c47212e910f096dccb21e6603686218253c0b2d"},"cell_type":"code","source":"# Create a new directory\nbase_dir = 'base_dir'\nos.mkdir(base_dir)\n\n\n#Folder Structure\n\n'''\n    * base_dir\n        |-- train_dir\n            |-- 0   #No Tumor\n            |-- 1   #Has Tumor\n        |-- val_dir\n            |-- 0\n            |-- 1\n        |-- test_dir\n            |-- 0\n            |-- 1\n'''\n# create a path to 'base_dir' to which we will join the names of the new folders\n# train_dir\ntrain_dir = os.path.join(base_dir, 'train_dir')\nos.mkdir(train_dir)\n\n# val_dir\nval_dir = os.path.join(base_dir, 'val_dir')\nos.mkdir(val_dir)\n\n# test_dir\ntest_dir = os.path.join(base_dir, 'test_dir')\nos.mkdir(test_dir)\n\n# create new folders inside train_dir\nno_tumor = os.path.join(train_dir, '0')\nos.mkdir(no_tumor)\nhas_tumor = os.path.join(train_dir, '1')\nos.mkdir(has_tumor)\n\n\n# create new folders inside val_dir\nno_tumor = os.path.join(val_dir, '0')\nos.mkdir(no_tumor)\nhas_tumor = os.path.join(val_dir, '1')\nos.mkdir(has_tumor)\n\n# create new folders inside test_dir\nno_tumor = os.path.join(test_dir, '0')\nos.mkdir(no_tumor)\nhas_tumor = os.path.join(test_dir, '1')\nos.mkdir(has_tumor)\n\n\nprint(os.listdir('base_dir/train_dir'))\nprint(os.listdir('base_dir/val_dir'))\nprint(os.listdir('base_dir/test_dir'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0d26cfa79a1373d0eac3ee1f13a74f5b11b93df1"},"cell_type":"markdown","source":"**Transfer the respective images into their respective folders**"},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8ca025d9524110a511941f81bffc6a689426589"},"cell_type":"code","source":"# Set the id as the index in df_data\ndf.set_index('id', inplace=True)\n\n# Get a list of train and val images\ntrain_list = list(df_train['id'])\nval_list = list(df_val['id'])\ntest_list = list(df_test['id'])\n\n\n\n# Transfer the train images\n\nfor image in train_list:\n    \n    # the id in the csv file does not have the .tif extension therefore we add it here\n    file_name = image + '.tif'\n    # get the label for a certain image\n    target = df.loc[image,'label']\n    \n    # these must match the folder names\n    if target == 0:\n        label = '0'\n    elif target == 1:\n        label = '1'\n    \n    # source path to image\n    src = os.path.join('../input/histopathologic-cancer-detection/train', file_name)\n    # destination path to image\n    dest = os.path.join(train_dir, label, file_name)\n    # copy the image from the source to the destination\n    shutil.copyfile(src, dest)\n\n\n# Transfer the val images\n\nfor image in val_list:\n    \n    # the id in the csv file does not have the .tif extension therefore we add it here\n    file_name = image + '.tif'\n    # get the label for a certain image\n    target = df.loc[image,'label']\n    \n    # these must match the folder names\n    if target == 0:\n        label = '0'\n    elif target == 1:\n        label = '1'\n    \n\n    # source path to image\n    src = os.path.join('../input/histopathologic-cancer-detection/train', file_name)\n    # destination path to image\n    dest = os.path.join(val_dir, label, file_name)\n    # copy the image from the source to the destination\n    shutil.copyfile(src, dest)\n    \n# Transfer the test images\n\nfor image in test_list:\n    \n    # the id in the csv file does not have the .tif extension therefore we add it here\n    file_name = image + '.tif'\n    # get the label for a certain image\n    target = df.loc[image,'label']\n    \n    # these must match the folder names\n    if target == 0:\n        label = '0'\n    elif target == 1:\n        label = '1'\n    \n\n    # source path to image\n    src = os.path.join('../input/histopathologic-cancer-detection/train', file_name)\n    # destination path to image\n    dest = os.path.join(test_dir, label, file_name)\n    # copy the image from the source to the destination\n    shutil.copyfile(src, dest)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2cbc03cce7295ba3d74ff0e79471bfc09b8e72e3"},"cell_type":"code","source":"print(len(os.listdir('base_dir/train_dir/0')))\nprint(len(os.listdir('base_dir/train_dir/1')))\nprint(len(os.listdir('base_dir/val_dir/0')))\nprint(len(os.listdir('base_dir/val_dir/1')))\nprint(len(os.listdir('base_dir/test_dir/0')))\nprint(len(os.listdir('base_dir/test_dir/1')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79b25f0ddbd2b0e25ae068121adee9babaac24aa"},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nIMAGE_SIZE = 96\ntrain_path = 'base_dir/train_dir'\nvalid_path = 'base_dir/val_dir'\ntest_path = 'base_dir/test_dir'\n\nnum_train_samples = len(df_train)\nnum_val_samples = len(df_val)\ntrain_batch_size = 32 #10\nval_batch_size = 32 #10\n\n\ntrain_steps = np.ceil(num_train_samples / train_batch_size)\nval_steps = np.ceil(num_val_samples / val_batch_size)\n\n\ndatagen = ImageDataGenerator(rescale=1.0/255)\n\ntrain_gen = datagen.flow_from_directory(train_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=train_batch_size,\n                                        class_mode='categorical')\n\nval_gen = datagen.flow_from_directory(valid_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=val_batch_size,\n                                        class_mode='categorical')\n\n# Note: shuffle=False causes the test dataset to not be shuffled\ntest_gen = datagen.flow_from_directory(test_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=1,\n                                        class_mode='categorical',\n                                        shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"41dc501ceb24e5b1bdd25c745fd9318267959e7e"},"cell_type":"markdown","source":"# **Create our Model (CancerNet)** "},{"metadata":{"trusted":true,"_uuid":"460dc2303b867c65d8597cd90fab142ddbe340e5"},"cell_type":"code","source":"#Import Keras\nimport keras\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, Dropout, MaxPooling2D, Flatten, Dense\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers.convolutional import SeparableConv2D\nfrom keras.layers.core import Activation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04bec957f0d95b3effae51c16a989b507ff687d6","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class Net:\n    @staticmethod\n    def build(width, height, depth, classes):\n            \n            #initializa model\n            model = Sequential()\n            \n            inputShape = (height, width, depth)\n            \n            #Add First Layer CONV => ReLU => Pooling\n            model.add(Conv2D(filters = 32, kernel_size = (5,5), padding=\"same\", activation='relu', input_shape= inputShape))\n            model.add(Conv2D(filters = 32, kernel_size = (3,3), padding=\"same\", activation='relu'))\n            model.add(Conv2D(filters = 32, kernel_size = (3,3), padding=\"same\", activation='relu'))\n            model.add(MaxPooling2D(pool_size=(2, 2)))\n            model.add(Dropout(0.2))\n                      \n            #Add Second Layer CONV => ReLU => Pooling\n            model.add(Conv2D(filters = 64, kernel_size = (3,3), padding=\"same\", activation='relu'))\n            model.add(Conv2D(filters = 64, kernel_size = (3,3), padding=\"same\", activation='relu'))\n            model.add(Conv2D(filters = 64, kernel_size = (3,3), padding=\"same\", activation='relu'))\n            model.add(MaxPooling2D(pool_size=(2, 2)))\n            model.add(Dropout(0.2))\n            \n            #Add Third Layer CONV => ReLU => Pooling\n            model.add(Conv2D(filters = 128, kernel_size = (3,3), padding=\"same\", activation='relu'))\n            model.add(Conv2D(filters = 128, kernel_size = (3,3), padding=\"same\", activation='relu'))\n            model.add(Conv2D(filters = 128, kernel_size = (3,3), padding=\"same\", activation='relu'))\n            model.add(MaxPooling2D(pool_size=(2, 2)))\n            model.add(Dropout(0.25))\n            \n            \n            #FC => ReLU\n            model.add(Flatten())\n            model.add(Dense(units = 500, activation = 'relu'))\n            model.add(Dropout(0.2))\n            #FC => Output\n            model.add(Dense(classes, activation='softmax'))\n            \n            model.summary()\n            \n            return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e0dffe24441e9c2916b0963f545a2f06dce484b"},"cell_type":"code","source":"class CancerNet:\n    @staticmethod\n    def build(width, height, depth, classes):\n        \n        # initialize the model along with the input shape to be\n        # \"channels last\" and the channels dimension itself\n        model = Sequential()\n        inputShape = (height, width, depth)\n        chanDim = -1\n        \n        # CONV => RELU => POOL\n        model.add(SeparableConv2D(32, (3, 3), padding=\"same\",input_shape = inputShape))\n        model.add(Activation(\"relu\"))\n        model.add(BatchNormalization(axis=chanDim))\n        model.add(MaxPooling2D(pool_size=(2, 2)))\n        model.add(Dropout(0.25))\n\n        # (CONV => RELU => POOL) * 2\n        model.add(SeparableConv2D(64, (3, 3), padding=\"same\"))\n        model.add(Activation(\"relu\"))\n        model.add(BatchNormalization(axis=chanDim))\n        model.add(SeparableConv2D(64, (3, 3), padding=\"same\"))\n        model.add(Activation(\"relu\"))\n        model.add(BatchNormalization(axis=chanDim))\n        model.add(MaxPooling2D(pool_size=(2, 2)))\n        model.add(Dropout(0.25))\n\n        # (CONV => RELU => POOL) * 3\n        model.add(SeparableConv2D(128, (3, 3), padding=\"same\"))\n        model.add(Activation(\"relu\"))\n        model.add(BatchNormalization(axis=chanDim))\n        model.add(SeparableConv2D(128, (3, 3), padding=\"same\"))\n        model.add(Activation(\"relu\"))\n        model.add(BatchNormalization(axis=chanDim))\n        model.add(SeparableConv2D(128, (3, 3), padding=\"same\"))\n        model.add(Activation(\"relu\"))\n        model.add(BatchNormalization(axis=chanDim))\n        model.add(MaxPooling2D(pool_size=(2, 2)))\n        model.add(Dropout(0.25))\n        \n        # first (and only) set of FC => RELU layers\n        model.add(Flatten())\n        model.add(Dense(256))\n        model.add(Activation(\"relu\"))\n        model.add(BatchNormalization())\n        model.add(Dropout(0.2))\n\n        # softmax classifier\n        model.add(Dense(classes))\n        model.add(Activation(\"softmax\"))\n        \n        model.summary()\n\n        # return the constructed network architecture\n        return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"712787094f0b0dcf96c3734089eef074e3e72b6d"},"cell_type":"markdown","source":"**Specify optimizer and loss function**"},{"metadata":{"trusted":true,"_uuid":"76501728ae07bbfadf444856cb960415b0d1446b"},"cell_type":"code","source":"model = Net.build(width = 96, height = 96, depth = 3, classes = 2)\nfrom keras.optimizers import SGD, Adam, Adagrad\nmodel.compile(optimizer = Adam(lr=0.0001), loss = 'binary_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e73437b74bbe2883abe87b3e43cff8c3d962b56b"},"cell_type":"markdown","source":"# **Model Training**"},{"metadata":{"_uuid":"a674ddc01ff2b2bacf4c32dfe5e1840fc9c8463f"},"cell_type":"markdown","source":"### **Define LR Scheduler and Save Model Checkpoint on Maximum Validation Accuracy**"},{"metadata":{"trusted":true,"_uuid":"956b166c16f529c98512e3ee429980ad7e6271b6"},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\nfilepath = \"checkpoint.h5\"\ncheckpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose = 1, \n                             save_best_only = True, mode = 'max') #Save Best Epoch\n\nreduce_lr = ReduceLROnPlateau(monitor='val_acc', factor = 0.5, patience = 2, verbose = 1, mode = 'max', min_lr = 0.00001)                              \ncallbacks_list = [checkpoint, reduce_lr] # LR Scheduler Used here\n\nhistory = model.fit_generator(train_gen, steps_per_epoch = train_steps, \n                    validation_data = val_gen,\n                    validation_steps = val_steps,\n                    epochs = 11,\n                    verbose = 1,\n                    callbacks = callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bbdf26a8e7e8dadda170032e835214200138374a"},"cell_type":"markdown","source":"# **Model Evaluation**"},{"metadata":{"_uuid":"94ed027e01cbea6fcadbb7e26f94316302146b98"},"cell_type":"markdown","source":"We can determine our epochs based on the convergence of below graphs."},{"metadata":{"trusted":true,"_uuid":"98303020b3956a521d482e5faec939247a25e7db"},"cell_type":"code","source":"# Plot training & validation accuracy values\nplt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])\nplt.title('Model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Test'], loc='best')\nplt.show()\n\n# Plot training & validation loss values\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Test'], loc='best')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ab9762944fd603c2e6397f386733aca8828d096b"},"cell_type":"markdown","source":"### **Load the saved weights**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# change val_gen batch_size to 1 for prediction\nval_gen = datagen.flow_from_directory(valid_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=1,\n                                        class_mode='categorical',\n                                        shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b87f923b7e9b77a2fe10012f7d63fb68636363b"},"cell_type":"code","source":"# Here the best epoch will be used.\nmodel.load_weights('checkpoint.h5')\n\nval_loss, val_acc = \\\nmodel.evaluate_generator(val_gen, steps=len(df_val))\nprint('val_loss:', val_loss)\nprint('val_acc:', val_acc)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3fa0a7472c5c7cd4bdf1882d4c6798e7524e0e39"},"cell_type":"markdown","source":"### **Validate the model (Measure Model Performance)**"},{"metadata":{"trusted":true,"_uuid":"2e486814e4c1ed4f8777e4f5efa5c1487056ab63"},"cell_type":"code","source":"# make a prediction\npredictions = model.predict_generator(val_gen, steps=len(df_val), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f32e41a752dcb071ba2b4dc79810d5bc7f105fd7"},"cell_type":"code","source":"# Put the predictions into a dataframe.\ndf_preds = pd.DataFrame(predictions, columns=['no_tumor', 'has_tumor'])\ndf_preds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4fc00f43fbf7473a8ac0abe3142f3ea36f0bb3fe"},"cell_type":"code","source":"# Get the true labels\ny_true = val_gen.classes\n\n# Get the predicted labels as probabilities\ny_pred = df_preds['has_tumor']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"84f67c255101f1031aff34e852a163f5e97a3b1a"},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, roc_curve, auc\nprint('ROC AUC Score = ',roc_auc_score(y_true, y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cda221a6c8ff013907be72c92f318708f3895ad9"},"cell_type":"code","source":"fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_true, y_pred)\nauc_keras = auc(fpr_keras, tpr_keras)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b228c0089bd4d23e1efcabee3745faad3a8d0870"},"cell_type":"markdown","source":"**Let's plot our ROC Curve**"},{"metadata":{"trusted":true,"_uuid":"4ab8c5f0d97d592895d645d1896e496c880740bc"},"cell_type":"code","source":"plt.figure(1)\nplt.plot([0, 1], [0, 1], 'k--')\nplt.plot(fpr_keras, tpr_keras, label='area = {:.2f}'.format(auc_keras))\nplt.xlabel('False positive rate')\nplt.ylabel('True positive rate')\nplt.title('ROC curve')\nplt.legend(loc='best')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d460346ca9dcb69d12636a1e41a721e13803117a"},"cell_type":"markdown","source":"## **Confusion Matrix**"},{"metadata":{"trusted":true,"_uuid":"62dd86efa45a118cf559c8ed2f0f1bc9a3ccdb11"},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n# For this to work we need y_pred as binary labels not as probabilities\ny_pred_binary = predictions.argmax(axis=1)\ncm = confusion_matrix(y_true, y_pred_binary)\n\nfrom mlxtend.plotting import plot_confusion_matrix\nfig, ax = plot_confusion_matrix(conf_mat=cm,\n                                show_absolute=True,\n                                show_normed=True,\n                                colorbar=True,\n                               cmap = 'Dark2')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"945a87b858b89305d3eeca6a75d4312eb9559cde"},"cell_type":"markdown","source":"## **Classification Report**"},{"metadata":{"trusted":true,"_uuid":"c600c3454c4d21890d9725e7c98b1d51e2f316d4"},"cell_type":"code","source":"from sklearn.metrics import classification_report\n# Generate a classification report\n\nreport = classification_report(y_true, y_pred_binary, target_names = ['no_tumor', 'has_tumor'])\nprint(report)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5cb8ed2b2d4b7122fca96982e47a48b29a8fb2fd"},"cell_type":"markdown","source":"**Recall** = The classifier's ability to detect a given class. It is the number of correct positive results divided by the number of all relevant samples (all samples that should have been identified as positive).  \n**Precision** = Given a class prediction from a classifier, how likely is it to be correct? It is the number of correct positive results divided by the number of positive results predicted by the classifier.  \n**F1 Score** = The harmonic mean of the recall and precision. Essentially, it punishes extreme values.  \n\n"},{"metadata":{"_uuid":"32947e3337e8df86109fb5f91fbab545fc37febd"},"cell_type":"markdown","source":"# **Make Test Predictions**"},{"metadata":{"trusted":true,"_uuid":"573552072125f0b853626484e60cc8c3454a4a8d"},"cell_type":"code","source":"test_data_predictions = model.predict_generator(test_gen, steps=len(df_test), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f02f0edef530aed42e2752fc13b07fa7ad3feae4"},"cell_type":"code","source":"if test_data_predictions.shape[0] == len(df_test):\n    print('All Predictions Done!')\nelse:\n    print('Error!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0743dc8a0144451ce1f612ac456fd917d05b32e0"},"cell_type":"code","source":"# Put the predictions into a dataframe\ndf_test_preds = pd.DataFrame(test_data_predictions, columns=['no_tumor', 'has_tumor'])\ndf_test_preds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get the true labels\ny_true_test = test_gen.classes\n\n# Get the predicted labels as probabilities\ny_pred_test = df_test_preds['has_tumor']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n# For this to work we need y_pred as binary labels not as probabilities\ny_test_pred_binary = test_data_predictions.argmax(axis=1)\ncm2 = confusion_matrix(y_true_test, y_test_pred_binary)\n\nfrom mlxtend.plotting import plot_confusion_matrix\nfig2, ax2 = plot_confusion_matrix(conf_mat=cm2,\n                                show_absolute=True,\n                                show_normed=True,\n                                colorbar=True,\n                               cmap = 'Dark2')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report\n# Generate a classification report\n\nreport2 = classification_report(y_true_test, y_test_pred_binary, target_names = ['no_tumor', 'has_tumor'])\nprint(report2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ca94a0c19aa76b904a5bc404c87cb652ca0e2d66"},"cell_type":"markdown","source":"***Remove the base_dir to free up memory and commit kernel successfully!***"},{"metadata":{"trusted":true,"_uuid":"78405611b84db8de39153ab22869092d4db20dd1"},"cell_type":"code","source":"# Delete the test_dir directory we created to prevent a Kaggle error.\n# Kaggle allows a max of 500 files to be saved.\nshutil.rmtree('base_dir')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}