{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Part 3: Modelling & Predicting Pneumonia w/ Neural Networks","metadata":{}},{"cell_type":"code","source":"# Imports\nimport os\nimport cv2\nimport glob\nimport time\nimport pydicom\nimport skimage\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom skimage import feature, filters\n%matplotlib inline\n\nfrom functools import partial\nfrom collections import defaultdict\nfrom joblib import Parallel, delayed\nfrom lightgbm import LGBMClassifier\nfrom tqdm import tqdm\n\n# Tensorflow / Keras\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.applications.vgg16 import VGG16\nfrom keras import models\nfrom keras import layers\n\n# sklearn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import RandomizedSearchCV\n\nsns.set_style('whitegrid')\nnp.warnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:42:54.587798Z","iopub.execute_input":"2021-07-31T11:42:54.588232Z","iopub.status.idle":"2021-07-31T11:43:05.025981Z","shell.execute_reply.started":"2021-07-31T11:42:54.588141Z","shell.execute_reply":"2021-07-31T11:43:05.024836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# List our paths\ntrainImagesPath = \"../input/rsna-pneumonia-detection-challenge/stage_2_train_images\"\ntestImagesPath = \"../input/rsna-pneumonia-detection-challenge/stage_2_test_images\"\n\nlabelsPath = \"../input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv\"\nclassInfoPath = \"../input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv\"\n\n# Read the labels and classinfo\nlabels = pd.read_csv(labelsPath)\ndetails = pd.read_csv(classInfoPath)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:43:11.919811Z","iopub.execute_input":"2021-07-31T11:43:11.920184Z","iopub.status.idle":"2021-07-31T11:43:12.094988Z","shell.execute_reply.started":"2021-07-31T11:43:11.920148Z","shell.execute_reply":"2021-07-31T11:43:12.093906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 3.1: Attaining our Training & Testing Data in Proper Format","metadata":{}},{"cell_type":"code","source":"\"\"\"\n@Description: Reads an array of dicom image paths, and returns an array of the images after they have been read\n\n@Inputs: An array of filepaths for the images\n\n@Output: Returns an array of the images after they have been read\n\"\"\"\ndef readDicomData(data):\n    \n    res = []\n    \n    for filePath in tqdm(data): # Loop over data\n        \n        # We use stop_before_pixels to avoid reading the image (Saves on speed/memory)\n        f = pydicom.read_file(filePath, stop_before_pixels=True)\n        res.append(f)\n    \n    return res","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:43:14.120855Z","iopub.execute_input":"2021-07-31T11:43:14.121239Z","iopub.status.idle":"2021-07-31T11:43:14.127974Z","shell.execute_reply.started":"2021-07-31T11:43:14.121203Z","shell.execute_reply":"2021-07-31T11:43:14.126789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get an array of the test & training file paths\ntrainFilepaths = glob.glob(f\"{trainImagesPath}/*.dcm\")\ntestFilepaths = glob.glob(f\"{testImagesPath}/*.dcm\")\n\n# Read data into an array\ntrainImages = readDicomData(trainFilepaths[:5000])\ntestImages = readDicomData(testFilepaths)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:43:20.683670Z","iopub.execute_input":"2021-07-31T11:43:20.684205Z","iopub.status.idle":"2021-07-31T11:45:26.137239Z","shell.execute_reply.started":"2021-07-31T11:43:20.684172Z","shell.execute_reply":"2021-07-31T11:45:26.136077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 3.2: Balancing our Data\n\nWe balance our data as CNNs work best on evenly balanced data","metadata":{}},{"cell_type":"code","source":"COUNT_NORMAL = len(labels.loc[labels['Target'] == 0]) # Number of patients with no pneumonia\nCOUNT_PNE = len(labels.loc[labels['Target'] == 1]) # Number of patients with pneumonia\nTRAIN_IMG_COUNT = len(trainFilepaths) # Total patients\n\n# We calculate the weight of each\nweight_for_0 = (1 / COUNT_NORMAL)*(TRAIN_IMG_COUNT)/2.0 \nweight_for_1 = (1 / COUNT_PNE)*(TRAIN_IMG_COUNT)/2.0\n\nclassWeight = {0: weight_for_0, \n               1: weight_for_1}\n\nprint(f\"Weights: {classWeight}\")","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:45:26.139119Z","iopub.execute_input":"2021-07-31T11:45:26.139580Z","iopub.status.idle":"2021-07-31T11:45:26.162683Z","shell.execute_reply.started":"2021-07-31T11:45:26.139514Z","shell.execute_reply":"2021-07-31T11:45:26.161655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 3.3: Get Train_Y & Test_Y","metadata":{}},{"cell_type":"code","source":"\"\"\"\n@Description: This function parses the medical images meta-data contained\n\n@Inputs: Takes in the dicom image after it has been read\n\n@Output: Returns the unpacked data and the group elements keywords\n\"\"\"\ndef parseMetadata(dcm):\n    \n    unpackedData = {}\n    groupElemToKeywords = {}\n    \n    for d in dcm: # Iterate here to force conversion from lazy RawDataElement to DataElement\n        pass\n    \n    # Un-pack Data\n    for tag, elem in dcm.items():\n        tagGroup = tag.group\n        tagElem = tag.elem\n        keyword = elem.keyword\n        groupElemToKeywords[(tagGroup, tagElem)] = keyword\n        value = elem.value\n        unpackedData[keyword] = value\n        \n    return unpackedData, groupElemToKeywords","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:45:26.164289Z","iopub.execute_input":"2021-07-31T11:45:26.164584Z","iopub.status.idle":"2021-07-31T11:45:26.173063Z","shell.execute_reply.started":"2021-07-31T11:45:26.164554Z","shell.execute_reply":"2021-07-31T11:45:26.171246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# These parse the metadata into dictionaries\ntrainMetaDicts, trainKeyword = zip(*[parseMetadata(x) for x in tqdm(trainImages)])\ntestMetaDicts, testKeyword = zip(*[parseMetadata(x) for x in tqdm(testImages)])","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:45:26.175102Z","iopub.execute_input":"2021-07-31T11:45:26.175445Z","iopub.status.idle":"2021-07-31T11:45:32.992700Z","shell.execute_reply.started":"2021-07-31T11:45:26.175413Z","shell.execute_reply":"2021-07-31T11:45:32.991622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n@Description: This function goes through the dicom image information and returns 1 or 0\n              depending on whether the image contains Pneumonia or not\n\n@Inputs: A dataframe containing the metadata\n\n@Output: Returns the Y result (i.e: our train and test y)\n\"\"\"\ndef createY(df):\n    y = (df['SeriesDescription'] == 'view: PA')\n    Y = np.zeros(len(y)) # Initialise Y\n    \n    for i in range(len(y)):\n        if(y[i] == True):\n            Y[i] = 1\n    \n    return Y\n\n\ntrain_df = pd.DataFrame.from_dict(data=trainMetaDicts)\ntest_df = pd.DataFrame.from_dict(data=testMetaDicts)\n\ntrain_df['dataset'] = 'train'\ntest_df['dataset'] = 'test'\n\ndf = train_df\ndf2 = test_df\n\ntrain_Y = createY(df) # Create training Y \ntest_Y = createY(df2) # Create testing Y","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:45:32.994159Z","iopub.execute_input":"2021-07-31T11:45:32.994473Z","iopub.status.idle":"2021-07-31T11:45:33.209015Z","shell.execute_reply.started":"2021-07-31T11:45:32.994439Z","shell.execute_reply":"2021-07-31T11:45:33.207915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 3.4: Get Train_X & Test_X","metadata":{}},{"cell_type":"code","source":"\"\"\"\n@Description: This decodes an image by reading the pixel array, resizing it into the correct format and\n              normalising the pixels\n\n@Inputs:\n    - filePath: This is the filepath of the image that we want to decode\n\n@Output:\n    - img: This is the image after it has been decoded\n\"\"\"\ndef decodeImage(filePath):\n    image = pydicom.read_file(filePath).pixel_array\n    image = cv2.resize(image, (128, 128))\n    return (image/255)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:45:33.210598Z","iopub.execute_input":"2021-07-31T11:45:33.211058Z","iopub.status.idle":"2021-07-31T11:45:33.217120Z","shell.execute_reply.started":"2021-07-31T11:45:33.210997Z","shell.execute_reply":"2021-07-31T11:45:33.215821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get our train x in the correct shape\ntrain_X = []\n\nfor filePath in tqdm(trainFilepaths[:5000]):\n    \n    img = decodeImage(filePath)\n    train_X.append(img)\n\ntrain_X = np.array(train_X) # Convert to np.array\ntrain_X_rgb = np.repeat(train_X[..., np.newaxis], 3, -1) # Reshape into rgb format","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:45:33.218686Z","iopub.execute_input":"2021-07-31T11:45:33.219144Z","iopub.status.idle":"2021-07-31T11:46:32.652053Z","shell.execute_reply.started":"2021-07-31T11:45:33.219100Z","shell.execute_reply":"2021-07-31T11:46:32.650951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get our test x in the correct shape for NN\ntest_X = []\n\nfor filePath in tqdm(testFilepaths):\n    img_test = decodeImage(filePath) # Decode & Resize\n    test_X.append(img_test)\n\ntest_X = np.array(test_X) # Convert to np array\ntest_X_rgb = np.repeat(test_X[..., np.newaxis], 3, -1) # Reshape into rgb format","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:46:32.655236Z","iopub.execute_input":"2021-07-31T11:46:32.655758Z","iopub.status.idle":"2021-07-31T11:47:06.646058Z","shell.execute_reply.started":"2021-07-31T11:46:32.655710Z","shell.execute_reply":"2021-07-31T11:47:06.645087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n@Description: This function plots our metrics for our models across epochs\n\n@Inputs: The history of the fitted model\n\n@Output: N/A\n\"\"\"\ndef plottingScores(hist):\n    fig, ax = plt.subplots(1, 5, figsize=(20, 3))\n    ax = ax.ravel()\n\n    for i, met in enumerate(['accuracy', 'precision', 'recall', 'AUC', 'loss']):\n        ax[i].plot(hist.history[met])\n        ax[i].plot(hist.history['val_' + met])\n        ax[i].set_title('Model {}'.format(met))\n        ax[i].set_xlabel('epochs')\n        ax[i].set_ylabel(met)\n        ax[i].legend(['train', 'val'])","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:47:06.647488Z","iopub.execute_input":"2021-07-31T11:47:06.647853Z","iopub.status.idle":"2021-07-31T11:47:06.656478Z","shell.execute_reply.started":"2021-07-31T11:47:06.647818Z","shell.execute_reply":"2021-07-31T11:47:06.655030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 3.5: Metrics Evaluation\nFor our metrics, we want to include <b><i>precision</i></b> and <b><i>recall</i></b> as they will provide use with more info on how good our model is\n \n \n- <b><u>Accuracy:</u></b> This tells us what fraction of the labels are correct.\n    - Since our data is not balanced, accuracy might give a skewed sense of a good model\n\n\n- <b><u>Precision:</u></b> This tells us the number of true positives (TP) over the sum of TP and false positives (FP). \n    - It shows what fraction of labeled positives are actually correct.\n\n\n- <b><u>Recall:</u></b> The number of TP over the sum of TP and false negatves (FN). \n    - It shows what fraction of actual positives are correct.","metadata":{}},{"cell_type":"code","source":"# These our our scoring metrics that are going to be used to evaluate our models\nMETRICS = ['accuracy', \n           tf.keras.metrics.Precision(name='precision'), \n           tf.keras.metrics.Recall(name='recall'), \n           tf.keras.metrics.AUC(name='AUC')]","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:47:06.658311Z","iopub.execute_input":"2021-07-31T11:47:06.658882Z","iopub.status.idle":"2021-07-31T11:47:06.735938Z","shell.execute_reply.started":"2021-07-31T11:47:06.658832Z","shell.execute_reply":"2021-07-31T11:47:06.734833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Tuning our Models with Callbacks\n\n- We'll use Keras callbacks to further finetune our model. \n- The <b>checkpoint callback</b> saves the best weights of the model, so next time we want to use the model, we do not have to spend time training it. \n- The <b>early stopping callback</b> stops the training process when the model starts becoming stagnant, or even worse, when the model starts overfitting. \n- Since we set restore_best_weights to True, the returned model at the end of the training process will be the model with the best weights (i.e. low loss and high accuracy).","metadata":{}},{"cell_type":"code","source":"# Define our callback functions to pass when fitting our NNs\ndef exponential_decay(lr0, s):\n    def exponential_decay_fn(epoch):\n        return lr0 * 0.1 **(epoch / s)\n    return exponential_decay_fn\n\nexponential_decay_fn = exponential_decay(0.01, 20)\n\nlr_scheduler = tf.keras.callbacks.LearningRateScheduler(exponential_decay_fn)\n\ncheckpoint_cb = tf.keras.callbacks.ModelCheckpoint(\"xray_model.h5\", save_best_only=True)\n\nearly_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=5, restore_best_weights=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:47:06.737397Z","iopub.execute_input":"2021-07-31T11:47:06.737845Z","iopub.status.idle":"2021-07-31T11:47:06.744978Z","shell.execute_reply.started":"2021-07-31T11:47:06.737799Z","shell.execute_reply":"2021-07-31T11:47:06.743711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 3.6: Building Model #1 - Fully Connected Model","metadata":{}},{"cell_type":"code","source":"\"\"\"\n@Description: This function builds our simple Fully-connected NN\n\n@Inputs: N/A\n\n@Output: Returns the FCNN Model\n\"\"\"\ndef build_fcnn_model():\n    \n    # Basic model with a flattening layer followng by 2 dense layers\n    # The first dense layer is using relu and the 2nd one is using sigmoid\n    model = tf.keras.models.Sequential([\n                tf.keras.layers.Flatten(input_shape = (128, 128, 3)), \n                tf.keras.layers.Dense(128, activation = \"relu\"), \n                tf.keras.layers.Dense(1, activation = \"sigmoid\")\n                ])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:47:06.746714Z","iopub.execute_input":"2021-07-31T11:47:06.747276Z","iopub.status.idle":"2021-07-31T11:47:06.759470Z","shell.execute_reply.started":"2021-07-31T11:47:06.747225Z","shell.execute_reply":"2021-07-31T11:47:06.758344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build our FCNN model and compile\nmodel_fcnn = build_fcnn_model()\nmodel_fcnn.summary()\nmodel_fcnn.compile(optimizer=\"adam\", loss=\"binary_crossentropy\", metrics=METRICS) # Compile","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:47:06.761086Z","iopub.execute_input":"2021-07-31T11:47:06.761553Z","iopub.status.idle":"2021-07-31T11:47:06.874644Z","shell.execute_reply.started":"2021-07-31T11:47:06.761482Z","shell.execute_reply":"2021-07-31T11:47:06.873514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fitting Model to Training Data","metadata":{}},{"cell_type":"code","source":"history_fcnn = model_fcnn.fit(train_X_rgb, \n                          train_Y,  \n                          epochs = 30,\n                          batch_size = 128,\n                          validation_split = 0.2, \n                          class_weight = classWeight, \n                          verbose = 1,\n                          callbacks = [checkpoint_cb, early_stopping_cb, lr_scheduler]) # Fit the model","metadata":{"execution":{"iopub.status.busy":"2021-07-29T08:18:24.045069Z","iopub.execute_input":"2021-07-29T08:18:24.045426Z","iopub.status.idle":"2021-07-29T08:18:53.936095Z","shell.execute_reply.started":"2021-07-29T08:18:24.045396Z","shell.execute_reply":"2021-07-29T08:18:53.934825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate and display results\nresults = model_fcnn.evaluate(test_X_rgb, test_Y) # Evaluate the model on test data\nresults = dict(zip(model_fcnn.metrics_names,results))\n\nprint(results)\nplottingScores(history_fcnn) # Visualise scores","metadata":{"execution":{"iopub.status.busy":"2021-07-29T08:18:59.050012Z","iopub.execute_input":"2021-07-29T08:18:59.050407Z","iopub.status.idle":"2021-07-29T08:19:01.712716Z","shell.execute_reply.started":"2021-07-29T08:18:59.050374Z","shell.execute_reply":"2021-07-29T08:19:01.711589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 3.7: Building Model #2 - CNN\n\nIn our CNN model, fewer parameters are needed because every convolutional layer reduces the dimensions of the input through the convolution operation.","metadata":{}},{"cell_type":"code","source":"\"\"\"\n@Description: This function builds our custom CNN Model\n\n@Inputs: N/A\n\n@Output: Returns the CNN model\n\"\"\"\ndef build_cnn_model():\n    model = tf.keras.Sequential([\n        tf.keras.layers.Conv2D(32, kernel_size=(3,3), strides=(1,1), padding = 'valid', activation = 'relu', input_shape=(128, 128, 3)), #  convolutional layer\n        tf.keras.layers.MaxPool2D(pool_size=(2,2)), # flatten output of conv\n        \n        tf.keras.layers.Conv2D(32, kernel_size=(3,3), strides=(1,1), padding = 'valid', activation = 'relu'), #  convolutional layer\n        tf.keras.layers.MaxPool2D(pool_size=(2,2)), # flatten output of conv\n        tf.keras.layers.Dropout(0.3),\n        \n        tf.keras.layers.Conv2D(64, 3, activation = 'relu', padding = 'valid'),\n        tf.keras.layers.Conv2D(128, 3, activation = 'relu', padding = 'valid'),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.MaxPool2D(),\n        tf.keras.layers.Dropout(0.4),\n        \n        tf.keras.layers.Flatten(), # flatten output of conv\n        tf.keras.layers.Dense(512, activation = \"relu\"), # hidden layer\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Dropout(0.5),\n        tf.keras.layers.Dense(128, activation = \"relu\"), #  output layer\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Dropout(0.5),\n        tf.keras.layers.Dense(1, activation = \"sigmoid\")])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:47:06.876023Z","iopub.execute_input":"2021-07-31T11:47:06.876337Z","iopub.status.idle":"2021-07-31T11:47:06.887799Z","shell.execute_reply.started":"2021-07-31T11:47:06.876304Z","shell.execute_reply":"2021-07-31T11:47:06.886791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build and compile model\nmodel_cnn = build_cnn_model()\nmodel_cnn.summary()\nmodel_cnn.compile(optimizer='adam', loss='binary_crossentropy', metrics=METRICS)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:47:06.888972Z","iopub.execute_input":"2021-07-31T11:47:06.889483Z","iopub.status.idle":"2021-07-31T11:47:07.132255Z","shell.execute_reply.started":"2021-07-31T11:47:06.889446Z","shell.execute_reply":"2021-07-31T11:47:07.129708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit model\nhistory_cnn = model_cnn.fit(train_X_rgb, \n                      train_Y,  \n                      epochs=30, \n                      validation_split = 0.15, \n                      batch_size=128,\n                      class_weight=classWeight,\n                      callbacks=[checkpoint_cb, early_stopping_cb, lr_scheduler],\n                      verbose=1) # Fit the model","metadata":{"execution":{"iopub.status.busy":"2021-07-28T12:09:59.319841Z","iopub.execute_input":"2021-07-28T12:09:59.320273Z","iopub.status.idle":"2021-07-28T12:31:57.656484Z","shell.execute_reply.started":"2021-07-28T12:09:59.320225Z","shell.execute_reply":"2021-07-28T12:31:57.65553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evalute the models results and put into a dict\nresults = model_cnn.evaluate(test_X_rgb, test_Y)\nresults = dict(zip(model_cnn.metrics_names,results))\n\nprint(results)\nplottingScores(history_cnn) # Visualise scores","metadata":{"execution":{"iopub.status.busy":"2021-07-28T23:09:30.991777Z","iopub.execute_input":"2021-07-28T23:09:30.992549Z","iopub.status.idle":"2021-07-28T23:09:31.077553Z","shell.execute_reply.started":"2021-07-28T23:09:30.992402Z","shell.execute_reply":"2021-07-28T23:09:31.075658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 3.8: Building Model #3 - Mobile Net with Transfer Learning","metadata":{}},{"cell_type":"code","source":"\"\"\"\n@Description: This function builds our MobileNet Model\n\n@Inputs: N/A\n\n@Output: Returns the Mobile Net model\n\"\"\"\ndef build_mn_model():\n    \n    model = tf.keras.Sequential([\n        tf.keras.applications.MobileNetV2(include_top = False, weights=\"imagenet\", input_shape=(128, 128, 3)),\n        tf.keras.layers.GlobalAveragePooling2D(),\n        Dense(1, activation = 'sigmoid')\n    ])\n    \n    model.layers[0].trainable = False\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:47:07.134074Z","iopub.execute_input":"2021-07-31T11:47:07.134640Z","iopub.status.idle":"2021-07-31T11:47:07.143804Z","shell.execute_reply.started":"2021-07-31T11:47:07.134581Z","shell.execute_reply":"2021-07-31T11:47:07.142351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build and compile mobile net model\nmodel_mn = build_mn_model()\nmodel_mn.summary()\nmodel_mn.compile(optimizer='adam', loss='binary_crossentropy', metrics=METRICS)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:47:07.145847Z","iopub.execute_input":"2021-07-31T11:47:07.146391Z","iopub.status.idle":"2021-07-31T11:47:08.972091Z","shell.execute_reply.started":"2021-07-31T11:47:07.146343Z","shell.execute_reply":"2021-07-31T11:47:08.971001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We run our best model here on a larger portion of the training data","metadata":{}},{"cell_type":"code","source":"history_mn = model_mn.fit(train_X_rgb, \n                          train_Y,  \n                          epochs = 30, \n                          validation_split = 0.20, \n                          class_weight = classWeight,\n                          batch_size = 64,\n                          callbacks = [checkpoint_cb, early_stopping_cb, lr_scheduler])","metadata":{"execution":{"iopub.status.busy":"2021-07-31T11:48:26.577919Z","iopub.execute_input":"2021-07-31T11:48:26.578292Z","iopub.status.idle":"2021-07-31T11:59:22.958363Z","shell.execute_reply.started":"2021-07-31T11:48:26.578262Z","shell.execute_reply":"2021-07-31T11:59:22.956892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show results and print graphs\nresults = model_mn.evaluate(test_X_rgb, test_Y)\nresults = dict(zip(model_mn.metrics_names,results))\n\nprint(results)\nplottingScores(history_mn) # Visualise scores","metadata":{"execution":{"iopub.status.busy":"2021-07-31T12:00:16.041076Z","iopub.execute_input":"2021-07-31T12:00:16.041624Z","iopub.status.idle":"2021-07-31T12:00:41.076202Z","shell.execute_reply.started":"2021-07-31T12:00:16.041576Z","shell.execute_reply":"2021-07-31T12:00:41.074996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Show Confusion Matrix","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\ny_pred = model_mn.predict_classes(test_X_rgb)\nconfusion_matrix(test_Y, y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T12:00:47.370303Z","iopub.execute_input":"2021-07-31T12:00:47.370837Z","iopub.status.idle":"2021-07-31T12:01:11.439601Z","shell.execute_reply.started":"2021-07-31T12:00:47.370787Z","shell.execute_reply":"2021-07-31T12:01:11.438638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Function to Perform K-Fold CV","metadata":{}},{"cell_type":"code","source":"\"\"\"\n@Description: This function performs K-Fold Cross Validation with a provided Deep Learning Model\n\n@Inputs:\n    - K: Number of folds\n    - build_model_func: Function to create model\n    - epochs: Number of epochs to train data\n    - batchSize: Batch size when fitting the model\n\n@Output: Dict of metric results from K-fold CV\n\"\"\"\ndef performCV(K, build_model_func, epochs, batchSize):\n    \n    kfold = KFold(n_splits = K, shuffle = True) # Split data into K Folds\n    \n    res = {\n        'acc_per_fold': [],\n        'precision_per_fold': [],\n        'recall_per_fold': [],\n        'auc_per_fold': [],\n        'loss_per_fold': []\n    }\n\n    fold_no = 1\n\n    for train_index, test_index in kfold.split(train_X_rgb):\n\n        X_train, X_test = train_X_rgb[train_index], train_X_rgb[test_index] # Split data\n        y_train, y_test = train_Y[train_index], train_Y[test_index]\n\n        model = build_model_func() # Build model\n        mets = ['accuracy', tf.keras.metrics.Precision(name='precision'), tf.keras.metrics.Recall(name='recall'), tf.keras.metrics.AUC(name='AUC')]\n\n        model.compile(optimizer='adam', loss='binary_crossentropy', metrics=mets) # Compile our model\n\n        print('------------------------------------------------------------------------')\n        print(f'Training for fold {fold_no} ...')\n\n        # Train the model on the current fold\n        history = model.fit(X_train,\n                            y_train, \n                            epochs = epochs,\n                            batch_size = batchSize,\n                            class_weight = classWeight,\n                            callbacks = [checkpoint_cb, early_stopping_cb, lr_scheduler]) # Fit data to model\n\n        scores = model.evaluate(X_test, y_test, verbose=0) # Evalute the model\n\n        print(f'Scores for fold {fold_no}:')\n        print(f'{model.metrics_names[0]}: {scores[0]}')\n        print(f'{model.metrics_names[1]}: {scores[1]*100}%')\n        print(f'{model.metrics_names[2]}: {scores[2]*100}%')\n        print(f'{model.metrics_names[3]}: {scores[3]*100}%')\n\n        res['loss_per_fold'].append(scores[0])\n        res['acc_per_fold'].append(scores[1] * 100)\n        res['precision_per_fold'].append(scores[2]*100)\n        res['recall_per_fold'].append(scores[3]*100)\n        res['auc_per_fold'].append(scores[4]*100)\n\n        gc.collect()\n        # Increase fold number\n        fold_no += 1\n    \n    return res # return our results dict","metadata":{"execution":{"iopub.status.busy":"2021-07-31T12:01:37.791317Z","iopub.execute_input":"2021-07-31T12:01:37.792055Z","iopub.status.idle":"2021-07-31T12:01:37.806154Z","shell.execute_reply.started":"2021-07-31T12:01:37.791997Z","shell.execute_reply":"2021-07-31T12:01:37.804964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 3.9: K-Fold Cross Validation with all 3 Networks","metadata":{}},{"cell_type":"code","source":"# Full-connected NN\nresFCNN = performCV(5, build_fcnn_model, 30, 128)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T13:32:08.196193Z","iopub.execute_input":"2021-07-31T13:32:08.196596Z","iopub.status.idle":"2021-07-31T13:36:47.385914Z","shell.execute_reply.started":"2021-07-31T13:32:08.196563Z","shell.execute_reply":"2021-07-31T13:36:47.384614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convolutional NN\nresCNN = performCV(5, build_cnn_model, 30, 64)","metadata":{"execution":{"iopub.status.busy":"2021-07-30T04:06:39.34779Z","iopub.execute_input":"2021-07-30T04:06:39.348452Z","iopub.status.idle":"2021-07-30T05:52:41.195812Z","shell.execute_reply.started":"2021-07-30T04:06:39.34841Z","shell.execute_reply":"2021-07-30T05:52:41.194774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# MobileNet\nresMB = performCV(5, build_mn_model, 30, 64)","metadata":{"execution":{"iopub.status.busy":"2021-07-31T12:02:06.523198Z","iopub.execute_input":"2021-07-31T12:02:06.523716Z","iopub.status.idle":"2021-07-31T13:27:48.480903Z","shell.execute_reply.started":"2021-07-31T12:02:06.523675Z","shell.execute_reply":"2021-07-31T13:27:48.479691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resMB","metadata":{"execution":{"iopub.status.busy":"2021-07-31T13:28:11.884092Z","iopub.execute_input":"2021-07-31T13:28:11.884579Z","iopub.status.idle":"2021-07-31T13:28:11.895640Z","shell.execute_reply.started":"2021-07-31T13:28:11.884514Z","shell.execute_reply":"2021-07-31T13:28:11.894437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Results For Architectures","metadata":{}},{"cell_type":"code","source":"\"\"\"\n5k Training\n3k Testing\n\nArchitecture 1:\n        tf.keras.layers.Conv2D(25, kernel_size=(3,3), strides=(1,1), padding='valid', activation='relu', input_shape=(128, 128, 3)), #  convolutional layer\n        tf.keras.layers.MaxPool2D(pool_size=(2,2)), # flatten output of conv\n        tf.keras.layers.Flatten(), # flatten output of conv\n        tf.keras.layers.Dense(100, activation='relu'), # hidden layer\n        tf.keras.layers.Dense(1, activation='sigmoid') #  output layer\n        \n    ---------------------------------------------------- Performance on Test Data ----------------------------------------------------\n    { 'loss': 0.255420297, 'accuracy': 0.904666662, 'precision': 0.881006836, 'recall': 0.951792359, 'AUC': 0.968622922}\n        \nArchitecture 2:\n        tf.keras.layers.Conv2D(25, kernel_size=(3,3), strides=(1,1), padding='valid', activation='relu', input_shape=(128, 128, 3)), #  convolutional layer\n        tf.keras.layers.MaxPool2D(pool_size=(2,2)), # flatten output of conv\n        tf.keras.layers.Conv2D(25, kernel_size=(3,3), strides=(1,1), padding='valid', activation='relu'), #  convolutional layer\n        tf.keras.layers.MaxPool2D(pool_size=(2,2)), # flatten output of conv\n        tf.keras.layers.Flatten(), # flatten output of conv\n        tf.keras.layers.Dense(100, activation='relu'), # hidden layer\n        tf.keras.layers.Dense(1, activation='sigmoid') #  output layer\n    \n    ---------------------------------------------------- Performance on Test Data ----------------------------------------------------\n    { 'loss': 0.198399558, 'accuracy': 0.950666666, 'precision': 0.933372616, 'recall': 0.978368341, 'AUC': 0.986180067}\n    \n    \nArchitecture 3:\n        tf.keras.layers.Conv2D(25, kernel_size=(3,3), strides=(1,1), padding='valid', activation='relu', input_shape=(128, 128, 3)), #  convolutional layer\n        tf.keras.layers.MaxPool2D(pool_size=(2,2)), # flatten output of conv\n        \n        tf.keras.layers.Conv2D(25, kernel_size=(3,3), strides=(1,1), padding='valid', activation='relu'), #  convolutional layer\n        tf.keras.layers.MaxPool2D(pool_size=(2,2)), # flatten output of conv\n        \n        tf.keras.layers.Conv2D(32, 3, activation='relu', padding='same'),\n        tf.keras.layers.Conv2D(64, 3, activation='relu', padding='same'),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.MaxPool2D(),\n        \n        tf.keras.layers.Flatten(), # flatten output of conv\n        tf.keras.layers.Dense(100, activation='relu'), # hidden layer\n        tf.keras.layers.Dense(1, activation='sigmoid') #  output layer\n\n\n    ---------------------------------------------------- Performance on Test Data ----------------------------------------------------\n    {'loss': 0.1422816216, 'accuracy': 0.976333320, 'precision': 0.984952986, 'recall': 0.970951795, 'AUC': 0.991799652}\n\nArchitecture 4:\n        tf.keras.layers.Conv2D(25, kernel_size=(3,3), strides=(1,1), padding='valid', activation='relu', input_shape=(128, 128, 3)), #  convolutional layer\n        tf.keras.layers.MaxPool2D(pool_size=(2,2)), # flatten output of conv\n        \n        tf.keras.layers.Conv2D(25, kernel_size=(3,3), strides=(1,1), padding='valid', activation='relu'), #  convolutional layer\n        tf.keras.layers.MaxPool2D(pool_size=(2,2)), # flatten output of conv\n        tf.keras.layers.Dropout(0.2),\n        \n        tf.keras.layers.Conv2D(32, 3, activation='relu', padding='valid'),\n        tf.keras.layers.Conv2D(64, 3, activation='relu', padding='valid'),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.MaxPool2D(),\n        tf.keras.layers.Dropout(0.3),\n        \n        tf.keras.layers.Flatten(), # flatten output of conv\n        tf.keras.layers.Dense(100, activation='relu'), # hidden layer\n        tf.keras.layers.Dropout(0.5),\n        tf.keras.layers.Dense(1, activation='sigmoid') #  output layer\n    \n    \n    ---------------------------------------------------- Performance on Test Data ----------------------------------------------------\n    { 'loss': 0.1239979043, 'accuracy': 0.982666671, 'precision': 0.985130131, 'recall': 0.982694685, 'AUC': 0.992944598}\n            \n            \nArchitecture 5:\n        tf.keras.layers.Conv2D(25, kernel_size=(3,3), strides=(1,1), padding='valid', activation='relu', input_shape=(128, 128, 3)), #  convolutional layer\n        tf.keras.layers.MaxPool2D(pool_size=(2,2)), # flatten output of conv\n        \n        tf.keras.layers.Conv2D(25, kernel_size=(3,3), strides=(1,1), padding='valid', activation='relu'), #  convolutional layer\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.MaxPool2D(pool_size=(2,2)), # flatten output of conv\n        tf.keras.layers.Dropout(0.2),\n        \n        tf.keras.layers.Conv2D(32, 3, activation='relu', padding='valid'),\n        tf.keras.layers.Conv2D(64, 3, activation='relu', padding='valid'),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.MaxPool2D(),\n        tf.keras.layers.Dropout(0.3),\n        \n\n        tf.keras.layers.Flatten(), # Flattening\n        \n        # Full Connection\n        tf.keras.layers.Dense(64, activation='relu'), # hidden layer\n        tf.keras.layers.Dropout(0.5), # Dropout\n        tf.keras.layers.Dense(1, activation='sigmoid') #  output layer\n        \n        ---------------------------------------------------- Performance on Test Data ----------------------------------------------------\n    { 'loss': 0.12123415754, 'accuracy': 0.984615671, 'precision': 0.987261581, 'recall': 0.985671211, 'AUC': 0.994511598}\n\n\"\"\"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Manual Hyper-parameter Tuning results for FCNN","metadata":{}},{"cell_type":"code","source":"\"\"\"\nManual Hyper-parameter Tuning\n\nbatch_size=32\n    cWeight: None {'loss': 0.22600425779819489, 'accuracy': 0.92166668176651, 'precision': 0.9292364716529846, 'recall': 0.9252163171768188, 'AUC': 0.9672043323516846}\n    cWeight: Balanced {'loss': 0.2335905283689499, 'accuracy': 0.9136666655540466, 'precision': 0.9155963063240051, 'recall': 0.9252163171768188, 'AUC': 0.9655577540397644}\n    \nbatch_size=64\n    cWeight: None {'loss': 0.22068753838539124, 'accuracy': 0.9193333387374878, 'precision': 0.9149577617645264, 'recall': 0.9375772476196289, 'AUC': 0.9699712991714478}\n    cWeight: Balanced {'loss': 0.2424456775188446, 'accuracy': 0.9079999923706055, 'precision': 0.8829908967018127, 'recall': 0.956118643283844, 'AUC': 0.9677296280860901}\n\nbatch_size=128\n    cWeight: None {'loss': 0.23750829696655273, 'accuracy': 0.9100000262260437, 'precision': 0.8855835199356079, 'recall': 0.95673668384552, 'AUC': 0.9694961309432983}\n    cWeight: Balanced {'loss': 0.2239508330821991, 'accuracy': 0.9196666479110718, 'precision': 0.9100655317306519, 'recall': 0.94437575340271, 'AUC': 0.9697944521903992}\n\nbatch_size=256\n    cWeight: None {'loss': 0.2305724024772644, 'accuracy': 0.9190000295639038, 'precision': 0.9109384417533875, 'recall': 0.9419035911560059, 'AUC': 0.9681356549263}\n    cWeight: Balanced {'loss': 0.22952377796173096, 'accuracy': 0.9203333258628845, 'precision': 0.9171203970909119, 'recall': 0.9369592070579529, 'AUC': 0.9694135785102844}\n\n\"\"\"","metadata":{},"execution_count":null,"outputs":[]}]}