{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Preprocessing, Data Visualization, EDA"},{"metadata":{},"cell_type":"markdown","source":"## Import Libraries"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport glob\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom matplotlib.patches import Rectangle\nfrom matplotlib import cm\nimport os\nimport seaborn as sns\nfrom skimage import measure\nimport keras\nimport tensorflow as tf\nimport cv2\n%matplotlib inline\n\n#setting up for customized printing\nfrom IPython.display import Markdown, display\nfrom IPython.display import HTML\ndef printmd(string, color=None):\n    colorstr = \"<span style='color:{}'>{}</span>\".format(color, string)\n    display(Markdown(colorstr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def random_color():\n    rgbl=[1,0,0]\n    np.random.shuffle(rgbl)\n    return tuple(rgbl)  \n\ndef distplot(data, figRows,figCols,xSize, ySize, features, colors):\n    f, axes = plt.subplots(figRows, figCols, figsize=(xSize, ySize))\n    \n    features = np.array(features).reshape(figRows, figCols)\n    colors = np.array(colors).reshape(figRows, figCols)\n    \n    for row in range(figRows):\n        for col in range(figCols):\n            if (figRows == 1 and figCols == 1) :\n                axesplt = axes\n            elif (figRows == 1 and figCols > 1) :\n                axesplt = axes[col]\n            elif (figRows > 1 and figCols == 1) :\n                axesplt = axes[row]\n            else:\n                axesplt = axes[row][col]\n            plot = sns.distplot(data[features[row][col]], color=colors[row][col], ax=axesplt, kde=True, hist_kws={\"edgecolor\":\"k\"})\n            plot.set_xlabel(features[row][col],fontsize=20)\n            \ndef boxplot(data, figRows,figCols,xSize, ySize, features, colors, hue=None, orient='h'):\n    f, axes = plt.subplots(figRows, figCols, figsize=(xSize, ySize))\n    \n    features = np.array(features).reshape(figRows, figCols)\n    colors = np.array(colors).reshape(figRows, figCols)\n    \n    for row in range(figRows):\n        for col in range(figCols):\n            if (figRows == 1 and figCols == 1) :\n                axesplt = axes\n            elif (figRows == 1 and figCols > 1) :\n                axesplt = axes[col]\n            elif (figRows > 1 and figCols == 1) :\n                axesplt = axes[row]\n            else:\n                axesplt = axes[row][col]\n            plot = sns.boxplot(features[row][col], data= data, color=colors[row][col], ax=axesplt, orient=orient, hue=hue)\n            plot.set_xlabel(features[row][col],fontsize=20)\n            \ndef countplot(data, figRows,figCols,xSize, ySize, features, colors=None,palette=None,hue=None, orient=None, rotation=90):\n    f, axes = plt.subplots(figRows, figCols, figsize=(xSize, ySize))\n    \n    features = np.array(features).reshape(figRows, figCols)\n    if(colors is not None):\n        colors = np.array(colors).reshape(figRows, figCols)\n    if(palette is not None):\n        palette = np.array(palette).reshape(figRows, figCols)\n    \n    for row in range(figRows):\n        for col in range(figCols):\n            if (figRows == 1 and figCols == 1) :\n                axesplt = axes\n            elif (figRows == 1 and figCols > 1) :\n                axesplt = axes[col]\n            elif (figRows > 1 and figCols == 1) :\n                axesplt = axes[row]\n            else:\n                axesplt = axes[row][col]\n                \n            if(colors is None):\n                plot = sns.countplot(features[row][col], data=data, palette=palette[row][col], ax=axesplt, orient=orient, hue=hue)\n            elif(palette is None):\n                plot = sns.countplot(features[row][col], data=data, color=colors[row][col], ax=axesplt, orient=orient, hue=hue)\n            plot.set_title(features[row][col],fontsize=20)\n            plot.set_xlabel(None)\n            plot.set_xticklabels(rotation=rotation, labels=plot.get_xticklabels(),fontweight='demibold',fontsize='large')\n            \ndef point_box_bar_plot(data, row, col, target, figRow, figCol, palette='rocket', fontsize='large', fontweight='demibold'):\n    sns.set(style=\"whitegrid\")\n    f, axes = plt.subplots(3, 1, figsize=(figRow, figCol))\n    pplot=sns.pointplot(row,col, data=data, ax=axes[0], linestyles=['--'])\n    pplot.set_xlabel(None)\n    pplot.set_xticklabels(labels=pplot.get_xticklabels(),fontweight=fontweight,fontsize=fontsize)\n    bxplot=sns.boxplot(row,col, data=data, hue=target, ax=axes[1],palette='viridis')\n    bxplot.set_xlabel(None)\n    bxplot.set_xticklabels(labels=bxplot.get_xticklabels(),fontweight=fontweight,fontsize=fontsize)\n    bplot=sns.barplot(row,col, data=data, hue=target, ax=axes[2],palette=palette)\n    bplot.set_xlabel(row,fontsize=20)\n    bplot.set_xticklabels(labels=bplot.get_xticklabels(),fontweight=fontweight,fontsize=fontsize)            \ndef catdist(data, cols):\n    dfs = []\n    for col in cols:\n        colData = pd.DataFrame(data[col].value_counts(), columns=[col])\n        colData['%'] = round((colData[col]/colData[col].sum())*100,2)\n        dfs.append(colData)\n        display(colData)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setup Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def formatMetadataString(val):\n    return str(val).split(':')[1].replace('\\'', '')\n\ndataDir = '/kaggle/input/rsna-pneumonia-detection-challenge'\ntrainDataDir = 'stage_2_train_images'\ntestDataDir = 'stage_2_test_images'\ntrainfiles = glob.glob(os.path.join(dataDir, trainDataDir, \"*.dcm\"))\n\nprintmd('**Reading the metadata from dicom training files**', color='brown')\n\nimage_data = []\nfor f in trainfiles:    \n    ds = pydicom.dcmread(f, stop_before_pixels=True)\n    patientId = formatMetadataString(ds['PatientID'])\n    age = formatMetadataString(ds['PatientAge'])\n    gender = formatMetadataString(ds['PatientSex'])\n    viewPos = formatMetadataString(ds['ViewPosition'])    \n    image_data.append([patientId, int(age), gender, viewPos])\n    \nimage_metadata_df = pd.DataFrame(image_data, columns=['patientId', 'Age', 'Gender','ViewPosition'])        \nimage_metadata_df.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Storing the DICOM files as a dictionary under trainDicomFiles such that the keys are the patient\\\nids and the values are the path of the files corresponding to the image.')\ntrainDicomFiles = {}\ntestDicomFiles = {}\n\nfor root, dirs, files in os.walk(os.path.join(dataDir, trainDataDir)):\n    for fileName in files:\n        if '.dcm' in fileName.lower():\n            trainDicomFiles[fileName.split(sep='.')[0]] = os.path.join(dataDir, trainDataDir, fileName)     \n        \nprint('\\nGenerating the data frame for the training data labels.')\ntrainLabelDf = pd.read_csv(os.path.join(dataDir, 'stage_2_train_labels.csv'))\ntrainLabelDetailedDf = pd.read_csv(os.path.join(dataDir, 'stage_2_detailed_class_info.csv'))\n\n# Joining the class information from the detailed class info CSV to the train labels CSV.\ntrainLabelDf['class'] = trainLabelDetailedDf['class']\n\n# As the training labels file contain multiple rows for each patient corresponding to the number of bounding boxes present\n# in case of a person with lung opacity, we can group the labels by patient id to find number of unique patients in \n# the label file.\n\nprint('\\nGenerating the dictionary for test DICOM files under testDicomFiles.')\nfor root, dirs, files in os.walk(os.path.join(dataDir, testDataDir)):\n    for fileName in files:\n        if '.dcm' in fileName.lower():\n            testDicomFiles[fileName.split(sep='.')[0]] = os.path.join(dataDir, testDataDir, fileName)\n\nprint('Total test data files: ', len(testDicomFiles))\n\n#print('\\nBelow is a sample of the content of training label data frame.')\n#display(trainLabelDf.head(10))\n\nprint('\\nAs the training label data frame may contain multiple records for a patient to represent the bounding box, we can \\\nstore the bounding box labels in a property called boundingBox that contains a array of box data stored as \\\n{x, y, width, height}')\n\nuniquePatientIds = np.unique(trainLabelDf.patientId)\ntrainLabelAndBoundingBoxDf = pd.DataFrame(columns=['patientId', 'target', 'class', 'boundingBox', 'num_of_opacities']\n                                          , index=uniquePatientIds)\n\npatientIdBasedGroups = trainLabelDf.groupby('patientId')\n#counter = 0\nfor patientId, group in patientIdBasedGroups:\n#     if counter > 20:\n#         break\n    trainLabelAndBoundingBoxDf.loc[patientId]['patientId'] = patientId\n    trainLabelAndBoundingBoxDf.loc[patientId]['num_of_opacities'] = 0\n    boundingBox = []\n    for index, groupItem in group.iterrows():\n        trainLabelAndBoundingBoxDf.loc[patientId]['target'] = groupItem['Target']\n        trainLabelAndBoundingBoxDf.loc[patientId]['class'] = groupItem['class']\n        \n        if(not np.isnan(groupItem['x'])):\n            boundingBox.append((groupItem['x'], groupItem['y'], groupItem['width'], groupItem['height']))\n            trainLabelAndBoundingBoxDf.loc[patientId]['num_of_opacities'] += 1\n            \n    trainLabelAndBoundingBoxDf.loc[patientId]['boundingBox'] = boundingBox\n#     counter += 1\n\ndisplay(trainLabelAndBoundingBoxDf.head())\n\nprint('\\nShape of training labels with bounding box list: ', trainLabelAndBoundingBoxDf.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"printmd('**Combining training data with labels and bounding boxes with the image metadata containing age, gender and view position attributes**', color='brown')\ntrainLabelAndBoundingBoxDf_2 = trainLabelAndBoundingBoxDf.reset_index().drop('index', axis=1)\npatient_df = image_metadata_df.join(trainLabelAndBoundingBoxDf_2, how='inner', rsuffix='_2')\npatient_df.drop('patientId_2', axis=1, inplace=True)\npatient_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Details of the Attributes\n\n1. **patientId**: Unique Identifier for a Patient.<br>\n2. **Age** : Patient's age in completed years.<br>\n3. **Gender**: Patient's gender is a categorical attribute with values of 'M'(male) & 'F'(female)<br>\n4. **ViewPosition**: This is a categorial attribute with values of 'PA' and 'AP'<br>\n    a. **PA - PosteroAnterior**: From back to front: When a chest x-ray is taken with the front against the film plate and the x-ray machine in back of the patient it is called an posteroanterior (PA) view.<br>\n    b. **AP - Anteroposterior**: From front to back. When a chest x-ray is taken with the back against the film plate and the x-ray machine in front of the patient it is called an anteroposterior (AP) view. <br>\n5. **class**: Categorical attribute with the following values,<br>\n    a. **Lung Opacity**<br>\n    b. **Normal**<br>\n    c. **No Lung Opacity / Not Normal**<br>\n6. **boundingBox** - These values denotes position of the lung opacity in the CXR image. There can be multiple opacities for a person<br>\n7. **target** - Target attribute denotes whether a patient is Pneumonia positive(1) or negative(0)<br>\n8. **num_of_opacities** - This denotes the number of opacities diagnosed for a person. "},{"metadata":{},"cell_type":"markdown","source":"## Perform Basic EDA"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Total training data files: ', len(trainDicomFiles))\nprint('Total rows in the labels file: ', trainLabelDf.shape[0])\nprint('Total unique patients in the labels file: ', trainLabelDf.groupby('patientId').count().shape[0])\nprint('Unique target values are:', np.unique(trainLabelDf['Target']).tolist())\nprint('Unique class values are: ', np.unique(trainLabelDf['class']).tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"printmd('**We can check if the bounding boxes are present only for the patients with lung opacity.**', color='brown')\nrecordsWithLungOpacityAndBoundingBox = trainLabelAndBoundingBoxDf[(trainLabelAndBoundingBoxDf['boundingBox'].apply(lambda row: len(row) > 0)) & (trainLabelAndBoundingBoxDf['class'] == 'Lung Opacity')].shape[0]\nrecordsWithNoLungOpacityAndBoundingBox = trainLabelAndBoundingBoxDf[(trainLabelAndBoundingBoxDf['boundingBox'].apply(lambda row: len(row) > 0)) & (trainLabelAndBoundingBoxDf['class'] != 'Lung Opacity')].shape[0]\nprint('Record count with bounding box for patients with lung opacity: ', recordsWithLungOpacityAndBoundingBox)\nprint('Record count with bounding box for patients with no lung opacity: ', recordsWithNoLungOpacityAndBoundingBox)\nprint('We can see that bounding boxes are present for only patients with a lung opacity i.e. the patients having pneumonia.')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Analysis of DICOM file samples"},{"metadata":{"trusted":true},"cell_type":"code","source":"printmd('**Analyzing a sample with normal lung image.**', color='brown')\nsampleNormalLungPatient = trainLabelAndBoundingBoxDf[trainLabelAndBoundingBoxDf['class'] == 'Normal'].iloc[0]\nsampleNormalLungDicomFile = pydicom.read_file(trainDicomFiles[sampleNormalLungPatient.patientId])\n#display(sampleNormalLungDicomFile)\n\nprintmd('**Below is the image plot for the normal lung image stored in the DICOM file.**', color='brown')\nplt.imshow(sampleNormalLungDicomFile.pixel_array, cmap=cm.bone)\nplt.show()\n\nprintmd('**Below is the class target for the normal lung sample.**', color='brown')\nsampleNormalLungPatient","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"printmd('**Analyzing a sample with lung opacity image.**', color='brown')\nsampleLungOpacityPatient = trainLabelAndBoundingBoxDf[trainLabelAndBoundingBoxDf['class'] == 'Lung Opacity'].iloc[0]\nsampleLungOpactiyDicomFile = pydicom.read_file(trainDicomFiles[sampleLungOpacityPatient.patientId])\n#display(sampleLungOpactiyDicomFile)\n\nprintmd('**Below is the image plot for the lung opacity image stored in the DICOM file.**', color='brown')\nfig = plt.figure()\nax = fig.add_subplot(111)\nax.imshow(sampleLungOpactiyDicomFile.pixel_array, cmap=cm.bone)\nfor box in sampleLungOpacityPatient['boundingBox']:\n    #print(box)\n    ax.add_patch(Rectangle((box[0], box[1]), box[2], box[3], linewidth=1, edgecolor=random_color(), facecolor='none'))\nplt.show()\n\nprintmd('**Below is the class target for the lung opacity sample.**', color='brown')\nsampleLungOpacityPatient","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"printmd('**Analyzing a sample with no lung opacity and not normal image.**', color='brown')\nsampleLungNotNormalPatient = trainLabelAndBoundingBoxDf[trainLabelAndBoundingBoxDf['class'] == 'No Lung Opacity / Not Normal'].iloc[0]\nsampleLungNotNormalDicomFile = pydicom.read_file(trainDicomFiles[sampleLungNotNormalPatient.patientId])\n#display(sampleLungNotNormalDicomFile)\n\nprintmd('**Below is the image plot for the no lung opacity and not normal image stored in the DICOM file.**', color='brown')\nplt.imshow(sampleLungNotNormalDicomFile.pixel_array, cmap=cm.bone)\nplt.show()\n\nprintmd('**Below is the class target for the no lung opacity and not normal image sample.**', color='brown')\nsampleLungNotNormalPatient","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dealing with missing values"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('We can check the missing values in the train class files.')\n\nprint('Using the count function on the labels data frame, we can find if there exist any column with \"nan\" values.')\ndisplay(trainLabelAndBoundingBoxDf.count())\nprint('\\nAs we can see that all patientId, target and class does not contain any nan values as count is same as total number \\\nof patients.')\n\nprint('\\nWe can check the unique target and class values to determine if there is any ambiguous/missing data.')\nprint('Unique target values are:', np.unique(trainLabelAndBoundingBoxDf['target']).tolist())\nprint('Unique class values are: ', np.unique(trainLabelAndBoundingBoxDf['class']).tolist())\nprint('We can see there exist no invalid value for target and class data.')\n\nprint('\\nNow we can analyze the box coordinates for all the data points where Target is 1 (Pneumonia detected).')\ndisplay(trainLabelDf[trainLabelDf['Target'] == 1].describe())\n\nprint('\\nWe can see from statistics all the x, y coordinates contain valid values and none of x, y, width and height columns\\\n contain the min value as 0 indicating the values would be correctly recorded.')\n\nprint('\\nWe can check to see if all the patients within the train label data set has a corresponding DICOM image available.')\nfor labelRow in trainLabelAndBoundingBoxDf.iterrows():\n    if labelRow[1].patientId not in trainDicomFiles.keys():\n        print(f'{labelRow[1].patientId} is not present in the DICOM files.')\n\nprint('We can see that all the patients in labels file have a corresponding DICOM image file.')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Distributions of Numeric Attribute - Age"},{"metadata":{"trusted":true},"cell_type":"code","source":"distplot(patient_df, 1,1, 12,5, \n         ['Age'], \n         ['red'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Observations\n\nThere is a good distribution of people from all ages in the data. From the plot, it looks is normally distributed with majority of people around ages 40 to 60. \n"},{"metadata":{},"cell_type":"markdown","source":"## Distribution of Categorical Attributes"},{"metadata":{"trusted":true},"cell_type":"code","source":"catdist(patient_df, ['Gender', 'class', 'target', 'ViewPosition'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Bivariate Analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"printmd('**Attributes target - Age**', color='brown')\npoint_box_bar_plot(patient_df, 'target', 'Age', 'target', 15, 10, palette='winter')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Observations\n\n1. Average age of people who are normal and who are affected by pneumonia are the same at around 47. \n2. Distribution of people's age is quite similar for both Normal and Pneumonia affected. "},{"metadata":{"trusted":true},"cell_type":"code","source":"printmd('**Attributes class - Age**', color='brown')\npoint_box_bar_plot(patient_df, 'class', 'Age', 'class', 15, 10, palette='winter')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Observations\n\n1. Average age of people across different classes (Normal, Lung Opacity, No Lung Opacity/Not Normal) are same at around 47. \n2. Distribution of people's age is quite similar for class types. "},{"metadata":{"trusted":true},"cell_type":"code","source":"printmd('**Distribution of data/images accross different classes.**', color='brown')\n\nfig, axes = plt.subplots(1)\ncol1plot = sns.countplot(x='class', data=patient_df)\nfor p in col1plot.patches:\n    col1plot.annotate(format(p.get_height(), '.2f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')\n\nfig.set_figwidth(10)\nfig.set_figheight(8)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Observations\n\nThere is a good ratio of class distributions. Different class types are distributed as below,<br>\n1. No Lung Opacity / Not Normal - 11821 - 44.30%\n2. Normal - 8851 - 33.17%\n3. Lung Opacity - 6012 - 22.53%"},{"metadata":{"trusted":true},"cell_type":"code","source":"printmd('**Distribution of categorical attributes wrt target attribute**', color='brown')\ncountplot(patient_df, 2,2, 20,10, \n         ['target', 'Gender', 'ViewPosition', 'class'], \n         palette=['viridis', 'Dark2_r', 'winter', 'Set1'], hue='target', rotation=60)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Observations\n\n1. There are more people not diagnosed with Pneumonia. Target attribute is distributed as, <br>\n    a. 0 - 20672 - 77.47 <br>\n    b. 1 - 6012 - 22.53 <br>\n2. Gender - Target -> Number of male patients is quite similar in count as the number of female patients for both Normal and Pneumonia affected, with male patients slightly higher. \n3. ViewPosition - Target -> Number of patients who have given CXR in PA viewposition is quite same in count as the number of patients in AP viewposition for both Normal and Pneumonia affected. \n4. Class - Target -> No Lung Opacity / Not Normal class is also considered as Non-Pneumonic along with Normal class. Lung Opacity confirms Positive Pneumonia in patients. "},{"metadata":{},"cell_type":"markdown","source":"### Observations from the visualizations"},{"metadata":{"trusted":true},"cell_type":"code","source":"printmd('**We can see the data is distributed across the different classes with adequate representation from each class. \\\nThereby, we can use the existing dataset for model generation.**', color='blue')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Building"},{"metadata":{},"cell_type":"markdown","source":"### 1) Building a pneumonia detection model starting from basic CNN and then improving upon it."},{"metadata":{"trusted":true},"cell_type":"code","source":"print('As we can have multiple bounding boxes for the lung opacity condition, we can consider a mask with all region with lung \\\nmarked by all 1s to indicate the presence of lung opacity. This can help us represent multiple lung opacity regions in a single mask.')\nprint('\\nAdditionally, we would be using a convolutional neural network based on resnet to receive high level of accuracy.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('As the data available for training is large, we can build a data generator class that can help in building the input data \\\nin batches to avoid overwhelming the system memory.')\nprint('We can extend the keras sequence class to get the data generator.')\n\nclass LungDataGenerator(keras.utils.Sequence):\n    def __init__(self, patientIds, batch_size=32, dim=(1024,1024), shuffle=True):\n        self.patientIds = patientIds\n        self.batch_size = batch_size\n        self.dim=dim\n        self.shuffle = shuffle\n        self.on_epoch_end()\n        \n    def __load__(self, patientId):\n        # Load the dicom file as a numpy array\n        dicom_image = pydicom.dcmread(trainDicomFiles[patientId]).pixel_array\n\n        # Create an empty mask to hold the list of bounding boxes as image segments.\n        bounding_box_mask = np.zeros(dicom_image.shape) \n\n        # Generate the mask using the label data frame\n        targetLabelDf = trainLabelAndBoundingBoxDf.loc[patientId]\n        if targetLabelDf['target'] == 1:\n            for box in targetLabelDf['boundingBox']:\n                x, y, width, height = [int(item) for item in box]\n                bounding_box_mask[y:y+height, x:x+width] = 1\n\n        # Resizing the image for reduction (to help build faster model).\n        dicom_image = cv2.resize(dicom_image, (self.dim[0], self.dim[1]))\n        bounding_box_mask = cv2.resize(bounding_box_mask, (self.dim[0], self.dim[1]))\n        dicom_image = np.expand_dims(dicom_image, -1)\n        bounding_box_mask = np.expand_dims(bounding_box_mask, -1)\n        target = targetLabelDf['target']\n        return dicom_image, bounding_box_mask, target\n\n    def __len__(self):\n        # Represents the number of batches per epoch\n        return int(len(self.patientIds) / self.batch_size)\n    \n    def __getitem__(self, index):\n        # Generates the indexes associated with one batch\n        targetPatientIds = self.patientIds[index * self.batch_size : (index + 1) * self.batch_size]\n        \n        items = [self.__load__(id) for id in targetPatientIds]\n\n        # Zip the images and masks for the target patients.\n        imgs, masks, targets = zip(*items)\n\n        # Create numpy batch\n        imgs = np.array(imgs)\n        masks = np.array(masks)\n        targets = np.array(targets)\n        return imgs, masks, targets\n        \n    def on_epoch_end(self):\n        # Shuffles the data after every epoch\n        if self.shuffle:\n            np.random.shuffle(self.patientIds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Now we can define the metrics for loss reduction as below:')\nprint('Define IOU or jaccard loss function')\ndef iou_loss(y_true, y_pred):\n    y_true = tf.reshape(y_true, [-1])\n    y_pred = tf.reshape(y_pred, [-1])\n    intersection = tf.reduce_sum(y_true * y_pred)\n    score = (intersection + 1.) / (tf.reduce_sum(y_true) + tf.reduce_sum(y_pred) - intersection + 1.)\n    return 1 - score\n\nprint('Combine BCE and IOU loss')\ndef iou_bce_loss(y_true, y_pred):\n    return 0.5 * keras.losses.binary_crossentropy(y_true, y_pred) + 0.5 * iou_loss(y_true, y_pred)\n    #return iou_loss(y_true, y_pred)\n\nprint('Get mean IOU as a metric')\ndef mean_iou(y_true, y_pred):\n    y_pred = tf.round(y_pred)\n    intersect = tf.reduce_sum(y_true * y_pred, axis = [1, 2, 3])\n    union = tf.reduce_sum(y_true, axis = [1, 2, 3]) + tf.reduce_sum(y_pred, axis = [1, 2, 3])\n    smooth = tf.ones(tf.shape(intersect))\n    return tf.reduce_mean((intersect + smooth) / (union - intersect + smooth))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('We would create the convolutional neural network using resnet blocks.\\\nThe model consists of a downsampling block (consisting of Batch normalization, leaky ReLU, Convolutional 2D and max pool layer) \\\nand a resnet block (consisting of batch normalization, leaky ReLU and convolutional 2D layers). The downsampling and resnet \\\nblocks are repeated based on configuration parameters of n_blocks and depth. In the end we upsample the data points to the same \\\nsize as before in order to get the target mask.')\ndef create_downsample(channels, inputs):\n    x = keras.layers.BatchNormalization(momentum=0.9)(inputs)\n    x = keras.layers.LeakyReLU(0)(x)\n    x = keras.layers.Conv2D(channels, 1, padding='same', use_bias=False)(x)\n    x = keras.layers.MaxPool2D(2)(x)\n    return x\n\ndef create_resblock(channels, inputs):\n    x = keras.layers.BatchNormalization(momentum=0.9)(inputs)\n    x = keras.layers.LeakyReLU(0)(x)\n    x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False)(x)\n    x = keras.layers.BatchNormalization(momentum=0.9)(x)\n    x = keras.layers.LeakyReLU(0)(x)\n    x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False)(x)\n    return keras.layers.add([x, inputs])\n\ndef create_network(input_size, channels, n_blocks=2, depth=4, isDropoutAdded=False):\n    # Input\n    inputs = keras.Input(shape=(input_size, input_size, 1))\n\n    x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False)(inputs)\n    # Residual blocks\n    for d in range(depth):\n        channels = channels * 2\n        x = create_downsample(channels, x)\n        \n        if isDropoutAdded:\n            # Add a dropout layer post down-sampling\n            x = keras.layers.Dropout(0.2)(x)\n        \n        for b in range(n_blocks):\n            x = create_resblock(channels, x)\n            if isDropoutAdded:\n                # Add a dropout layer post each resnet block\n                x = keras.layers.Dropout(0.2)(x)\n\n    # Output\n    x = keras.layers.BatchNormalization(momentum=0.9)(x)\n    x = keras.layers.LeakyReLU(0)(x)\n    x = keras.layers.Conv2D(1, 1, activation='sigmoid')(x)\n    outputs = keras.layers.UpSampling2D(2**depth)(x)\n    model = keras.Model(inputs=inputs, outputs=outputs)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"print('Generating the network instance.')\nmodel = create_network(input_size=256, channels=32, n_blocks=2, depth=4, isDropoutAdded=False)\n# Using the optimizer with default learning rate of 0.01.\nlearn_rate = 0.001\nadamOpt = keras.optimizers.Adam(learning_rate=learn_rate)\nmodel.compile(optimizer=adamOpt, loss=iou_bce_loss, metrics=['accuracy', mean_iou])\n\nepoch_count = 5\n\n# Define cosine annealing\ndef cosine_annealing(x):\n    lr = learn_rate\n    epochs = epoch_count\n    return lr * (np.cos(np.pi * x / epochs) + 1.) / 2\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Training the model.')\nlearning_rate = tf.keras.callbacks.LearningRateScheduler(cosine_annealing)\n\ntraining_data_percent = 0.7\ntraining_data_end_index = int(trainLabelAndBoundingBoxDf.shape[0] * training_data_percent)\ntraining_patientIds_box = trainLabelAndBoundingBoxDf['patientId'][0:training_data_end_index]\nvalidation_patientIds_box = trainLabelAndBoundingBoxDf['patientId'][training_data_end_index:]\n\nparams = {'dim': (256, 256), 'batch_size': 32, 'shuffle': True}\ntraining_data_generator_box = LungDataGenerator(training_patientIds_box, **params)\nvalidation_data_generator_box = LungDataGenerator(validation_patientIds_box, **params)\n\nhistory = model.fit(training_data_generator_box, validation_data=validation_data_generator_box, callbacks=[learning_rate], epochs=epoch_count, workers=4, use_multiprocessing=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Below is the representation of variation of loss and mean_iou for training and validation data.')\nplt.figure(figsize=(15,6))\nplt.subplot(121)\nplt.plot(history.epoch, history.history[\"loss\"], label=\"Train loss\")\nplt.plot(history.epoch, history.history[\"val_loss\"], label=\"Valid loss\")\nplt.legend()\nplt.subplot(122)\nplt.plot(history.epoch, history.history[\"mean_iou\"], label=\"Train iou\")\nplt.plot(history.epoch, history.history[\"val_mean_iou\"], label=\"Valid iou\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, classification_report\nprint('We can generate the y_pred and y_true arrays to hold the target values (predicted and actual) for all the validation \\\ndata points.')\ndef get_confusion_matrix(target_model, data_generator, isDisplayRequired=False):\n    y_pred = []\n    y_true = []\n\n    for imgs, masks, targets in data_generator:\n        predictions = target_model.predict(imgs)\n\n        # Loop through the predictions along with the actual targets\n        for img, mask, prediction, target in zip(imgs, masks, predictions, targets):\n            # Apply threshold to the predicted mask\n            comp = prediction[:,:,0] > 0.5\n\n            # Apply connected components.\n            comp = measure.label(comp)\n\n            # Take the initial prediction as 0\n            predTarget = 0\n\n            # Apply bounding boxes\n            for region in measure.regionprops(comp):\n                # If a region is found mark the target value as 1\n                predTarget = 1\n\n            y_pred.append(predTarget)\n            y_true.append(target)\n            \n    cm = confusion_matrix(y_true, y_pred)\n    \n    if isDisplayRequired:\n        print('Below is the confusion matrix:')\n        ax = sns.heatmap(cm, annot=True, fmt='d')\n        plt.show()\n\n        print('\\nBelow is the classification report based on the actual and predicted target values:')\n        print(classification_report(y_true, y_pred, labels=[0,1]))\n        \n    return cm\n\ncm = get_confusion_matrix(model, validation_data_generator_box, True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('We can now analyze the predictions on the validation data.')\nprops = dict(boxstyle='round', facecolor='wheat', alpha=0.5)\nfor imgs, masks, targets in validation_data_generator_box:\n    # Predict a batch of images\n    predictions = model.predict(imgs)\n    # Create a plot for the images\n    fig, axes = plt.subplots(3, 5, figsize=(20, 15))\n    axes = axes.ravel()\n    axidx = 0\n    \n    # Loop through the batch\n    for img, mask, prediction, target in zip(imgs, masks, predictions, targets):\n        axes[axidx].imshow(img[:,:,0])\n        \n        # Apply threshold to true mask\n        comp = mask[:,:,0] > 0.5\n        \n        # Apply connected components.\n        comp = measure.label(comp)\n        \n        # Apply bounding boxes\n        for region in measure.regionprops(comp):\n            y1, x1, y2, x2 = region.bbox\n            height = y2-y1\n            width = x2-x1\n            axes[axidx].add_patch(patches.Rectangle((x1, y1), width, height, linewidth=2, edgecolor='b', facecolor='none'))\n            \n        # Apply threshold to the predicted mask\n        comp = prediction[:,:,0] > 0.5\n        \n        # Apply connected components.\n        comp = measure.label(comp)\n        \n        predTarget = 0\n        \n        # Apply bounding boxes\n        for region in measure.regionprops(comp):\n            # If a region is found mark the target value as 1\n            predTarget = 1\n            y1, x1, y2, x2 = region.bbox\n            height = y2-y1\n            width = x2-x1\n            axes[axidx].add_patch(patches.Rectangle((x1, y1), width, height, linewidth=2, edgecolor='r', facecolor='none'))\n        \n        axes[axidx].text(0.05, 0.95, f'Actual target: {target} \\nPredicted target: {predTarget}', transform=axes[axidx].transAxes, fontsize=14, verticalalignment='top', bbox=props)\n        axidx += 1\n        if(axidx >= 15):\n            break;\n        \n    plt.show()\n    \n    # Display only one plot per batch\n    break\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('We have successfully built a Pneumonia Detection model using ResNet architecture that is able to predict Pneumonia along with the lung opacity region from the CXR images with an accuracy of 96%.')\nprint('We can improve the performance of the current model by performing hyper-parameter tuning for the model by \\\n\\n1) Adjusting the number of ResNet blocks used for the model\\\n\\n2) Tuning the depth for the model.\\\n\\n3) Changing the target size to which the input images are reduced.\\\n\\n4) Fine tuning the number of channels used throughout the model.')\n\nprint('\\nAdditionally, we would look at some pre-built model libraries available for masked R-CNN like matterport to see if it \\\nimproves the performance.')\n\nprint('\\nWe would also try using a pre-trained model like mobilenet by training the last few layers of the model.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('First, we can try the U-Net model for bounding box detection.')\ndef unet(input_size=(256,256,1)):\n    inputs = keras.Input(input_size)\n    \n    conv1 = keras.layers.Conv2D(32, (3, 3), activation='relu', padding='same')(inputs)\n    conv1 = keras.layers.Conv2D(32, (3, 3), activation='relu', padding='same')(conv1)\n    pool1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(conv1)\n\n    conv2 = keras.layers.Conv2D(64, (3, 3), activation='relu', padding='same')(pool1)\n    conv2 = keras.layers.Conv2D(64, (3, 3), activation='relu', padding='same')(conv2)\n    pool2 = keras.layers.MaxPooling2D(pool_size=(2, 2))(conv2)\n\n    conv3 = keras.layers.Conv2D(128, (3, 3), activation='relu', padding='same')(pool2)\n    conv3 = keras.layers.Conv2D(128, (3, 3), activation='relu', padding='same')(conv3)\n    pool3 = keras.layers.MaxPooling2D(pool_size=(2, 2))(conv3)\n\n    conv4 = keras.layers.Conv2D(256, (3, 3), activation='relu', padding='same')(pool3)\n    conv4 = keras.layers.Conv2D(256, (3, 3), activation='relu', padding='same')(conv4)\n    pool4 = keras.layers.MaxPooling2D(pool_size=(2, 2))(conv4)\n\n    conv5 = keras.layers.Conv2D(512, (3, 3), activation='relu', padding='same')(pool4)\n    conv5 = keras.layers.Conv2D(512, (3, 3), activation='relu', padding='same')(conv5)\n\n    up6 = keras.layers.concatenate([keras.layers.Conv2DTranspose(256, (2, 2), strides=(2, 2), padding='same')(conv5), conv4], axis=3)\n    conv6 = keras.layers.Conv2D(256, (3, 3), activation='relu', padding='same')(up6)\n    conv6 = keras.layers.Conv2D(256, (3, 3), activation='relu', padding='same')(conv6)\n\n    up7 = keras.layers.concatenate([keras.layers.Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(conv6), conv3], axis=3)\n    conv7 = keras.layers.Conv2D(128, (3, 3), activation='relu', padding='same')(up7)\n    conv7 = keras.layers.Conv2D(128, (3, 3), activation='relu', padding='same')(conv7)\n\n    up8 = keras.layers.concatenate([keras.layers.Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(conv7), conv2], axis=3)\n    conv8 = keras.layers.Conv2D(64, (3, 3), activation='relu', padding='same')(up8)\n    conv8 = keras.layers.Conv2D(64, (3, 3), activation='relu', padding='same')(conv8)\n\n    up9 = keras.layers.concatenate([keras.layers.Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(conv8), conv1], axis=3)\n    conv9 = keras.layers.Conv2D(32, (3, 3), activation='relu', padding='same')(up9)\n    conv9 = keras.layers.Conv2D(32, (3, 3), activation='relu', padding='same')(conv9)\n\n    conv10 = keras.layers.Conv2D(1, (1, 1), activation='sigmoid')(conv9)\n\n    return keras.Model(inputs=[inputs], outputs=[conv10])\n\nmodel_unet = unet(input_size=(256,256,1))\nmodel_unet.compile(optimizer='adam', loss=iou_bce_loss, metrics=['accuracy', mean_iou])\nmodel_unet.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Training the U-Net model.') \nlearning_rate = tf.keras.callbacks.LearningRateScheduler(cosine_annealing)\n\ntraining_data_percent = 0.7\ntraining_data_end_index = int(trainLabelAndBoundingBoxDf.shape[0] * training_data_percent)\ntraining_patientIds_box = trainLabelAndBoundingBoxDf['patientId'][0:training_data_end_index]\nvalidation_patientIds_box = trainLabelAndBoundingBoxDf['patientId'][training_data_end_index:]\n\nparams = {'dim': (256, 256), 'batch_size': 32, 'shuffle': True}\ntraining_data_generator_box = LungDataGenerator(training_patientIds_box, **params)\nvalidation_data_generator_box = LungDataGenerator(validation_patientIds_box, **params)\n\nhistory_unet = model_unet.fit(training_data_generator_box, validation_data=validation_data_generator_box, callbacks=[learning_rate], epochs=epoch_count, workers=4, use_multiprocessing=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Below is the representation of variation of loss, accuracy and mean_iou for training and validation data.')\nplt.figure(figsize=(15,6))\nplt.subplot(121)\nplt.plot(history_unet.epoch, history_unet.history[\"loss\"], label=\"Train loss\")\nplt.plot(history_unet.epoch, history_unet.history[\"val_loss\"], label=\"Valid loss\")\nplt.legend()\nplt.subplot(122)\nplt.plot(history_unet.epoch, history_unet.history[\"mean_iou\"], label=\"Train iou\")\nplt.plot(history_unet.epoch, history_unet.history[\"val_mean_iou\"], label=\"Valid iou\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('We can now analyze the confusion matrix and classification report for U-Net model.')\ncm_unet = get_confusion_matrix(model_unet, validation_data_generator_box, True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('We can now analyze the predictions on the validation data.')\nprops = dict(boxstyle='round', facecolor='wheat', alpha=0.5)\nfor imgs, masks, targets in validation_data_generator_box:\n    # Predict a batch of images\n    predictions = model_unet.predict(imgs)\n    # Create a plot for the images\n    fig, axes = plt.subplots(3, 5, figsize=(20, 15))\n    axes = axes.ravel()\n    axidx = 0\n    \n    # Loop through the batch\n    for img, mask, prediction, target in zip(imgs, masks, predictions, targets):\n        axes[axidx].imshow(img[:,:,0])\n        \n        # Apply threshold to true mask\n        comp = mask[:,:,0] > 0.5\n        \n        # Apply connected components.\n        comp = measure.label(comp)\n        \n        # Apply bounding boxes\n        for region in measure.regionprops(comp):\n            y1, x1, y2, x2 = region.bbox\n            height = y2-y1\n            width = x2-x1\n            axes[axidx].add_patch(patches.Rectangle((x1, y1), width, height, linewidth=2, edgecolor='b', facecolor='none'))\n            \n        # Apply threshold to the predicted mask\n        comp = prediction[:,:,0] > 0.5\n        \n        # Apply connected components.\n        comp = measure.label(comp)\n        \n        predTarget = 0\n        \n        # Apply bounding boxes\n        for region in measure.regionprops(comp):\n            # If a region is found mark the target value as 1\n            predTarget = 1\n            y1, x1, y2, x2 = region.bbox\n            height = y2-y1\n            width = x2-x1\n            axes[axidx].add_patch(patches.Rectangle((x1, y1), width, height, linewidth=2, edgecolor='r', facecolor='none'))\n        \n        axes[axidx].text(0.05, 0.95, f'Actual target: {target} \\nPredicted target: {predTarget}', transform=axes[axidx].transAxes, fontsize=14, verticalalignment='top', bbox=props)\n        axidx += 1\n        if(axidx >= 15):\n            break;\n        \n    plt.show()\n    \n    # Display only one plot per batch\n    break\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Now we can compare the results of the Res-Net and U-Net architectures used.')\nprint('First we can compare the changes in the loss and mean_iou for validation data for ResNet vs U-Net architectures')\nplt.figure(figsize=(15,6))\nplt.subplot(121)\nplt.plot(history.epoch, history.history[\"val_loss\"], label=\"Res-Net Validation loss\")\nplt.plot(history.epoch, history_unet.history[\"val_loss\"], label=\"U-Net Validation loss\")\nplt.legend()\nplt.subplot(122)\nplt.plot(history.epoch, history.history[\"val_mean_iou\"], label=\"Res-Net Validation IOU\")\nplt.plot(history.epoch, history_unet.history[\"val_mean_iou\"], label=\"U-Net Validation IOU\")\nplt.legend()\nplt.show()\n\nprint('Now we can compare the confusion matrices for Res-Net and U-Net architectures.')\ndef annotate_plot(plot):\n    for p in plot.patches:\n        plot.annotate(format(p.get_height(), '.2f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')  \n\ncm_comparison_df = pd.DataFrame(np.array([['Res-Net', cm[0][0], cm[0, 1], cm[1,0], cm[1,1]], \n                                          ['U-Net', cm_unet[0][0], cm_unet[0, 1], cm_unet[1,0], cm_unet[1,1]]])\n                               , columns=['architecture', 'True_Negatives', 'False_Positives', 'False_Negatives', 'True_Positives'])\n\n\ncm_comparison_df['architecture'] = cm_comparison_df['architecture'].astype('category')\n\nfig, axes = plt.subplots(2, 2)\ntrue_negative_plot = sns.barplot(x='architecture', y='True_Negatives', data=cm_comparison_df, ax=axes[0][0])\nfalse_positive_plot = sns.barplot(x='architecture', y='False_Positives', data=cm_comparison_df, ax=axes[0][1])\nfalse_negative_plot = sns.barplot(x='architecture', y='False_Negatives', data=cm_comparison_df, ax=axes[1][0])\ntrue_positive_plot = sns.barplot(x='architecture', y='True_Positives', data=cm_comparison_df, ax=axes[1][1])\nannotate_plot(true_negative_plot)\nannotate_plot(false_positive_plot)\nannotate_plot(false_negative_plot)\nannotate_plot(true_positive_plot)\nfig.set_figwidth(15)\nfig.set_figheight(12)\nfig.suptitle('Confusion matrix comparison for Res-Net and U-Net architectures')\nfig.tight_layout(pad=5.0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('We can see that the overall performance for the Res-Net architecture is better than the U-Net architecture. One point to \\\nnote is that the number of false negatives is greater in case of Res-Net. It is important to lower the number of false negatives \\\nas we may end up calling out a patient with pneumonia as not having issues, however, as the target classification is done based \\\non the presence of bounding boxes, we can attempt to improve the IOU metric for ResNet in order to reduce the number of false \\\nnegatives.')\nprint('1) Changing the number of channels.')\nparams = {'dim': (256, 256), 'batch_size': 32, 'shuffle': True}\ntraining_data_generator_box = LungDataGenerator(training_patientIds_box, **params)\nvalidation_data_generator_box = LungDataGenerator(validation_patientIds_box, **params)\n\nmodel_channel = create_network(input_size=256, channels=16, n_blocks=2, depth=4, isDropoutAdded=False)\n\nmodel_channel.compile(optimizer='adam', loss=iou_bce_loss, metrics=['accuracy', mean_iou])\nhistory_channel = model_channel.fit(training_data_generator_box, validation_data=validation_data_generator_box, callbacks=[learning_rate], epochs=epoch_count, workers=4, use_multiprocessing=True)\ncm_channel = get_confusion_matrix(model_channel, validation_data_generator_box, False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('2) Changing the number of blocks.')\nparams = {'dim': (256, 256), 'batch_size': 32, 'shuffle': True}\ntraining_data_generator_box = LungDataGenerator(training_patientIds_box, **params)\nvalidation_data_generator_box = LungDataGenerator(validation_patientIds_box, **params)\nmodel_blocks = create_network(input_size=256, channels=32, n_blocks=4, depth=4, isDropoutAdded=False)\n\nmodel_blocks.compile(optimizer='adam', loss=iou_bce_loss, metrics=['accuracy', mean_iou])\nhistory_blocks = model_blocks.fit(training_data_generator_box, validation_data=validation_data_generator_box, callbacks=[learning_rate], epochs=epoch_count, workers=4, use_multiprocessing=True)\ncm_blocks = get_confusion_matrix(model_blocks, validation_data_generator_box, False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('3) Changing the depth of the model.')\nparams = {'dim': (256, 256), 'batch_size': 32, 'shuffle': True}\ntraining_data_generator_box = LungDataGenerator(training_patientIds_box, **params)\nvalidation_data_generator_box = LungDataGenerator(validation_patientIds_box, **params)\nmodel_depth = create_network(input_size=256, channels=32, n_blocks=2, depth=6, isDropoutAdded=False)\n\nmodel_depth.compile(optimizer='adam', loss=iou_bce_loss, metrics=['accuracy', mean_iou])\nhistory_depth = model_depth.fit(training_data_generator_box, validation_data=validation_data_generator_box, callbacks=[learning_rate], epochs=epoch_count, workers=4, use_multiprocessing=True)\ncm_depth = get_confusion_matrix(model_depth, validation_data_generator_box, False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('4) Changing the dimension of the image that is given as input.')\nparams = {'dim': (512, 512), 'batch_size': 16, 'shuffle': True}\ntraining_data_generator_box = LungDataGenerator(training_patientIds_box, **params)\nvalidation_data_generator_box = LungDataGenerator(validation_patientIds_box, **params)\nmodel_dim = create_network(input_size=512, channels=32, n_blocks=2, depth=4, isDropoutAdded=False)\n\nmodel_dim.compile(optimizer='adam', loss=iou_bce_loss, metrics=['accuracy', mean_iou])\nhistory_dim = model_dim.fit(training_data_generator_box, validation_data=validation_data_generator_box, callbacks=[learning_rate], epochs=epoch_count, workers=4, use_multiprocessing=True)\ncm_dim = get_confusion_matrix(model_dim, validation_data_generator_box, False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('5) Adding dropout layers to ResNet based model.')\nparams = {'dim': (256, 256), 'batch_size': 32, 'shuffle': True}\ntraining_data_generator_box = LungDataGenerator(training_patientIds_box, **params)\nvalidation_data_generator_box = LungDataGenerator(validation_patientIds_box, **params)\nmodel_dropout = create_network(input_size=256, channels=32, n_blocks=2, depth=4, isDropoutAdded=True)\n\nmodel_dropout.compile(optimizer='adam', loss=iou_bce_loss, metrics=['accuracy', mean_iou])\nhistory_dropout = model_dropout.fit(training_data_generator_box, validation_data=validation_data_generator_box, callbacks=[learning_rate], epochs=epoch_count, workers=4, use_multiprocessing=True)\ncm_dropout = get_confusion_matrix(model_dropout, validation_data_generator_box, False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('6) Changing the learning rate to 0.01.')\nparams = {'dim': (256, 256), 'batch_size': 32, 'shuffle': True}\ntraining_data_generator_box = LungDataGenerator(training_patientIds_box, **params)\nvalidation_data_generator_box = LungDataGenerator(validation_patientIds_box, **params)\nmodel_lr = create_network(input_size=256, channels=32, n_blocks=2, depth=4, isDropoutAdded=False)\n\nlearn_rate = 0.01\nadamOpt01 = keras.optimizers.Adam(learning_rate=learn_rate)\nmodel_lr.compile(optimizer=adamOpt01, loss=iou_bce_loss, metrics=['accuracy', mean_iou])\n\n# Define cosine annealing\ndef cosine_annealing01(x):\n    lr = learn_rate\n    epochs = epoch_count\n    return lr * (np.cos(np.pi * x / epochs) + 1.) / 2\n\nlearning_rate01 = tf.keras.callbacks.LearningRateScheduler(cosine_annealing01)\nhistory_lr = model_lr.fit(training_data_generator_box, validation_data=validation_data_generator_box, callbacks=[learning_rate01], epochs=epoch_count, workers=4, use_multiprocessing=True)\ncm_lr = get_confusion_matrix(model_lr, validation_data_generator_box, False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"targetModel = 'Res-Net'\nprint(f'Now we can compare the results of the variations of the {targetModel} architecture.')\nprint(f'First we can compare the changes in the loss and mean_iou for validation data for different {targetModel} architectures')\nplt.figure(figsize=(15,6))\nplt.subplot(121)\nplt.plot(history_channel.epoch, history_channel.history[\"val_loss\"], label=\"Validation loss Post channel tuning\")\nplt.plot(history_blocks.epoch, history_blocks.history[\"val_loss\"], label=\"Validation loss Post number of blocks tuning\")\nplt.plot(history_depth.epoch, history_depth.history[\"val_loss\"], label=\"Validation loss Post depth tuning\")\nplt.plot(history_dim.epoch, history_dim.history[\"val_loss\"], label=\"Validation loss Post dimension tuning\")\nplt.plot(history_dropout.epoch, history_dropout.history[\"val_loss\"], label=\"Validation loss Post adding dropouts\")\nplt.plot(history_lr.epoch, history_lr.history[\"val_loss\"], label=\"Validation loss Post changing learning rate\")\nplt.legend()\nplt.subplot(122)\nplt.plot(history_channel.epoch, history_channel.history[\"val_mean_iou\"], label=\"Validation IOU Post channel tuning\")\nplt.plot(history_blocks.epoch, history_blocks.history[\"val_mean_iou\"], label=\"Validation IOU Post number of blocks tuning\")\nplt.plot(history_depth.epoch, history_depth.history[\"val_mean_iou\"], label=\"Validation IOU Post depth tuning\")\nplt.plot(history_dim.epoch, history_dim.history[\"val_mean_iou\"], label=\"Validation IOU Post dimension tuning\")\nplt.plot(history_dropout.epoch, history_dropout.history[\"val_mean_iou\"], label=\"Validation IOU Post adding dropouts\")\nplt.plot(history_lr.epoch, history_lr.history[\"val_mean_iou\"], label=\"Validation IOU Post changing learning rate\")\nplt.legend()\nplt.show()\n\nprint(f'Now we can compare the confusion matrices for validation data for different {targetModel} architectures.')\ndef annotate_plot(plot):\n    for p in plot.patches:\n        plot.annotate(format(p.get_height(), '.2f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')  \n\ndef get_cm_array(cmObj):\n    return cmObj[0][0], cmObj[0, 1], cmObj[1,0], cmObj[1,1]\n\nget_cm_array(cm_channel)\n\ncm_comparison_df = pd.DataFrame(np.array([[f'{targetModel}_channel', cm_channel[0][0], cm_channel[0, 1], cm_channel[1,0], cm_channel[1,1]], \n                                          [f'{targetModel}_blocks', cm_blocks[0][0], cm_blocks[0, 1], cm_blocks[1,0], cm_blocks[1,1]],\n                                          [f'{targetModel}_depth', cm_depth[0][0], cm_depth[0, 1], cm_depth[1,0], cm_depth[1,1]],\n                                          [f'{targetModel}_dim', cm_dim[0][0], cm_dim[0, 1], cm_dim[1,0], cm_dim[1,1]],\n                                          [f'{targetModel}_dropouts', cm_dropout[0][0], cm_dropout[0, 1], cm_dropout[1,0], cm_dropout[1,1]],\n                                          [f'{targetModel}_learningrate', cm_lr[0][0], cm_lr[0, 1], cm_lr[1,0], cm_lr[1,1]]\n                                         ])\n                               , columns=['architecture', 'True_Negatives', 'False_Positives', 'False_Negatives', 'True_Positives'])\n\n\ncm_comparison_df['architecture'] = cm_comparison_df['architecture'].astype('category')\n\nfig, axes = plt.subplots(2, 2)\ntrue_negative_plot = sns.barplot(x='architecture', y='True_Negatives', data=cm_comparison_df, ax=axes[0][0])\nfalse_positive_plot = sns.barplot(x='architecture', y='False_Positives', data=cm_comparison_df, ax=axes[0][1])\nfalse_negative_plot = sns.barplot(x='architecture', y='False_Negatives', data=cm_comparison_df, ax=axes[1][0])\ntrue_positive_plot = sns.barplot(x='architecture', y='True_Positives', data=cm_comparison_df, ax=axes[1][1])\nannotate_plot(true_negative_plot)\nannotate_plot(false_positive_plot)\nannotate_plot(false_negative_plot)\nannotate_plot(true_positive_plot)\naxes[0,0].tick_params('x', labelrotation=45)\naxes[0,1].tick_params('x', labelrotation=45)\naxes[1,0].tick_params('x', labelrotation=45)\naxes[1,1].tick_params('x', labelrotation=45)\nfig.set_figwidth(15)\nfig.set_figheight(12)\nfig.suptitle('Confusion matrix comparison for varitions of Res-Net.')\nfig.tight_layout(pad=5.0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Conclusion"},{"metadata":{},"cell_type":"markdown","source":"Based on the metrics for validation loss, mean IoU and confusion matrix comparison, the **Resnet model** with modified dimension of image as **512\\*512** has the least number of **false negatives (approx. 150)** and has a **low validation loss of around 0.05**. Thereby, we can consider the resnet model variation having the initial image size as 512\\*512 as the target model for the peneumonia detection model.\n\n***Below are few interesting observations:***\n1. We were able to identify the bounding boxes from the mask by segmenting the image with the help of **skimage.measure (regionprops) method**.\n2. On **increasing the depth** of the Res-Net architecture, the number of **false negatives increased drastically** leading us to avoid the increase in depth of the model.\n\n***Below are few challenges we faced during model generation and tuning:***\n1. As the images were heavy (1024X1024), we had to **reduce the size of the image** in order to cope up with the restricted memory availability.\n2. The model training was taking excessive time because of which we made use of **GPU acceleration** to speed up the process.8\n\n***Further model optimization plan:***\n1. We can try combining the **variations in the hyperparameters (number of blocks/channel/depth, dimension, dropouts, learning rate)** to get a better mean IoU and loss. \n2. Additionally, we can utilize **image augmentation** on the class of images with pneumonia to balance the entire data set. This can help improve the model.\n3. For further optimizations of the selected model for real-life usage, we can train the model on images from the problem domain using **transfer learning** (making the last few layers of the model learnable).\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}