{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Intro\n### What is Pneumonia\n### What is the aim of the project","metadata":{}},{"cell_type":"markdown","source":"Example\n![Various Class of Chest XRays](https://ars.els-cdn.com/content/image/1-s2.0-S0092867418301545-figs6_lrg.jpg)","metadata":{}},{"cell_type":"markdown","source":"### Importing all the python Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport sys\nimport math\nimport cv2\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport multiprocessing\n\n# To calculate accuracy measures and confusion matrix\nfrom sklearn import metrics\n# To get Recall and precision values\nfrom sklearn.metrics import classification_report\n\n# !pip install -q pydicom\n# After installing pydicom. This is needed to load .dcm files\nimport pydicom\nimport pydicom as dcm\nfrom pydicom import dcmread\n\nfrom matplotlib import pyplot\nimport matplotlib.patches as patches\n\nfrom skimage import measure\nfrom skimage.transform import resize\n\nimport tensorflow.keras.utils as pltUtil\nfrom tensorflow.keras.utils import Sequence\n\nfrom tensorflow.keras.layers import Layer\nfrom tensorflow.keras.layers import Concatenate, UpSampling2D, Conv2D, Reshape, BatchNormalization, Activation\nfrom tensorflow.keras.models import Model\n\nfrom tensorflow.keras.applications.mobilenet import MobileNet\nfrom tensorflow.keras.applications.mobilenet import preprocess_input \n\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\n\nfrom tensorflow.keras.applications.resnet import ResNet50\nfrom tensorflow.keras.applications.resnet import preprocess_input as resnetProcess_input\n","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:01.639359Z","iopub.execute_input":"2021-11-16T15:05:01.639714Z","iopub.status.idle":"2021-11-16T15:05:06.528517Z","shell.execute_reply.started":"2021-11-16T15:05:01.639683Z","shell.execute_reply":"2021-11-16T15:05:06.52769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Setting up project path","metadata":{"editable":false}},{"cell_type":"code","source":"# Load train set image metadata\ndataDirPath = '../input/rsna-pneumonia-detection-challenge/'\n\nTrain_Image_path = dataDirPath + 'stage_2_train_images'\n\nSAVED_FILES_ROOT = '../input/reportfiles/'\nRES_FILES_ROOT = '../input/resreportfiles/'","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:06.530363Z","iopub.execute_input":"2021-11-16T15:05:06.530736Z","iopub.status.idle":"2021-11-16T15:05:06.536002Z","shell.execute_reply.started":"2021-11-16T15:05:06.530699Z","shell.execute_reply":"2021-11-16T15:05:06.535117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Function definations","metadata":{"editable":false}},{"cell_type":"markdown","source":"#### Function to load the metadata from images","metadata":{"editable":false}},{"cell_type":"code","source":"class ImageMetadata():\n    \"\"\"Structure to hold image metadata\n\n    Arguments:\n        setName -- name of the data set\n\n        file -- file name\n    \"\"\"\n    \n    def __init__(self, setName, file):\n        # print(name, file)\n        # dataset name(train or test)\n        self.setName = setName\n        # image file name\n        self.file = file\n\n    def __repr__(self):\n        return self.imagePath()\n\n    def imagePath(self):\n        return os.path.join(self.setName, self.file) \n    \n\n# function to load image metadada   \ndef loadImageMetadata(dataSetName):\n    \"\"\"Load image file names from the images from dataset\n\n    Arguments:\n        dataSetName -- path of the data set folder\n    \"\"\"\n    imageMetadata = []\n    for f in os.listdir(dataSetName):\n        # Check file extension. Allow only .dcm files.\n        ext = os.path.splitext(f)[1]\n        if ext == '.dcm' :\n            imageMetadata.append(ImageMetadata(dataSetName, f))\n            \n    # Return an array of filenames\n    return np.array(imageMetadata)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:06.537682Z","iopub.execute_input":"2021-11-16T15:05:06.538367Z","iopub.status.idle":"2021-11-16T15:05:06.552006Z","shell.execute_reply.started":"2021-11-16T15:05:06.538329Z","shell.execute_reply":"2021-11-16T15:05:06.551187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Functions to load image and patientId(file name) from the given path","metadata":{"editable":false}},{"cell_type":"code","source":"def loadImage(path):\n    \"\"\"Load image from the path \n\n    Arguments:\n        path -- path of the image\n    \"\"\"\n    img = pydicom.dcmread(path)\n    \n    # Return image\n    return img\n\n\ndef getImgId(imgPath) :\n    \"\"\"Get image name which is patientId from the given image path \n\n    Arguments:\n        imgPath -- path of the image\n    \"\"\"\n    \n    # Return patientId \n    return str(imgPath).split(\".dcm\")[0].split(\"/\")[4]","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:06.553469Z","iopub.execute_input":"2021-11-16T15:05:06.554064Z","iopub.status.idle":"2021-11-16T15:05:06.562862Z","shell.execute_reply.started":"2021-11-16T15:05:06.553887Z","shell.execute_reply":"2021-11-16T15:05:06.561885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Getting Metadata Information","metadata":{"editable":false}},{"cell_type":"code","source":"trainSetImageMetadata = loadImageMetadata(Train_Image_path)\n\nprint(\"trainSetImageMetadata.shape : \", trainSetImageMetadata.shape)\n\nprint(\"Sample image path : \", trainSetImageMetadata[0])","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:06.567275Z","iopub.execute_input":"2021-11-16T15:05:06.567738Z","iopub.status.idle":"2021-11-16T15:05:07.405525Z","shell.execute_reply.started":"2021-11-16T15:05:06.567699Z","shell.execute_reply":"2021-11-16T15:05:07.404567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Insights from Metadata\n\nThere are 26684 images in the Training data\nThese are DICOM Images which has pixel information as well as several tags added to it like patientid, age,gender etc.","metadata":{"editable":false}},{"cell_type":"markdown","source":"#### Loaging a sample Image","metadata":{"editable":false}},{"cell_type":"code","source":"def printOnIndex(index):\n    imgIndex = index\n    imgPath = trainSetImageMetadata[imgIndex]\n    imgPath = imgPath.imagePath()\n    imgData = loadImage(imgPath)\n    pyplot.imshow(imgData.pixel_array, cmap=pyplot.cm.bone)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-16T15:05:07.408535Z","iopub.execute_input":"2021-11-16T15:05:07.408901Z","iopub.status.idle":"2021-11-16T15:05:07.414359Z","shell.execute_reply.started":"2021-11-16T15:05:07.408863Z","shell.execute_reply":"2021-11-16T15:05:07.413301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"printOnIndex(11) #Normal","metadata":{"execution":{"iopub.status.busy":"2021-11-16T15:05:07.415857Z","iopub.execute_input":"2021-11-16T15:05:07.416213Z","iopub.status.idle":"2021-11-16T15:05:07.707801Z","shell.execute_reply.started":"2021-11-16T15:05:07.416178Z","shell.execute_reply":"2021-11-16T15:05:07.706787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"printOnIndex(10) #No Lung Opacity / Not Normal","metadata":{"execution":{"iopub.status.busy":"2021-11-16T15:05:07.708924Z","iopub.execute_input":"2021-11-16T15:05:07.709253Z","iopub.status.idle":"2021-11-16T15:05:07.936563Z","shell.execute_reply.started":"2021-11-16T15:05:07.709215Z","shell.execute_reply":"2021-11-16T15:05:07.935631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"printOnIndex(5) #Lung Opacity","metadata":{"execution":{"iopub.status.busy":"2021-11-16T15:05:07.938087Z","iopub.execute_input":"2021-11-16T15:05:07.9387Z","iopub.status.idle":"2021-11-16T15:05:08.175585Z","shell.execute_reply.started":"2021-11-16T15:05:07.938656Z","shell.execute_reply":"2021-11-16T15:05:08.174715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Preparing Dataset with patient id and respective image paths","metadata":{"editable":false}},{"cell_type":"code","source":"trainSetImageMetadata_df = pd.DataFrame(trainSetImageMetadata, columns=[\"Path\"])\ntrainSetImageMetadata_df.head(2)\n\nimageIdPaths = pd.DataFrame(columns=[\"patientId\", \"imgPath\"])\nimageIdPaths[\"patientId\"] = trainSetImageMetadata_df[\"Path\"].apply(getImgId)\nimageIdPaths[\"imgPath\"] = trainSetImageMetadata_df[\"Path\"]\n\nprint(\"imageIdPaths\", imageIdPaths.shape)\nimageIdPaths.head(2)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:08.177364Z","iopub.execute_input":"2021-11-16T15:05:08.177716Z","iopub.status.idle":"2021-11-16T15:05:08.296216Z","shell.execute_reply.started":"2021-11-16T15:05:08.177679Z","shell.execute_reply":"2021-11-16T15:05:08.295348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis on Train Labels and Detail Info CSV data sets","metadata":{"editable":false}},{"cell_type":"markdown","source":"**Analyzing Detailed Classes CSV file**","metadata":{"editable":false}},{"cell_type":"markdown","source":"Import CSV file","metadata":{}},{"cell_type":"code","source":"classesPath =  dataDirPath + 'stage_2_detailed_class_info.csv'\n\ndetailedClasses = pd.read_csv(classesPath)\n\ndetailedClasses.head(2)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:08.297617Z","iopub.execute_input":"2021-11-16T15:05:08.298234Z","iopub.status.idle":"2021-11-16T15:05:08.351915Z","shell.execute_reply.started":"2021-11-16T15:05:08.298195Z","shell.execute_reply":"2021-11-16T15:05:08.351079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check for missing values","metadata":{}},{"cell_type":"code","source":"detailedClasses.isna().apply(pd.value_counts)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:08.353199Z","iopub.execute_input":"2021-11-16T15:05:08.353745Z","iopub.status.idle":"2021-11-16T15:05:08.375188Z","shell.execute_reply.started":"2021-11-16T15:05:08.353707Z","shell.execute_reply":"2021-11-16T15:05:08.374364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"No Missing Values found","metadata":{"editable":false}},{"cell_type":"markdown","source":"Check the shape of data frame","metadata":{}},{"cell_type":"code","source":"print(\"detailedClasses.shape : \", detailedClasses.shape)\n\n# File has 30227 rows and 2 columns - PatientID & Class","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:08.376453Z","iopub.execute_input":"2021-11-16T15:05:08.377047Z","iopub.status.idle":"2021-11-16T15:05:08.382159Z","shell.execute_reply.started":"2021-11-16T15:05:08.37701Z","shell.execute_reply":"2021-11-16T15:05:08.38122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Find number of unique patients.","metadata":{}},{"cell_type":"code","source":"print(\"Unique patientIds : \", detailedClasses['patientId'].nunique())","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:08.383645Z","iopub.execute_input":"2021-11-16T15:05:08.384318Z","iopub.status.idle":"2021-11-16T15:05:08.403403Z","shell.execute_reply.started":"2021-11-16T15:05:08.384281Z","shell.execute_reply":"2021-11-16T15:05:08.402578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Total number of unique patients in data - 26684\n\n**Observation - As we have total 30227 records and out of that 26684 are unique records, this shows presence of multiple records for some patients**","metadata":{"editable":false}},{"cell_type":"markdown","source":"Check unique classes","metadata":{}},{"cell_type":"code","source":"print(\"Unique patientIds : \", detailedClasses['class'].nunique(), )\n\nprint(detailedClasses['class'].unique)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:08.404956Z","iopub.execute_input":"2021-11-16T15:05:08.405292Z","iopub.status.idle":"2021-11-16T15:05:08.416549Z","shell.execute_reply.started":"2021-11-16T15:05:08.405259Z","shell.execute_reply":"2021-11-16T15:05:08.415585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3 Unique classes observed \n1 - No Lung Opacity/Not Normal,\n2 - Normal,\n3 - Lung Opacity","metadata":{"editable":false}},{"cell_type":"markdown","source":"#### Analyzing Train Lables Dataset","metadata":{"editable":false}},{"cell_type":"markdown","source":"Import and read label data","metadata":{}},{"cell_type":"code","source":"labelsPath = dataDirPath + 'stage_2_train_labels.csv'\n\ntrainLabels = pd.read_csv(labelsPath)\n\ntrainLabels.head(2)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:08.704596Z","iopub.execute_input":"2021-11-16T15:05:08.710779Z","iopub.status.idle":"2021-11-16T15:05:08.788714Z","shell.execute_reply.started":"2021-11-16T15:05:08.710737Z","shell.execute_reply":"2021-11-16T15:05:08.787845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check for missing values","metadata":{}},{"cell_type":"code","source":"trainLabels.isna().apply(pd.value_counts)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:08.793549Z","iopub.execute_input":"2021-11-16T15:05:08.796732Z","iopub.status.idle":"2021-11-16T15:05:08.835866Z","shell.execute_reply.started":"2021-11-16T15:05:08.796682Z","shell.execute_reply":"2021-11-16T15:05:08.835089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Observation - For around 20672 patients Bounding box cordinates not available where as for 9555 patients its avaialable","metadata":{"editable":false}},{"cell_type":"code","source":"trainLabels[trainLabels['Target']==0].head(2)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:08.84038Z","iopub.execute_input":"2021-11-16T15:05:08.843247Z","iopub.status.idle":"2021-11-16T15:05:08.866744Z","shell.execute_reply.started":"2021-11-16T15:05:08.840813Z","shell.execute_reply":"2021-11-16T15:05:08.865983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Few records have observed with missing values in **x, y, width** and **height** coulmn, but no missing values observed in **patientId** and **Target**.\n\nAlso this is observed such missing columns are present for those records with **Target** as '0'.\n\n**x, y, width** and **height** columns have the information for bounding boxes in Images where Penumonia is detected.\n\n**Explaination on missing values - These are not the missing values instead it is expected not to have Bounding Box co-ordinates for those images where Pneumonia is not detected (Target - '0')**\n\nHence concluding there are no missing values in this dataset as well","metadata":{"editable":false}},{"cell_type":"markdown","source":"Check for number of unique Patients","metadata":{}},{"cell_type":"code","source":"print(\"Unique patientIds : \", trainLabels['patientId'].nunique(), )","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:08.871376Z","iopub.execute_input":"2021-11-16T15:05:08.874218Z","iopub.status.idle":"2021-11-16T15:05:08.899178Z","shell.execute_reply.started":"2021-11-16T15:05:08.871827Z","shell.execute_reply":"2021-11-16T15:05:08.898406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Total Unique patients found - 26684\n\nThis is same as the number of patients in Detailed CSV sheet hence both sheets share the information for same patients","metadata":{"editable":false}},{"cell_type":"markdown","source":"Check for Target values","metadata":{}},{"cell_type":"code","source":"print(\"Unique patientIds : \", trainLabels['Target'].nunique(), )\n\nprint(\"Unique patientIds : \", trainLabels['Target'].unique(), )","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:08.903692Z","iopub.execute_input":"2021-11-16T15:05:08.906531Z","iopub.status.idle":"2021-11-16T15:05:08.919443Z","shell.execute_reply.started":"2021-11-16T15:05:08.906483Z","shell.execute_reply":"2021-11-16T15:05:08.918097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Trail label has only 2 target variables [0 & 1] \n\n**Conclusion - In Train labels only two target variables are present 0 & 1, where as in Detailed_Info sheet we have 3 classes.**","metadata":{"editable":false}},{"cell_type":"markdown","source":"**As we have 3 Classes in Detailed_Info dataset and 2 Target Variables in Train_Labels, concatenating to get better insight into the data**","metadata":{"editable":false}},{"cell_type":"markdown","source":"Step 1 -> Sorting both the datasets based on patientId","metadata":{"editable":false}},{"cell_type":"code","source":"trainLabels.sort_values(\"patientId\", inplace=True)\ndetailedClasses.sort_values(\"patientId\", inplace=True)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:08.922714Z","iopub.execute_input":"2021-11-16T15:05:08.923278Z","iopub.status.idle":"2021-11-16T15:05:09.028142Z","shell.execute_reply.started":"2021-11-16T15:05:08.923239Z","shell.execute_reply":"2021-11-16T15:05:09.027452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Step 2 -> Concatenating the data\n\n* To get the Target and classes into one dataframe","metadata":{"editable":false}},{"cell_type":"code","source":"Combined_Data = pd.concat([trainLabels, detailedClasses[\"class\"]], axis=1, sort=False)\nCombined_Data.head(3)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:09.029533Z","iopub.execute_input":"2021-11-16T15:05:09.029883Z","iopub.status.idle":"2021-11-16T15:05:09.04832Z","shell.execute_reply.started":"2021-11-16T15:05:09.029846Z","shell.execute_reply":"2021-11-16T15:05:09.047381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Validating the concatenation results**","metadata":{"editable":false}},{"cell_type":"code","source":"Combined_Data.shape","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:09.049591Z","iopub.execute_input":"2021-11-16T15:05:09.050138Z","iopub.status.idle":"2021-11-16T15:05:09.056717Z","shell.execute_reply.started":"2021-11-16T15:05:09.050099Z","shell.execute_reply":"2021-11-16T15:05:09.055446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Combined_Data.isna().apply(pd.value_counts)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:09.058485Z","iopub.execute_input":"2021-11-16T15:05:09.05929Z","iopub.status.idle":"2021-11-16T15:05:09.091731Z","shell.execute_reply.started":"2021-11-16T15:05:09.059253Z","shell.execute_reply":"2021-11-16T15:05:09.090898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Combined_Data[Combined_Data[\"Target\"] == 1].isna().apply(pd.value_counts)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:09.09346Z","iopub.execute_input":"2021-11-16T15:05:09.093851Z","iopub.status.idle":"2021-11-16T15:05:09.120282Z","shell.execute_reply.started":"2021-11-16T15:05:09.093801Z","shell.execute_reply":"2021-11-16T15:05:09.11956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Combined_Data[Combined_Data[\"Target\"] == 0].isna().apply(pd.value_counts)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:09.121754Z","iopub.execute_input":"2021-11-16T15:05:09.122268Z","iopub.status.idle":"2021-11-16T15:05:09.153239Z","shell.execute_reply.started":"2021-11-16T15:05:09.122229Z","shell.execute_reply":"2021-11-16T15:05:09.152281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Combined_Data[Combined_Data[\"class\"] == \"Lung Opacity\"].isna().apply(pd.value_counts)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:09.155099Z","iopub.execute_input":"2021-11-16T15:05:09.155462Z","iopub.status.idle":"2021-11-16T15:05:09.184899Z","shell.execute_reply.started":"2021-11-16T15:05:09.155428Z","shell.execute_reply":"2021-11-16T15:05:09.183932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From Above Analysis our concatenated data is correct","metadata":{"editable":false}},{"cell_type":"markdown","source":"### Visualization\n","metadata":{"editable":false}},{"cell_type":"code","source":"sns.countplot(x=\"class\",hue=\"class\",data=Combined_Data)","metadata":{"execution":{"iopub.status.busy":"2021-11-16T15:05:09.186312Z","iopub.execute_input":"2021-11-16T15:05:09.186685Z","iopub.status.idle":"2021-11-16T15:05:09.364855Z","shell.execute_reply.started":"2021-11-16T15:05:09.18665Z","shell.execute_reply":"2021-11-16T15:05:09.364018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A look at our Target Distribution.","metadata":{}},{"cell_type":"code","source":"class_info=Combined_Data['Target'].value_counts()\nexplode = (0.1,0.0)  \n\nfig1, ax1 = plt.subplots(figsize=(5,5))\nax1.pie(class_info.values, explode=explode, labels=['Normal','Lung Opacity'], autopct='%1.1f%%',\n        shadow=True, startangle=100)\nax1.axis('equal') \nplt.title('Target Distribution')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-16T15:05:09.366111Z","iopub.execute_input":"2021-11-16T15:05:09.36663Z","iopub.status.idle":"2021-11-16T15:05:09.423967Z","shell.execute_reply.started":"2021-11-16T15:05:09.366584Z","shell.execute_reply":"2021-11-16T15:05:09.423109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Density Plot\n\nFor the class Lung Opacity, corresponding to values of Target = 1, we plot the density of x, y, width and height.","metadata":{}},{"cell_type":"code","source":"target1 = Combined_Data[Combined_Data['Target']==1]\nsns.set_style('whitegrid')\nplt.figure()\nfig, ax = plt.subplots(2,2,figsize=(12,12))\nsns.distplot(target1['x'],kde=True,bins=50, color=\"red\", ax=ax[0,0])\nsns.distplot(target1['y'],kde=True,bins=50, color=\"blue\", ax=ax[0,1])\nsns.distplot(target1['width'],kde=True,bins=50, color=\"green\", ax=ax[1,0])\nsns.distplot(target1['height'],kde=True,bins=50, color=\"magenta\", ax=ax[1,1])\nlocs, labels = plt.xticks()\nplt.tick_params(axis='both', which='major', labelsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-16T15:05:09.425639Z","iopub.execute_input":"2021-11-16T15:05:09.426013Z","iopub.status.idle":"2021-11-16T15:05:10.584727Z","shell.execute_reply.started":"2021-11-16T15:05:09.425978Z","shell.execute_reply":"2021-11-16T15:05:10.582519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style('darkgrid')\nsns.set_context('notebook', font_scale=1.2)\nplt.rcParams['figure.figsize'] = [14, 8]\nplt.rcParams['lines.linewidth'] = 2.5\n\n# Get all data\ntr = pd.read_csv('../input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv')\ntr['aspect_ratio'] = (tr['width']/tr['height'])\ntr['area'] = tr['width'] * tr['height']\n\ndef get_info(patientId, root_dir='../input/rsna-pneumonia-detection-challenge/stage_2_train_images/'):\n    fn = os.path.join(root_dir, f'{patientId}.dcm')\n    dcm_data = pydicom.read_file(fn)\n    return {'age': dcm_data.PatientAge, \n            'gender': dcm_data.PatientSex, \n            'id': os.path.basename(fn).split('.')[0],\n            'pixel_spacing': float(dcm_data.PixelSpacing[0]),\n            'mean_black_pixels': np.mean(dcm_data.pixel_array == 0)}\n\npatient_ids = list(tr.patientId.unique())\nwith multiprocessing.Pool(4) as pool:\n    result = pool.map(get_info, patient_ids)\n    \ndemo = pd.DataFrame(result)\ndemo['gender'] = demo['gender'].astype('category')\ndemo['age'] = demo['age'].astype(int)\n\ntr = (tr.merge(demo, left_on='patientId', right_on='id', how='left')\n        .drop(columns='id'))","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:05:10.591532Z","iopub.execute_input":"2021-11-16T15:05:10.592061Z","iopub.status.idle":"2021-11-16T15:10:38.98894Z","shell.execute_reply.started":"2021-11-16T15:05:10.592019Z","shell.execute_reply":"2021-11-16T15:10:38.987944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g = sns.FacetGrid(col='Target', hue='gender', \n                  data=tr.drop_duplicates(subset=['patientId']), \n                  height=9, palette=dict(F=\"red\", M=\"blue\"))\n_ = g.map(sns.distplot, 'age', hist_kws={'alpha': 0.3}).add_legend()\n_ = g.fig.suptitle(\"What is the age distribution by gender and illness?\", y=1.02, fontsize=20)\n\n#use z score to figure out ","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:38.990769Z","iopub.execute_input":"2021-11-16T15:10:38.991152Z","iopub.status.idle":"2021-11-16T15:10:40.25055Z","shell.execute_reply.started":"2021-11-16T15:10:38.991101Z","shell.execute_reply":"2021-11-16T15:10:40.249736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"boxes_per_patient = tr.groupby('patientId')['Target'].sum()\n\nax = boxes_per_patient.value_counts().plot.bar()\n_ = ax.set_title('How many cases are there per image?')\n_ = ax.set_xlabel('Number of cases')\n_ = ax.xaxis.set_tick_params(rotation=0)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:40.251963Z","iopub.execute_input":"2021-11-16T15:10:40.252547Z","iopub.status.idle":"2021-11-16T15:10:40.474342Z","shell.execute_reply.started":"2021-11-16T15:10:40.252506Z","shell.execute_reply":"2021-11-16T15:10:40.473439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare data for training\n\n\nStep 1 -> \n* Convert data to only two classes, 'Normal' and 'Lung Opacity'\n* Splitting the data in three parts, train, validation and test sets.\n* Replace all 'NaN' values to zero(0)\n","metadata":{"editable":false}},{"cell_type":"code","source":"# Conver data to only two classes, 'Normal' and 'Lung Opacity'\nCombined_Data[\"class\"].replace(\"No Lung Opacity / Not Normal\", \"Normal\", inplace=True)\nCombined_Data.head(3)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:40.475739Z","iopub.execute_input":"2021-11-16T15:10:40.476345Z","iopub.status.idle":"2021-11-16T15:10:40.494617Z","shell.execute_reply.started":"2021-11-16T15:10:40.476304Z","shell.execute_reply":"2021-11-16T15:10:40.493793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_CombinedData = Combined_Data[0:15000]\nvalidate_CombinedData = Combined_Data[15000:25000]\ntest_CombinedData = Combined_Data[25000:30227]\n\nprint(\"train_CombinedData.shape : \", train_CombinedData.shape)\nprint(\"validate_CombinedData.shape : \", validate_CombinedData.shape)\nprint(\"test_CombinedData.shape : \", test_CombinedData.shape)\n\nprint(\"\\nunique train patients : \", train_CombinedData[\"patientId\"].nunique())\nprint(\"unique validate patients : \", validate_CombinedData[\"patientId\"].nunique())\nprint(\"unique test patients : \", test_CombinedData[\"patientId\"].nunique())\n\nprint(\"\\nTotal unique patients : \", imageIdPaths[\"patientId\"].nunique())\nprint(\"Total of unique train and test : \", train_CombinedData[\"patientId\"].nunique() + validate_CombinedData[\"patientId\"].nunique() + test_CombinedData[\"patientId\"].nunique())\n\nprint(\"\\nLast from train set : \", train_CombinedData.iloc[14999][\"patientId\"])\nprint(\"First from validate set : \", validate_CombinedData.iloc[0][\"patientId\"])\nprint(\"\\nLast from validate set : \", validate_CombinedData.iloc[9999][\"patientId\"])\nprint(\"First from test set : \", test_CombinedData.iloc[0][\"patientId\"])\n\n# Set all NaN values to 0 in train and test data sets. While training NaN will not have any meaning.\n#    * x, y, width and hight values as zero(0) means no bounding box.\ntrain_CombinedData.fillna(0, inplace=True)\nvalidate_CombinedData.fillna(0, inplace=True)\ntest_CombinedData.fillna(0, inplace=True)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:40.496281Z","iopub.execute_input":"2021-11-16T15:10:40.496634Z","iopub.status.idle":"2021-11-16T15:10:40.556695Z","shell.execute_reply.started":"2021-11-16T15:10:40.496598Z","shell.execute_reply":"2021-11-16T15:10:40.554772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imageIdPaths.sort_values(\"patientId\", inplace=True)\n\ntrain_imageIdPaths = imageIdPaths[0:13163]\nvalidate_imageIdPaths = imageIdPaths[13163:21764]\ntest_imageIdPaths = imageIdPaths[21764:26684]\n\nprint(\"train_imageIdPaths.shape : \", train_imageIdPaths.shape)\nprint(\"validate_imageIdPaths.shape : \", validate_imageIdPaths.shape)\nprint(\"test_imageIdPaths.shape : \", test_imageIdPaths.shape)\n\nprint(\"\\nunique train patients : \", train_imageIdPaths[\"patientId\"].nunique())\nprint(\"unique validate patients : \", validate_imageIdPaths[\"patientId\"].nunique())\nprint(\"unique test patients : \", test_imageIdPaths[\"patientId\"].nunique())\n\nprint(\"\\nTotal unique patients : \", imageIdPaths[\"patientId\"].nunique())\nprint(\"Total of unique train and test : \", train_imageIdPaths[\"patientId\"].nunique() + validate_imageIdPaths[\"patientId\"].nunique() + test_imageIdPaths[\"patientId\"].nunique())\n\nprint(\"\\nLast from train set : \", train_imageIdPaths.iloc[13162][\"patientId\"])\nprint(\"First from validate set : \", validate_imageIdPaths.iloc[0][\"patientId\"])\nprint(\"Last from validate set : \", validate_imageIdPaths.iloc[8600][\"patientId\"])\nprint(\"First from test set : \", test_imageIdPaths.iloc[0][\"patientId\"])","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:40.558517Z","iopub.execute_input":"2021-11-16T15:10:40.558984Z","iopub.status.idle":"2021-11-16T15:10:40.640698Z","shell.execute_reply.started":"2021-11-16T15:10:40.558945Z","shell.execute_reply":"2021-11-16T15:10:40.640067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x=\"Target\",hue=\"class\",data=train_CombinedData)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:40.642786Z","iopub.execute_input":"2021-11-16T15:10:40.643424Z","iopub.status.idle":"2021-11-16T15:10:40.80073Z","shell.execute_reply.started":"2021-11-16T15:10:40.643387Z","shell.execute_reply":"2021-11-16T15:10:40.799926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x=\"Target\",hue=\"class\",data=validate_CombinedData)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:40.801923Z","iopub.execute_input":"2021-11-16T15:10:40.802421Z","iopub.status.idle":"2021-11-16T15:10:40.964432Z","shell.execute_reply.started":"2021-11-16T15:10:40.802379Z","shell.execute_reply":"2021-11-16T15:10:40.96357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x=\"Target\",hue=\"class\",data=test_CombinedData)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:40.965721Z","iopub.execute_input":"2021-11-16T15:10:40.966253Z","iopub.status.idle":"2021-11-16T15:10:41.136343Z","shell.execute_reply.started":"2021-11-16T15:10:40.966214Z","shell.execute_reply":"2021-11-16T15:10:41.135371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Observation\n    * train, validation and test sets are not well balanced. \n    * Images with 'Lung Opacity' are less in proportion in all these sets.","metadata":{"editable":false}},{"cell_type":"markdown","source":"# Build UNet","metadata":{"editable":false}},{"cell_type":"markdown","source":"### Constants","metadata":{"editable":false}},{"cell_type":"code","source":"IMAGE_SIZE = 224\n\nIMG_WIDTH = 1024\nIMG_HEIGHT = 1024\n\nTRAIN_BATCH_SIZE = 10\nTEST_BATCH_SIZE = 10\n\nALPHA = 1.0","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.137656Z","iopub.execute_input":"2021-11-16T15:10:41.13819Z","iopub.status.idle":"2021-11-16T15:10:41.143917Z","shell.execute_reply.started":"2021-11-16T15:10:41.138148Z","shell.execute_reply":"2021-11-16T15:10:41.142721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Function and Class definations","metadata":{"editable":false}},{"cell_type":"code","source":"# define iou or jaccard loss function\ndef iou_loss(y_true, y_pred):\n    \"\"\"Get Intersection over Union(IoU) ratio from ground truth and predicted masks.\n    Arguments:\n        y_true -- ground truth mask\n        \n        y_pred -- predicted mask\n    \"\"\"\n    y_true = tf.reshape(y_true, [-1])\n    y_pred = tf.reshape(y_pred, [-1])\n    intersection = tf.reduce_sum(y_true * y_pred)\n    score = (intersection + 1.) / (tf.reduce_sum(y_true) + tf.reduce_sum(y_pred) - intersection + 1.)\n    return 1 - score\n\n# mean iou as a metric\ndef mean_iou(y_true, y_pred):\n    \"\"\"Get mean Intersection over Union(IoU) ratio from ground truth and predicted masks.\n    Arguments:\n        y_true -- ground truth mask\n        \n        y_pred -- predicted mask\n    \"\"\"\n    y_pred = tf.round(y_pred)    \n    intersect = tf.reduce_sum(y_true * y_pred, axis=[1]) \n    union = tf.reduce_sum(y_true, axis=[1]) + tf.reduce_sum(y_pred, axis=[1])\n    smooth = tf.ones(tf.shape(intersect))\n    return tf.reduce_mean((intersect + smooth) / (union - intersect + smooth))","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.145458Z","iopub.execute_input":"2021-11-16T15:10:41.146094Z","iopub.status.idle":"2021-11-16T15:10:41.158507Z","shell.execute_reply.started":"2021-11-16T15:10:41.146057Z","shell.execute_reply":"2021-11-16T15:10:41.157723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def iouFromCoords(boxA, boxB) :\n    \"\"\"Get Intersection over Union(IoU) ratio from ground truth and predicted box coordinates.\n    Arguments:\n        boxA -- ground truth mask\n        \n        boxB -- predicted mask\n    \"\"\"\n    # determine the (x, y)-coordinates of the intersection rectangle\n    xA = max(boxA[0], boxB[0])\n    yA = max(boxA[1], boxB[1])\n    xB = min(boxA[2], boxB[2])\n    yB = min(boxA[3], boxB[3])\n\n    # compute the area of intersection rectangle\n    interArea = abs(max((xB - xA, 0)) * max((yB - yA), 0))\n    if interArea == 0:\n        return 0\n    # compute the area of both the prediction and ground-truth\n    # rectangles\n    boxAArea = abs((boxA[2] - boxA[0]) * (boxA[3] - boxA[1]))\n    boxBArea = abs((boxB[2] - boxB[0]) * (boxB[3] - boxB[1]))\n\n    # compute the intersection over union by taking the intersection\n    # area and dividing it by the sum of prediction + ground-truth\n    # areas - the interesection area\n    iou = interArea / float(boxAArea + boxBArea - interArea)\n\n    # return the intersection over union value\n    return iou","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.159928Z","iopub.execute_input":"2021-11-16T15:10:41.160518Z","iopub.status.idle":"2021-11-16T15:10:41.172815Z","shell.execute_reply.started":"2021-11-16T15:10:41.160454Z","shell.execute_reply":"2021-11-16T15:10:41.171954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef showMaskedImage(_imageSet, _maskSet, _index) :\n    \"\"\"To show image with imposing mask on it for the given index, form the given sets of images and masks.\n    Arguments:\n        _imageSet -- set of images\n        \n        _maskSet -- set of masks\n        \n        _index -- index of a set/collection\n    \"\"\"\n    maskImage = _imageSet[_index]\n\n    #pyplot.imshow(maskImage[:,:,0], cmap=pyplot.cm.bone)\n    maskImage[:,:,0] = _maskSet[_index] * _imageSet[_index][:,:,0]\n    maskImage[:,:,1] = _maskSet[_index] * _imageSet[_index][:,:,1]\n    maskImage[:,:,2] = _maskSet[_index] * _imageSet[_index][:,:,2]\n\n    pyplot.imshow(maskImage[:,:,0], cmap=pyplot.cm.bone)\n","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.174374Z","iopub.execute_input":"2021-11-16T15:10:41.175105Z","iopub.status.idle":"2021-11-16T15:10:41.185845Z","shell.execute_reply.started":"2021-11-16T15:10:41.175062Z","shell.execute_reply":"2021-11-16T15:10:41.185035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass UNetTrainGenerator(Sequence):\n    \"\"\"Generator class to get data batches for training model. Extends Sequence class.\n        \n    Arguments:\n        _imageIdPaths -- dataframe having patientId and image paths to load image for the patientId\n        \n        _CombinedData -- dataframe having patientId, image labels, target and class values combined together\n        \n        idx -- index of a batch\n    \"\"\"\n    def __init__(self, _imageIdPaths, _CombinedData):       \n        self.pids = _CombinedData[\"patientId\"].to_numpy()\n        self.imgIdPaths = _imageIdPaths\n        self.coords = _CombinedData[[\"x\", \"y\", \"width\", \"height\"]].to_numpy()\n        # Resize Bounding box\n        self.coords = self.coords * IMAGE_SIZE / IMG_WIDTH\n        \n\n    def __len__(self):\n        return math.ceil(len(self.coords) / TRAIN_BATCH_SIZE)\n    \n\n    def __getitem__(self, idx): # Get a batch\n        batch_coords = self.coords[idx * TRAIN_BATCH_SIZE:(idx + 1) * TRAIN_BATCH_SIZE] # Image coords\n        batch_pids = self.pids[idx * TRAIN_BATCH_SIZE:(idx + 1) * TRAIN_BATCH_SIZE] # Image pids    \n        \n        batch_images = np.zeros((len(batch_pids), IMAGE_SIZE, IMAGE_SIZE, 3), dtype=np.float32)\n        batch_masks = np.zeros((len(batch_pids), IMAGE_SIZE, IMAGE_SIZE))\n        for _indx, _pid in enumerate(batch_pids):\n            _path = self.imgIdPaths[self.imgIdPaths[\"patientId\"] == _pid][\"imgPath\"].array[0]\n            _imgData = loadImage(str(_path)) # Read image\n            img = _imgData.pixel_array \n            \n            # Resize image\n            resized_img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)\n    \n            #print(\"batch_images[_indx] shape :\", batch_images[_indx][:,:,0].shape)\n            # preprocess image for the batch\n            batch_images[_indx][:,:,0] = preprocess_input(np.array(resized_img[:,:], dtype=np.float32)) # Convert to float32 array\n            batch_images[_indx][:,:,1] = preprocess_input(np.array(resized_img[:,:], dtype=np.float32)) # Convert to float32 array\n            batch_images[_indx][:,:,2] = preprocess_input(np.array(resized_img[:,:], dtype=np.float32)) # Convert to float32 array  \n            \n            x = int(batch_coords[_indx, 0])\n            y = int(batch_coords[_indx, 1])\n            width = int(batch_coords[_indx, 2])\n            height = int(batch_coords[_indx, 3])\n            \n            batch_masks[_indx][y:y+height, x:x+width] = 1\n            \n        # Returns images and ground truth masks for training\n        return batch_images, batch_masks","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.187563Z","iopub.execute_input":"2021-11-16T15:10:41.187998Z","iopub.status.idle":"2021-11-16T15:10:41.21209Z","shell.execute_reply.started":"2021-11-16T15:10:41.187957Z","shell.execute_reply":"2021-11-16T15:10:41.211352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generator for Predicting/Testing model\nclass UNetTestGenerator(Sequence):\n    \"\"\"Generator class to get data batches for testing model. Used to get batches of test data used for getting predictions.\n        \n    Arguments:\n        _imageIdPaths -- dataframe having patientId and image paths to load image for the patientId\n        \n        _CombinedData -- dataframe having patientId, image labels, target and class values combined together\n        \n        idx -- index of a batch\n    \"\"\"\n    def __init__(self, _imageIdPaths, _CombinedData):       \n        self.pids = _CombinedData[\"patientId\"].to_numpy()\n        self.imgIdPaths = _imageIdPaths\n        self.coords = _CombinedData[[\"x\", \"y\", \"width\", \"height\", \"Target\"]].to_numpy() #for (1024, 1024)\n        self.classes = _CombinedData[\"class\"]\n        # Resize Bounding box\n        self.coordsOrig = self.coords #for (1024, 1024)\n        self.coords = self.coords * IMAGE_SIZE / IMG_WIDTH   #for (224, 224)\n        \n\n    def __len__(self):\n        # Returns total number of batches\n        return math.ceil(len(self.coords) / TEST_BATCH_SIZE)\n    \n\n    def __getitem__(self, idx): # Get a batch\n        batch_coords = self.coords[idx * TEST_BATCH_SIZE:(idx + 1) * TEST_BATCH_SIZE] # Image coords for (224, 224)\n        batch_coordsOrig = self.coordsOrig[idx * TEST_BATCH_SIZE:(idx + 1) * TEST_BATCH_SIZE] # Image coords for (1024, 1024)\n        batch_pids = self.pids[idx * TEST_BATCH_SIZE:(idx + 1) * TEST_BATCH_SIZE] # Image pids    \n        batch_classes = self.classes[idx * TEST_BATCH_SIZE:(idx + 1) * TEST_BATCH_SIZE] # Image classes           \n        \n        batch_images = np.zeros((len(batch_pids), IMAGE_SIZE, IMAGE_SIZE, 3), dtype=np.float32)\n        batch_masks = np.zeros((len(batch_pids), IMAGE_SIZE, IMAGE_SIZE))\n        for _indx, _pid in enumerate(batch_pids):\n            _path = self.imgIdPaths[self.imgIdPaths[\"patientId\"] == _pid][\"imgPath\"].array[0]\n            _imgData = loadImage(str(_path)) # Read image\n            img = _imgData.pixel_array \n            \n            # Resize image\n            resized_img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA) #(224, 224)\n            #resized_img = cv2.resize(img[200:824, 200:824], dsize=(IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)\n    \n            #print(\"batch_images[_indx] shape :\", batch_images[_indx][:,:,0].shape)\n            # preprocess image for the batch\n            batch_images[_indx][:,:,0] = preprocess_input(np.array(resized_img[:,:], dtype=np.float32)) # Convert to float32 array\n            batch_images[_indx][:,:,1] = preprocess_input(np.array(resized_img[:,:], dtype=np.float32)) # Convert to float32 array\n            batch_images[_indx][:,:,2] = preprocess_input(np.array(resized_img[:,:], dtype=np.float32)) # Convert to float32 array  \n            \n            x = int(batch_coords[_indx, 0])\n            y = int(batch_coords[_indx, 1])\n            width = int(batch_coords[_indx, 2])\n            height = int(batch_coords[_indx, 3])\n            target = int(batch_coords[_indx, 4])\n            \n            batch_coords[_indx, 0] = x\n            batch_coords[_indx, 1] = y \n            batch_coords[_indx, 2] = width \n            batch_coords[_indx, 3] = height    \n            batch_coords[_indx, 4] = target \n            \n            batch_masks[_indx][y:y+height, x:x+width] = 1\n\n        # Returns images, ground truth masks, patientIds, resized-coordinates, class targets and ground truth coordinates/lables.   \n        return batch_images, batch_masks, batch_pids, batch_coords, batch_classes, batch_coordsOrig  #for (224, 224) and (1024, 1024)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.214469Z","iopub.execute_input":"2021-11-16T15:10:41.214748Z","iopub.status.idle":"2021-11-16T15:10:41.236548Z","shell.execute_reply.started":"2021-11-16T15:10:41.214713Z","shell.execute_reply":"2021-11-16T15:10:41.235687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef create_UNetModel(trainable=True):\n    \"\"\"Function to create UNet architecture with MobileNet.\n        \n    Arguments:\n        trainable -- Flag to make layers trainable. Default value is 'True'.\n    \"\"\"\n    # Get all layers with 'imagenet' weights\n    model = MobileNet(input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3), include_top=False, alpha=ALPHA, weights=\"imagenet\") \n    # Top layer is last layer of the model\n    \n    # Make all layers trainable\n    for layer in model.layers:\n        layer.trainable = trainable\n\n    # Add all the UNET layers here\n    convLayer_112by112 = model.get_layer(\"conv_pw_1_relu\").output\n    convLayer_56by56 = model.get_layer(\"conv_pw_3_relu\").output\n    convLayer_28by28 = model.get_layer(\"conv_pw_5_relu\").output\n    convLayer_14by14 = model.get_layer(\"conv_pw_11_relu\").output\n    convLayer_7by7 = model.get_layer(\"conv_pw_13_relu\").output\n    # The last layer of mobilenet model is of dimensions (7x7x1024)\n\n    # Start upsampling from 7x7 to 14x14 ...up to 224x224 to form UNET\n    # concatinate with the original image layer of the same size from MobileNet\n    x = Concatenate()([UpSampling2D()(convLayer_7by7), convLayer_14by14])\n    x = Concatenate()([UpSampling2D()(x), convLayer_28by28])\n    x = Concatenate()([UpSampling2D()(x), convLayer_56by56])\n    x = Concatenate()([UpSampling2D()(x), convLayer_112by112])\n    x = UpSampling2D(name=\"unet_last\")(x) # upsample to 224x224\n\n    # Add classification layer\n    x = Conv2D(1, kernel_size=1, activation=\"sigmoid\", name=\"masks\")(x)\n    x = Reshape((IMAGE_SIZE, IMAGE_SIZE))(x) \n\n    return Model(inputs=model.input, outputs=x)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.23791Z","iopub.execute_input":"2021-11-16T15:10:41.238617Z","iopub.status.idle":"2021-11-16T15:10:41.25476Z","shell.execute_reply.started":"2021-11-16T15:10:41.238572Z","shell.execute_reply":"2021-11-16T15:10:41.254003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef conv_block_simple(prevlayer, filters, prefix, strides=(1, 1)):\n    \"\"\"Function to create part of UNet architecture with ResNet50. \n    Adds convolutional layer followed by Batch Normalization and Activation layers.\n        \n    Arguments:\n        prevlayer -- previous layer of the convolution block\n        \n        filters -- number of filters for convolution\n        \n        prefix -- prefix for the layer name\n        \n        strides -- convolution stride. Default is 1x1.\n    \"\"\"\n    conv = Conv2D(filters, (3, 3), padding=\"same\", kernel_initializer=\"he_normal\", strides=strides, name=prefix + \"_conv\")(prevlayer)\n    conv = BatchNormalization(name=prefix + \"_bn\")(conv)\n    conv = Activation('relu', name=prefix + \"_activation\")(conv)\n    \n    # Returns the built layers of the block.\n    return conv","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.256382Z","iopub.execute_input":"2021-11-16T15:10:41.256938Z","iopub.status.idle":"2021-11-16T15:10:41.267812Z","shell.execute_reply.started":"2021-11-16T15:10:41.256898Z","shell.execute_reply":"2021-11-16T15:10:41.266976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Trained all layers of ResNet50 starting with initial pre-trained  weights on ‘imagenet’. All U-Net decoder layer weights are trained from zero. \n* We have image dataset very different from the ‘imagenet’. Here identifying features/segments for pattern is important, than finding an object. Hence decided to train all the weights, starting with pre-loaded ‘imagenet’ weights.\n","metadata":{"editable":false}},{"cell_type":"code","source":"\ndef create_ResNetUNetModel(trainable=True):\n    \"\"\"Function to create UNet architecture with ResNet50.\n        \n    Arguments:\n        trainable -- Flag to make layers trainable. Default value is 'True'.\n    \"\"\"\n    resnetLayers = ResNet50(weights='imagenet', input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3), include_top=False) # Load pre-trained Resnet\n    # Top layer is last layer of the model\n\n    for layer in resnetLayers.layers:\n        layer.trainable = trainable\n\n    # Add all the UNet layers here\n    convLayer_112by112 = resnetLayers.get_layer(\"conv1_relu\").output\n    convLayer_56by56 = resnetLayers.get_layer(\"conv2_block3_out\").output # conv2_block3_2_relu\n    convLayer_28by28 = resnetLayers.get_layer(\"conv3_block4_out\").output # conv3_block4_2_relu\n    convLayer_14by14 = resnetLayers.get_layer(\"conv4_block6_out\").output # conv4_block6_2_relu\n    convLayer_7by7 = resnetLayers.get_layer(\"conv5_block3_out\").output # conv5_block3_2_relu\n    # The last layer of resnet model(conv5_block3_out) is of dimensions (7x7x2048)\n\n    # Start upsampling from 7x7 to 14x14 ...up to 224x224 to form UNet\n    # concatinate with the original image layer of the same size from ResNet50\n    up14by14 = Concatenate()([UpSampling2D()(convLayer_7by7), convLayer_14by14])\n    upConvLayer_14by14 = conv_block_simple(up14by14, 256, \"upConvLayer_14by14_1\")\n    upConvLayer_14by14 = conv_block_simple(upConvLayer_14by14, 256, \"upConvLayer_14by14_2\")\n    \n    up28by28 = Concatenate()([UpSampling2D()(upConvLayer_14by14), convLayer_28by28])\n    upConvLayer_28by28 = conv_block_simple(up28by28, 256, \"upConvLayer_28by28_1\")\n    upConvLayer_28by28 = conv_block_simple(upConvLayer_28by28, 256, \"upConvLayer_28by28_2\")\n     \n    up56by56 = Concatenate()([UpSampling2D()(upConvLayer_28by28), convLayer_56by56])\n    upConvLayer_56by56 = conv_block_simple(up56by56, 256, \"upConvLayer_56by56_1\")\n    upConvLayer_56by56 = conv_block_simple(upConvLayer_56by56, 256, \"upConvLayer_56by56_2\")    \n    \n    up112by112 = Concatenate()([UpSampling2D()(upConvLayer_56by56), convLayer_112by112])\n    upConvLayer_112by112 = conv_block_simple(up112by112, 256, \"upConvLayer_112by112_1\")\n    upConvLayer_112by112 = conv_block_simple(upConvLayer_112by112, 256, \"upConvLayer_112by112_2\")   \n    \n    up224by224 = UpSampling2D(name=\"unet_last\")(upConvLayer_112by112) # upsample to 224x224\n    upConvLayer_224by224 = conv_block_simple(up224by224, 256, \"upConvLayer_224by224_1\")\n    upConvLayer_224by224 = conv_block_simple(upConvLayer_224by224, 256, \"upConvLayer_224by224_2\")   \n\n    # Add classification layer\n    upConvLayer_224by224 = Conv2D(1, kernel_size=1, activation=\"sigmoid\", name=\"masks\")(upConvLayer_224by224)\n    upConvLayer_224by224 = Reshape((IMAGE_SIZE, IMAGE_SIZE))(upConvLayer_224by224) \n\n    return Model(inputs=resnetLayers.input, outputs=upConvLayer_224by224)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.269817Z","iopub.execute_input":"2021-11-16T15:10:41.270361Z","iopub.status.idle":"2021-11-16T15:10:41.289419Z","shell.execute_reply.started":"2021-11-16T15:10:41.270253Z","shell.execute_reply":"2021-11-16T15:10:41.288571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef showConfusionMatrixFromFile(report30IOU) :\n    \"\"\"To show Confusion Matrix and Classification report from given dataframe.\n        \n    Arguments:\n        report30IOU -- dataframe having target and prediction columns.\n    \"\"\"\n    report30IOU.fillna(0, inplace=True) # set NA IOU values to zero\n    # Get Targets and Predictions\n    y_30_test = report30IOU[\"Target\"]\n    y_30_predicted = report30IOU[\"predTarget\"]\n    print(\"Predictions above 30% IOU :\\n\")\n    print(\"Confusion Matrix:- \\n\", metrics.confusion_matrix(y_30_test, y_30_predicted), \"\\n\")\n    print(\"Classification Report:- \\n\", metrics.classification_report(y_30_test, y_30_predicted))\n    \n    \ndef showConfusionMatrixFromYs(y_test, y_predicted) :\n    \"\"\"To show Confusion Matrix and Classification report from given dataframes y_test and y_predicted.\n        \n    Arguments:\n        y_test -- 1D dataframe having target values.\n        y_predicted -- 1D dataframe having predicted values.\n    \"\"\"\n    print(\"Predictions above 30% IOU :\\n\")\n    print(\"Confusion Matrix:- \\n\", metrics.confusion_matrix(y_test, y_predicted), \"\\n\")\n    print(\"Classification Report:- \\n\", metrics.classification_report(y_test, y_predicted))","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.291567Z","iopub.execute_input":"2021-11-16T15:10:41.291909Z","iopub.status.idle":"2021-11-16T15:10:41.302796Z","shell.execute_reply.started":"2021-11-16T15:10:41.291876Z","shell.execute_reply":"2021-11-16T15:10:41.301693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Funstion to plot history curves\ndef plotHistory(_HISTORY_FILE) :\n    \"\"\"To plot history curves for training and validation losses\n        \n    Arguments:\n        _HISTORY_FILE -- training history file name with path.\n    \"\"\"\n    unetSavedHistory = np.load(_HISTORY_FILE, allow_pickle=True).item()\n    unetSavedHistoryDF = pd.DataFrame(unetSavedHistory)\n    \n    # list data in history\n    # summarize history for loss\n    pyplot.plot(unetSavedHistoryDF['loss'])\n    pyplot.plot(unetSavedHistoryDF['val_loss'])\n    pyplot.title('model loss')\n    pyplot.ylabel('loss')\n    pyplot.xlabel('epoch')\n    pyplot.legend(['train', 'validation'], loc='best')\n    pyplot.show()\n    # summarize history for mean IOU\n    pyplot.plot(unetSavedHistoryDF['mean_iou'])\n    pyplot.plot(unetSavedHistoryDF['val_mean_iou'])\n    pyplot.plot(unetSavedHistoryDF['lr'])\n    pyplot.title('model IOU and Leraning rate')\n    pyplot.ylabel('IOU and LR')\n    pyplot.xlabel('epoch')\n    pyplot.legend(['train', 'validation', 'Lerning Rate'], loc='best')\n    pyplot.show()\n","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.304555Z","iopub.execute_input":"2021-11-16T15:10:41.305517Z","iopub.status.idle":"2021-11-16T15:10:41.317365Z","shell.execute_reply.started":"2021-11-16T15:10:41.305476Z","shell.execute_reply":"2021-11-16T15:10:41.316559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef predictBatches(_test_CombinedData, _test_imageIdPaths, _UNetModel) :\n    \"\"\"Method to predict test data set and save the submission report into a csv file.\n        \n    Arguments:\n        _test_CombinedData -- test set dataframe having patientId, image labels, target and class values combined together\n\n        _imageIdPaths -- test set dataframe having patientId and image paths to load image for the patientId\n        \n        _UNetModel -- UNet model with trainined weights used for predicting test data\n    \"\"\"\n    print('Number of Test Samples :', _test_CombinedData[\"patientId\"].nunique()) # about 20% of the dataset\n    \n    # create test generator instance\n    testUNetDataGen = UNetTestGenerator(_test_imageIdPaths, _test_CombinedData) #for (224, 224)\n    \n    # create submission dafa frame with column names\n    submissionDF = pd.DataFrame(columns=['patientId', 'x', 'y', 'width', 'height', 'Target', 'class', 'x_pred', 'y_pred', 'width_pred', \n                                         'height_pred', 'predTarget', 'iou', 'class_pred'])\n    dfIndex = 0\n    iouThreshold = 0.3 # IoU above 30%\n\n    # loop through testset\n    # for batches from testUNetDataGen\n    print(\"Predicting Batches \", end='')\n    for batchImages, gtBatchMasks, batchPids, batchCoords, batchClasses, batchCoordsOrig in testUNetDataGen:    #for (224, 224)\n        print(\".\", end = '')    \n        \n        # predict batch of images\n        batchPreds = _UNetModel.predict(batchImages)    #for (224, 224)\n\n        prevPid = \"\"\n        # loop through batch\n        for pred, gtMask, pid, coords, gtClass, coordsOrig in zip(batchPreds, gtBatchMasks, batchPids, batchCoords, batchClasses, batchCoordsOrig):   #for (224, 224)\n\n            if prevPid != pid :\n                prevPid = pid\n\n                # resize predicted mask\n                pred = resize(pred, (1024, 1024), mode='reflect')   #for (1024, 1024)\n                # recompute coords for resized pred\n                coords = coordsOrig   #for (1024, 1024)\n\n                # threshold predicted mask\n                strongPred = pred[:, :] > 0.5   \n\n                # apply connected components\n                strongPred = measure.label(strongPred)\n\n                loopIndx = 0\n                # collect all reagions for the prediction\n                iouCoordsDF = pd.DataFrame(columns=['iou', 'x', 'y', 'width', 'height'])\n                for region in measure.regionprops(strongPred) :\n                    # retrieve x, y, height and width\n                    y, x, y2, x2 = region.bbox\n                    height = y2 - y\n                    width = x2 - x\n                    # Get IOUs\n                    coordsXYs = np.array([coords[0], coords[1], coords[2]+coords[0], coords[3]+coords[1]])\n                    regionXYs = np.array([x, y, x2, y2])\n                    IOU = iouFromCoords(coordsXYs, regionXYs)\n                    #print(\"IOU \", IOU)\n                    iouCoordsRow = [IOU, x, y, width, height]\n                    iouCoordsDF.loc[loopIndx] = iouCoordsRow\n                    loopIndx = loopIndx + 1\n\n                GTDFRow = [pid, coords[0], coords[1], coords[2], coords[3], coords[4], gtClass] # ground truth data \n                prevGTDFRow = []\n                # Get top 2 predictions based on IOU \n                iouCoordsDF.sort_values(\"iou\", ascending=False, inplace=True)\n                predIOUCoordCount = 0\n                # If predictions exist\n                if len(iouCoordsDF) > 0 :\n                    for predIOUCoordIdx in (0, len(iouCoordsDF)-1) :\n                        if iouCoordsDF.loc[predIOUCoordIdx][\"iou\"] > iouThreshold :\n                            # add row with ground truth and prediction values to data frame    \n                            submissionDFRow = [pid, coords[0], coords[1], coords[2], coords[3], coords[4],\n                                               gtClass, int(iouCoordsDF.loc[predIOUCoordIdx][\"x\"]), int(iouCoordsDF.loc[predIOUCoordIdx][\"y\"]), \n                                               int(iouCoordsDF.loc[predIOUCoordIdx][\"width\"]), int(iouCoordsDF.loc[predIOUCoordIdx][\"height\"]), \n                                               1, iouCoordsDF.loc[predIOUCoordIdx][\"iou\"], \"Lung Opacity\"]\n                            if predIOUCoordCount < 2 :\n                                if GTDFRow != prevGTDFRow : \n                                    submissionDF.loc[dfIndex] = submissionDFRow\n                                    dfIndex = dfIndex + 1 \n                                    predIOUCoordCount = predIOUCoordCount + 1\n                                    prevGTDFRow = GTDFRow\n                            else :\n                                break;\n                        else : # Normal if IOU below threshold\n                            # add row with ground truth and prediction values to data frame\n                            if GTDFRow != prevGTDFRow :  \n                                submissionDFRow = [pid, coords[0], coords[1], coords[2], coords[3], coords[4], \n                                                   gtClass, 0, 0, 0, 0, 0, iouCoordsDF.loc[predIOUCoordIdx][\"iou\"], \"Normal\"]\n                                submissionDF.loc[dfIndex] = submissionDFRow\n                                dfIndex = dfIndex + 1  \n                                prevGTDFRow = GTDFRow\n                                break;\n                            # end of if\n                        # end of if\n                    # end of for\n\n                else : # else of If predictions exist. Normal if no predictions\n                    # add row with ground truth and prediction values to data frame\n                    submissionDFRow = [pid, coords[0], coords[1], coords[2], coords[3], coords[4], \n                                       gtClass, 0, 0, 0, 0, 0, 'NA', \"Normal\"]\n                    submissionDF.loc[dfIndex] = submissionDFRow\n                    dfIndex = dfIndex + 1      \n\n        # To stop at certain count. Used to debug code.\n#             if len(submissionDF) >= 5 :\n#                 break\n\n    # save dictionary as csv file\n    submissionDF.to_csv('submission.csv', index=False)\n    print(\"Prediction Complete!\")\n    \n    test_y = submissionDF[\"Target\"]\n    predicted_y = submissionDF[\"predTarget\"]\n    \n    return test_y.apply(int), predicted_y.apply(int)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.319341Z","iopub.execute_input":"2021-11-16T15:10:41.319947Z","iopub.status.idle":"2021-11-16T15:10:41.3552Z","shell.execute_reply.started":"2021-11-16T15:10:41.319908Z","shell.execute_reply":"2021-11-16T15:10:41.354312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef visualizePredictions(_predReportDF, _topNum) :\n    \"\"\"Method to visualize predictions by displaying ground truth and predicted bounding boxes.\n        \n    Arguments:\n        _predReportDF -- dataframe having target and prediction coordinate columns; patientId and IoUs.\n\n        _topNum -- number indicating the count of top predictions to be visualized.\n    \"\"\"\n    # Sort on IOU to get higher IOUs on top\n    _predReportDF.sort_values(\"iou\", ascending=False, inplace=True)\n    # Get patientIds\n    topPids = _predReportDF[\"patientId\"].head(_topNum)\n    topPidsAry = np.array(topPids)\n    # Get IOUs\n    topIOUs = _predReportDF[\"iou\"].head(_topNum)\n    topIOUsAry = np.array(topIOUs)\n\n    # To get ground truth images for top IOU scored pids\n    imageCollc = np.zeros((_topNum, IMG_WIDTH, IMG_HEIGHT), np.float32) # (1024, 1024)\n\n    # Get ground truth coordinates for top IOU scored rows and prepare masks\n    gtCoordCollc = _predReportDF[[\"x\", \"y\", \"width\", \"height\"]].to_numpy()  # (1024, 1024)\n    # To get ground truth masks\n    gtMaskCollc  = np.zeros((_topNum, IMG_WIDTH, IMG_HEIGHT), np.int) # (1024, 1024)\n\n    # Get ground truth coordinates for top IOU scored rows and prepare masks\n    predCoordCollc = _predReportDF[[\"x_pred\", \"y_pred\", \"width_pred\", \"height_pred\"]].to_numpy()  # (1024, 1024)\n    # To get ground truth masks\n    predMaskCollc  = np.zeros((_topNum, IMG_WIDTH, IMG_HEIGHT), np.int)\n\n    # Get ground truth and prediction masks\n    for indx in range(0, _topNum) :\n        # Get images\n        path = test_imageIdPaths[test_imageIdPaths[\"patientId\"] == topPidsAry[indx]][\"imgPath\"].array[0]\n        imgData = loadImage(str(path)) # Read image\n        img = imgData.pixel_array\n        imageCollc[indx][:,:] = preprocess_input(np.array(img[:,:], dtype=np.float32)) # Convert to float32 array\n\n        # prepare ground truth masks\n        x = int(gtCoordCollc[indx, 0])\n        y = int(gtCoordCollc[indx, 1])\n        width = int(gtCoordCollc[indx, 2])\n        height = int(gtCoordCollc[indx, 3])\n        gtMaskCollc[indx][y:y+height, x:x+width] = 1   # (1024, 1024)\n\n        # prepare predicted masks\n        x_pred = int(predCoordCollc[indx, 0])\n        y_pred = int(predCoordCollc[indx, 1])\n        width_pred = int(predCoordCollc[indx, 2])\n        height_pred = int(predCoordCollc[indx, 3])\n        predMaskCollc[indx][y_pred:y_pred+height_pred, x_pred:x_pred+width_pred] = 1   # (1024, 1024)\n        \n    # Show images and bounding boxes\n    imageArea, axesArry = pyplot.subplots(int(_topNum/2), 2, figsize=(18,18))\n    axesArry = axesArry.ravel()\n    for axidx in range(0, _topNum) :\n        axesArry[axidx].imshow(imageCollc[axidx][:, :], cmap=pyplot.cm.bone)\n\n        gtComp = gtMaskCollc[axidx][:, :] > 0.5\n        # apply connected components\n        gtComp = measure.label(gtComp)\n        # apply ground truth bounding boxes\n        for region in measure.regionprops(gtComp):\n            # retrieve x, y, height and width\n            y1, x1, y2, x2 = region.bbox\n            heightReg = y2 - y1\n            widthReg = x2 - x1\n            axesArry[axidx].add_patch(patches.Rectangle((x1, y1), widthReg, heightReg, linewidth=1, edgecolor='r', \n                                                        facecolor='none'))\n\n        predComp = predMaskCollc[axidx][:, :] > 0.5\n        # apply connected components\n        predComp = measure.label(predComp)\n        # apply predicted bounding boxes\n        for region_pred in measure.regionprops(predComp):\n            # retrieve x, y, height and width\n            y1_pred, x1_pred, y2_pred, x2_pred = region_pred.bbox\n            heightReg_pred = y2_pred - y1_pred\n            widthReg_pred = x2_pred - x1_pred\n            axesArry[axidx].add_patch(patches.Rectangle((x1_pred, y1_pred), widthReg_pred, heightReg_pred, linewidth=1, edgecolor='b', \n                                                        facecolor='none'))\n            axesArry[axidx].set_title('IOU : '+str(topIOUsAry[axidx]))\n    # Show subplots\n    pyplot.show()","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.358737Z","iopub.execute_input":"2021-11-16T15:10:41.359183Z","iopub.status.idle":"2021-11-16T15:10:41.3861Z","shell.execute_reply.started":"2021-11-16T15:10:41.359145Z","shell.execute_reply":"2021-11-16T15:10:41.384548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create Generator instances for Train and Validation datasets\n\n* These are further used in fit() method while training the model.","metadata":{"editable":false}},{"cell_type":"code","source":"trainUNetDataGen = UNetTrainGenerator(train_imageIdPaths, train_CombinedData)\nvalidateUNetDataGen = UNetTrainGenerator(validate_imageIdPaths, validate_CombinedData)\n\nprint(len(trainUNetDataGen), \"# of iterations in one train epoch\")\nprint(len(validateUNetDataGen), \"# of iterations in one validate epoch\")","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.387737Z","iopub.execute_input":"2021-11-16T15:10:41.388387Z","iopub.status.idle":"2021-11-16T15:10:41.403254Z","shell.execute_reply.started":"2021-11-16T15:10:41.388329Z","shell.execute_reply":"2021-11-16T15:10:41.402075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### visualize sample image with a mask to verify train generator instance ","metadata":{"editable":false}},{"cell_type":"code","source":"imageSet0 = trainUNetDataGen[0][0]\nmaskSet0 = trainUNetDataGen[0][1]    \nshowMaskedImage(imageSet0, maskSet0, 5)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-11-16T15:10:41.40487Z","iopub.execute_input":"2021-11-16T15:10:41.405484Z","iopub.status.idle":"2021-11-16T15:10:42.143572Z","shell.execute_reply.started":"2021-11-16T15:10:41.405445Z","shell.execute_reply":"2021-11-16T15:10:42.14257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build UNet with ResNet50\n\n* We started with a simple UNet architecture using MobileNet layers pretrained with 'imagenet' weights.\n\n* Now we do advancement to it by using ResNet50 layers  pretrained with 'imagenet' weights to build the UNet architecture.","metadata":{"editable":false}},{"cell_type":"markdown","source":"#### Train the Model","metadata":{"editable":false}},{"cell_type":"code","source":"trainResnetUnetModel = True\nEPOCHS = 3\n\nres_adamOptimizer = Adam(lr=1e-1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\nres_checkpoint = ModelCheckpoint(\"resnetModel-{loss:.2f}.h5\", monitor=\"loss\", verbose=1, save_best_only=False,\n                             save_weights_only=True, mode=\"min\", period=1)\nres_stop = EarlyStopping( monitor=\"loss\", patience=5, mode=\"min\")\nres_reduce_lr = ReduceLROnPlateau(monitor=\"loss\", factor=0.2, patience=5, min_lr=1e-4, verbose=1, mode=\"min\")\n\nResNetUNetModel = create_ResNetUNetModel()\nResNetUNetModel.compile(loss=iou_loss, optimizer=res_adamOptimizer, metrics=[mean_iou]) \n\n\nif trainResnetUnetModel==True :\n    # Make layers trainable\n    for layer in ResNetUNetModel.layers:\n        layer.trainable = True\n\n    reshist = ResNetUNetModel.fit_generator(generator=trainUNetDataGen,\n                        epochs=EPOCHS,\n                        validation_data=validateUNetDataGen,\n                        callbacks=[res_checkpoint, res_reduce_lr, res_stop],\n                        shuffle=True,\n                        verbose=1)\n\n    resnetUnet_history = np.array(reshist.history)\n    np.save(\"resnetUnetTrainHist\", resnetUnet_history, allow_pickle=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-16T15:10:42.145005Z","iopub.execute_input":"2021-11-16T15:10:42.14549Z","iopub.status.idle":"2021-11-16T15:59:38.471014Z","shell.execute_reply.started":"2021-11-16T15:10:42.145449Z","shell.execute_reply":"2021-11-16T15:59:38.470145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nObserve ResNetUNet Model training history\n\n    See how loss and mean-IoU improved over the epochs\n\n","metadata":{"editable":false}},{"cell_type":"code","source":"RES_HISTORY_FILE = \"resnetUnetTrainHist.npy\"\nplotHistory(RES_HISTORY_FILE)","metadata":{"execution":{"iopub.status.busy":"2021-11-16T15:59:38.473066Z","iopub.execute_input":"2021-11-16T15:59:38.473595Z","iopub.status.idle":"2021-11-16T15:59:39.385294Z","shell.execute_reply.started":"2021-11-16T15:59:38.47355Z","shell.execute_reply":"2021-11-16T15:59:39.384315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nAfter 4th epoch we see that the validation loss getting increased which indicates overfitting of the model.\n\n    Hence we take the weights saved after 4th epoch (i.e. ‘resnetModel-0.78.h5’) for performing predictions.\n\n","metadata":{}},{"cell_type":"code","source":"predictResNetUNetModel = True\n\nif predictResNetUNetModel == True :\n    y_test_res, y_predicted_res = predictBatches(test_CombinedData, test_imageIdPaths, ResNetUNetModel)","metadata":{"execution":{"iopub.status.busy":"2021-11-16T15:59:39.390538Z","iopub.execute_input":"2021-11-16T15:59:39.39103Z","iopub.status.idle":"2021-11-16T16:07:25.699657Z","shell.execute_reply.started":"2021-11-16T15:59:39.390985Z","shell.execute_reply":"2021-11-16T16:07:25.698748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Confusion matrix","metadata":{}},{"cell_type":"code","source":"!ls ../working/","metadata":{"execution":{"iopub.status.busy":"2021-11-16T16:07:25.700966Z","iopub.execute_input":"2021-11-16T16:07:25.701486Z","iopub.status.idle":"2021-11-16T16:07:26.412943Z","shell.execute_reply.started":"2021-11-16T16:07:25.701445Z","shell.execute_reply":"2021-11-16T16:07:26.411983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RESREPORT_30_FILE = \"../working/submission.csv\"\nResReport30IOU = pd.read_csv(RESREPORT_30_FILE)\n\nif predictResNetUNetModel == True :\n    showConfusionMatrixFromYs(y_test_res, y_predicted_res)\nelse :\n    showConfusionMatrixFromFile(ResReport30IOU)","metadata":{"execution":{"iopub.status.busy":"2021-11-16T16:07:26.416156Z","iopub.execute_input":"2021-11-16T16:07:26.416563Z","iopub.status.idle":"2021-11-16T16:07:26.457507Z","shell.execute_reply.started":"2021-11-16T16:07:26.416517Z","shell.execute_reply":"2021-11-16T16:07:26.456608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Try Predictions with Bounding Box","metadata":{}},{"cell_type":"code","source":"topNum = 10 # provide even number\nvisualizePredictions(ResReport30IOU, topNum)","metadata":{"execution":{"iopub.status.busy":"2021-11-16T16:07:26.458779Z","iopub.execute_input":"2021-11-16T16:07:26.459304Z","iopub.status.idle":"2021-11-16T16:07:29.642811Z","shell.execute_reply.started":"2021-11-16T16:07:26.459265Z","shell.execute_reply":"2021-11-16T16:07:29.642012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n    Even though some of the IoU valuses are less than 80%, we observe that the ground truth boxes are completely covered within the predicted boxes.\n\n    It is important to see that images with different brightness, clarity, contrast are amoung the top predictions.\n\n","metadata":{}}]}