{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport sys\nimport math\nimport cv2\nimport seaborn as sns\nimport tensorflow as tf\n\n# To calculate accuracy measures and confusion matrix\nfrom sklearn import metrics\n# To get Recall and precision values\nfrom sklearn.metrics import classification_report\n\n# !pip install -q pydicom\n# After installing pydicom. This is needed to load .dcm files\nimport pydicom\nimport pydicom as dcm\nfrom pydicom import dcmread\n\nfrom matplotlib import pyplot\nimport matplotlib.patches as patches\n\nfrom skimage import measure\nfrom skimage.transform import resize\n\nimport tensorflow.keras.utils as pltUtil\nfrom tensorflow.keras.utils import Sequence\n\nfrom tensorflow.keras.layers import Layer\nfrom tensorflow.keras.layers import Concatenate, UpSampling2D, Conv2D, Reshape\nfrom tensorflow.keras.models import Model\n\nfrom tensorflow.keras.applications.mobilenet import MobileNet\nfrom tensorflow.keras.applications.mobilenet import preprocess_input \n\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\n#from tensorflow.keras.optimizers import SGD\n\nfrom tensorflow.keras.applications.resnet import ResNet50\nfrom tensorflow.keras.applications.resnet import preprocess_input as resnetProcess_input\n\n#from keras.preprocessing.image import ImageDataGenerator\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Setting up project path**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load train set image metadata\ndataDirPath = '../input/rsna-pneumonia-detection-challenge/'\n\nTrain_Image_path = dataDirPath + 'stage_2_train_images'\n\nSAVED_FILES_ROOT = '../input/reportfiles/'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"****Creating a function to load the metadata from images****","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageMetadata():\n    def __init__(self, setName, file):\n        # print(name, file)\n        # dataset name(train or test)\n        self.setName = setName\n        # image file name\n        self.file = file\n\n    def __repr__(self):\n        return self.imagePath()\n\n    def imagePath(self):\n        return os.path.join(self.setName, self.file) \n    \n\n# function to load image metadada   \ndef loadImageMetadata(dataSetName):\n    imageMetadata = []\n    for f in os.listdir(dataSetName):\n        # Check file extension. Allow only .dcm files.\n        ext = os.path.splitext(f)[1]\n        if ext == '.dcm' :\n            imageMetadata.append(ImageMetadata(dataSetName, f))\n    return np.array(imageMetadata)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Getting Metadata Information**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"trainSetImageMetadata = loadImageMetadata(Train_Image_path)\n\nprint(\"trainSetImageMetadata.shape : \", trainSetImageMetadata.shape)\n\nprint(\"Sample image path : \", trainSetImageMetadata[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"******Insights from Metadata******\n\nThere are 26684 images in the Training data\nThese are DICOM Images which has pixel information as well as several tags added to it like patientid, age,gender etc.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**Create a function to load image and loaging a sample Image**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def loadImage(path):\n    img = pydicom.dcmread(path)\n    return img\n\nimgIndex = 4\nimgPath = trainSetImageMetadata[imgIndex]\nimgPath = imgPath.imagePath()\nimgData = loadImage(imgPath)\n\npyplot.imshow(imgData.pixel_array, cmap=pyplot.cm.bone)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Preparing Dataset with patient id and respective image paths**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"trainSetImageMetadata_df = pd.DataFrame(trainSetImageMetadata, columns=[\"Path\"])\ntrainSetImageMetadata_df.head(2)\n\ndef getImgId(_imgData) :\n    return str(_imgData).split(\".dcm\")[0].split(\"/\")[4]\n\nimageIdPaths = pd.DataFrame(columns=[\"patientId\", \"imgPath\"])\nimageIdPaths[\"patientId\"] = trainSetImageMetadata_df[\"Path\"].apply(getImgId)\nimageIdPaths[\"imgPath\"] = trainSetImageMetadata_df[\"Path\"]\n\nprint(\"imageIdPaths\", imageIdPaths.shape)\nimageIdPaths.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Exloratory Data Analysis on Train Labels and Detail Info CSV data sets","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**Analyzing Detailed Classes CSV file**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Step 1 -> Define and read the Detail_Info CSV File ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"classesPath =  dataDirPath + 'stage_2_detailed_class_info.csv'\n\ndetailedClasses = pd.read_csv(classesPath)\n\ndetailedClasses.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Step 2 -> Check for Missing Values\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"detailedClasses.isna().apply(pd.value_counts)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"No Missing Values found","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Step 3 -> Checking the shape of data frame","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"detailedClasses.shape : \", detailedClasses.shape, )\n\n# File has 30227 rows and 2 columns - PatientID & Class","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Step 3 -> Checking Unique Patients","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Unique patientIds : \", detailedClasses['patientId'].nunique(), )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Total number of unique patients in data - 26684\n\n**Observation - As we have total 30227 records and out of that 26684 are unique records, this shows presence of multiple records for some patients**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Step 4 -> Checking unique Classes","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Unique patientIds : \", detailedClasses['class'].nunique(), )\n\nprint(detailedClasses['class'].unique)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"3 Unique classes observed \n1 - No Lung Opacity/Not Normal,\n2 - Normal,\n3 - Lung Opacity","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(x=\"class\",hue=\"class\",data=detailedClasses)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Analyzing Train Lables Dataset**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Step 1 -> Reading the data ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"labelsPath = dataDirPath + 'stage_2_train_labels.csv'\n\ntrainLabels = pd.read_csv(labelsPath)\n\ntrainLabels.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Step 2 -> Checking the missing values if any","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"trainLabels.isna().apply(pd.value_counts)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Observation -For around 20672 patients Bounding box cordinates not available where as for 9555 patients its avaialable**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"trainLabels[trainLabels['Target']==0].head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Few records have observed with missing values in x,y,width and height coulmn, but no missing values observed in patientid and Target.\n\nAlso this is observed such missing columns are present for those records with Target as '0'.\n\nx,y,width and height columns have the information for bounding boxes in Images where Penumonia is detected.\n\n**Explaination on missing values - These are not the missing values instead it is expected not to have Bounding Box co-ordinates for those images where Pneumonia is not detected (Target - '0')**\n\nHence concluding there are no missing values in this dataset as well","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Step 3 -> Checking unique Patients","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Unique patientIds : \", trainLabels['patientId'].nunique(), )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Total Unique patients found - 26684\nThis is same as the number of patients in Detailed CSV sheet hence both sheets share the information for same patients","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Step 4 -> Checkin unqiue Targets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Unique patientIds : \", trainLabels['Target'].nunique(), )\n\nprint(\"Unique patientIds : \", trainLabels['Target'].unique(), )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Trail label has only 2 target variables [0 & 1] \n\n**Conclusion - In Train labels only two target variables are present 0 & 1, where as in Detailed_Info sheet we have 3 classes.**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**As we have 3 Classes in Detailed_Info dataset and 2 Target Variables in Train_Labels, concatenating to get better insight into the data**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Step 1 -> Sorting both the datasets based on patientId","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"trainLabels.sort_values(\"patientId\", inplace=True)\ndetailedClasses.sort_values(\"patientId\", inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Step 2 -> Concatenating the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"Combined_Data = pd.concat([trainLabels, detailedClasses[\"class\"]], axis=1, sort=False)\nCombined_Data.head(3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Validating the concatenation results**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"Combined_Data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Combined_Data.isna().apply(pd.value_counts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Combined_Data[Combined_Data[\"Target\"] == 1].isna().apply(pd.value_counts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Combined_Data[Combined_Data[\"Target\"] == 0].isna().apply(pd.value_counts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Combined_Data[Combined_Data[\"class\"] == \"Lung Opacity\"].isna().apply(pd.value_counts)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"From Above Analysis our concatenated data is correct","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Prepare data for training\n\n\nStep 1 -> \n* Conver data to only two classes, 'Normal' and 'Lung Opacity'\n* Splitting the data in three parts, train, validation and test sets.\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Conver data to only two classes, 'Normal' and 'Lung Opacity'\nCombined_Data[\"class\"].replace(\"No Lung Opacity / Not Normal\", \"Normal\", inplace=True)\nCombined_Data.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_CombinedData = Combined_Data[0:15000]\nvalidate_CombinedData = Combined_Data[15000:25000]\ntest_CombinedData = Combined_Data[25000:30227]\n\nprint(\"train_CombinedData.shape : \", train_CombinedData.shape)\nprint(\"validate_CombinedData.shape : \", validate_CombinedData.shape)\nprint(\"test_CombinedData.shape : \", test_CombinedData.shape)\n\nprint(\"\\nunique train patients : \", train_CombinedData[\"patientId\"].nunique())\nprint(\"unique validate patients : \", validate_CombinedData[\"patientId\"].nunique())\nprint(\"unique test patients : \", test_CombinedData[\"patientId\"].nunique())\n\nprint(\"\\nTotal unique patients : \", imageIdPaths[\"patientId\"].nunique())\nprint(\"Total of unique train and test : \", train_CombinedData[\"patientId\"].nunique() + validate_CombinedData[\"patientId\"].nunique() + test_CombinedData[\"patientId\"].nunique())\n\nprint(\"\\nLast from train set : \", train_CombinedData.iloc[14999][\"patientId\"])\nprint(\"First from validate set : \", validate_CombinedData.iloc[0][\"patientId\"])\nprint(\"\\nLast from validate set : \", validate_CombinedData.iloc[9999][\"patientId\"])\nprint(\"First from test set : \", test_CombinedData.iloc[0][\"patientId\"])\n\n# Set all NaN values to 0 in train and test data sets. While training NaN will not have any meaning.\n#    * x, y, width and hight values as zero(0) means no bounding box.\ntrain_CombinedData.fillna(0, inplace=True)\nvalidate_CombinedData.fillna(0, inplace=True)\ntest_CombinedData.fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imageIdPaths.sort_values(\"patientId\", inplace=True)\n\ntrain_imageIdPaths = imageIdPaths[0:13163]\nvalidate_imageIdPaths = imageIdPaths[13163:21764]\ntest_imageIdPaths = imageIdPaths[21764:26684]\n\nprint(\"train_imageIdPaths.shape : \", train_imageIdPaths.shape)\nprint(\"validate_imageIdPaths.shape : \", validate_imageIdPaths.shape)\nprint(\"test_imageIdPaths.shape : \", test_imageIdPaths.shape)\n\nprint(\"\\nunique train patients : \", train_imageIdPaths[\"patientId\"].nunique())\nprint(\"unique validate patients : \", validate_imageIdPaths[\"patientId\"].nunique())\nprint(\"unique test patients : \", test_imageIdPaths[\"patientId\"].nunique())\n\nprint(\"\\nTotal unique patients : \", imageIdPaths[\"patientId\"].nunique())\nprint(\"Total of unique train and test : \", train_imageIdPaths[\"patientId\"].nunique() + validate_imageIdPaths[\"patientId\"].nunique() + test_imageIdPaths[\"patientId\"].nunique())\n\nprint(\"\\nLast from train set : \", train_imageIdPaths.iloc[13162][\"patientId\"])\nprint(\"First from validate set : \", validate_imageIdPaths.iloc[0][\"patientId\"])\nprint(\"Last from validate set : \", validate_imageIdPaths.iloc[8600][\"patientId\"])\nprint(\"First from test set : \", test_imageIdPaths.iloc[0][\"patientId\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(x=\"Target\",hue=\"class\",data=train_CombinedData)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(x=\"Target\",hue=\"class\",data=validate_CombinedData)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(x=\"Target\",hue=\"class\",data=test_CombinedData)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build UNet","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = 224\n\nIMG_WIDTH = 1024\nIMG_HEIGHT = 1024","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 10\n\nclass UNetTrainGenerator(Sequence):\n\n    def __init__(self, _imageIdPaths, _CombinedData):       \n        self.pids = _CombinedData[\"patientId\"].to_numpy()\n        self.imgIdPaths = _imageIdPaths\n        self.coords = _CombinedData[[\"x\", \"y\", \"width\", \"height\"]].to_numpy()\n        # Resize Bounding box\n        self.coords = self.coords * IMAGE_SIZE / IMG_WIDTH\n        \n\n    def __len__(self):\n        return math.ceil(len(self.coords) / BATCH_SIZE)\n    \n\n    def __getitem__(self, idx): # Get a batch\n        batch_coords = self.coords[idx * BATCH_SIZE:(idx + 1) * BATCH_SIZE] # Image coords\n        batch_pids = self.pids[idx * BATCH_SIZE:(idx + 1) * BATCH_SIZE] # Image pids    \n        \n        batch_images = np.zeros((len(batch_pids), IMAGE_SIZE, IMAGE_SIZE, 3), dtype=np.float32)\n        batch_masks = np.zeros((len(batch_pids), IMAGE_SIZE, IMAGE_SIZE))\n        for _indx, _pid in enumerate(batch_pids):\n            _path = self.imgIdPaths[self.imgIdPaths[\"patientId\"] == _pid][\"imgPath\"].array[0]\n            _imgData = loadImage(str(_path)) # Read image\n            img = _imgData.pixel_array \n            \n            # Resize image\n            resized_img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)\n            #resized_img = cv2.resize(img[200:824, 200:824], dsize=(IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)\n    \n            #print(\"batch_images[_indx] shape :\", batch_images[_indx][:,:,0].shape)\n            # preprocess image for the batch\n            batch_images[_indx][:,:,0] = preprocess_input(np.array(resized_img[:,:], dtype=np.float32)) # Convert to float32 array\n            batch_images[_indx][:,:,1] = preprocess_input(np.array(resized_img[:,:], dtype=np.float32)) # Convert to float32 array\n            batch_images[_indx][:,:,2] = preprocess_input(np.array(resized_img[:,:], dtype=np.float32)) # Convert to float32 array  \n            \n            x = int(batch_coords[_indx, 0])\n            y = int(batch_coords[_indx, 1])\n            width = int(batch_coords[_indx, 2])\n            height = int(batch_coords[_indx, 3])\n            \n            batch_masks[_indx][y:y+height, x:x+width] = 1\n\n        return batch_images, batch_masks","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainUNetDataGen = UNetTrainGenerator(train_imageIdPaths, train_CombinedData)\nvalidateUNetDataGen = UNetTrainGenerator(validate_imageIdPaths, validate_CombinedData)\n\nprint(len(trainUNetDataGen), \"# of iterations in one train epoch\")\nprint(len(validateUNetDataGen), \"# of iterations in one validate epoch\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# To show image with mask\ndef showMaskedImage(_imageSet, _maskSet, _index) :\n    maskImage = _imageSet[_index]\n\n    #pyplot.imshow(maskImage[:,:,0], cmap=pyplot.cm.bone)\n    maskImage[:,:,0] = _maskSet[_index] * _imageSet[_index][:,:,0]\n    maskImage[:,:,1] = _maskSet[_index] * _imageSet[_index][:,:,1]\n    maskImage[:,:,2] = _maskSet[_index] * _imageSet[_index][:,:,2]\n\n    pyplot.imshow(maskImage[:,:,0], cmap=pyplot.cm.bone)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imageSet0 = trainUNetDataGen[0][0]\nmaskSet0 = trainUNetDataGen[0][1]    \nshowMaskedImage(imageSet0, maskSet0, 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nALPHA = 1.0\n\ndef create_UNetModel(trainable=True):\n    model = MobileNet(input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3), include_top=False, alpha=ALPHA, weights=\"imagenet\") \n    # Top layer is last layer of the model\n\n    for layer in model.layers:\n        layer.trainable = trainable\n\n    # Add all the UNET layers here\n    #### Add your code here ####\n    convLayer_112by112 = model.get_layer(\"conv_pw_1_relu\").output\n    convLayer_56by56 = model.get_layer(\"conv_pw_3_relu\").output\n    convLayer_28by28 = model.get_layer(\"conv_pw_5_relu\").output\n    convLayer_14by14 = model.get_layer(\"conv_pw_11_relu\").output\n    convLayer_7by7 = model.get_layer(\"conv_pw_13_relu\").output\n    # The last layer of mobilenet model is of dimensions (7x7x1024)\n\n    # Start upsampling from 7x7 to 14x14 ...up to 224x224 to form UNET\n    # concatinate with the original image layer of the same size from MobileNet\n    x = Concatenate()([UpSampling2D()(convLayer_7by7), convLayer_14by14])\n    x = Concatenate()([UpSampling2D()(x), convLayer_28by28])\n    x = Concatenate()([UpSampling2D()(x), convLayer_56by56])\n    x = Concatenate()([UpSampling2D()(x), convLayer_112by112])\n    x = UpSampling2D(name=\"unet_last\")(x) # upsample to 224x224\n\n    # Add classification layer\n    x = Conv2D(1, kernel_size=1, activation=\"sigmoid\", name=\"masks\")(x)\n    x = Reshape((IMAGE_SIZE, IMAGE_SIZE))(x) \n\n    return Model(inputs=model.input, outputs=x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define iou or jaccard loss function\ndef iou_loss(y_true, y_pred):\n    y_true = tf.reshape(y_true, [-1])\n    y_pred = tf.reshape(y_pred, [-1])\n    intersection = tf.reduce_sum(y_true * y_pred)\n    score = (intersection + 1.) / (tf.reduce_sum(y_true) + tf.reduce_sum(y_pred) - intersection + 1.)\n    return 1 - score\n\n# mean iou as a metric\ndef mean_iou(y_true, y_pred):\n    y_pred = tf.round(y_pred)    \n    #intersect = tf.reduce_sum(y_true * y_pred, axis=[1, 2, 3])\n    intersect = tf.reduce_sum(y_true * y_pred, axis=[1])\n    #union = tf.reduce_sum(y_true, axis=[1, 2, 3]) + tf.reduce_sum(y_pred, axis=[1, 2, 3])\n    union = tf.reduce_sum(y_true, axis=[1]) + tf.reduce_sum(y_pred, axis=[1])\n    smooth = tf.ones(tf.shape(intersect))\n    return tf.reduce_mean((intersect + smooth) / (union - intersect + smooth))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nadamOptimizer = Adam(lr=1e-4, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\n\ncheckpoint = ModelCheckpoint(\"unetModel-{loss:.2f}.h5\", monitor=\"loss\", verbose=1, save_best_only=True,\n                             save_weights_only=True, mode=\"min\", period=1)\nstop = EarlyStopping( monitor=\"loss\", patience=5, mode=\"min\")\nreduce_lr = ReduceLROnPlateau(monitor=\"loss\", factor=0.2, patience=5, min_lr=1e-6, verbose=1, mode=\"min\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainUnetModel = False\n\nEPOCHS = 3\n\nWEIGHTS_FILE = SAVED_FILES_ROOT + \"unetModel-0.73.h5\"\nUNetModel = create_UNetModel()\nUNetModel.compile(loss=iou_loss, optimizer=adamOptimizer, metrics=[mean_iou, 'accuracy']) \n\nUNetModel.load_weights(WEIGHTS_FILE)\n\nif trainUnetModel==True :\n    # Make layers trainable\n    for layer in UNetModel.layers:\n        layer.trainable = True\n\n    hist = UNetModel.fit_generator(generator=trainUNetDataGen,\n                        epochs=EPOCHS,\n                        validation_data=validateUNetDataGen,\n                        callbacks=[checkpoint, reduce_lr, stop],\n                        shuffle=True,\n                        verbose=1)\n    \n    unet_history = np.array(hist.history)\n    np.save(\"unetTrainHist-3\", unet_history, allow_pickle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Read saved training history and plot curves\n\nHISTORY_FILE = SAVED_FILES_ROOT + \"unetTrainHist.npy\"\nunetSavedHistory = np.load(HISTORY_FILE, allow_pickle=True).item()\n\nunetSavedHistoryDF = pd.DataFrame(unetSavedHistory)\n\n# list data in history\n# summarize history for loss\npyplot.plot(unetSavedHistoryDF['loss'])\npyplot.plot(unetSavedHistoryDF['val_loss'])\npyplot.title('model loss')\npyplot.ylabel('loss')\npyplot.xlabel('epoch')\npyplot.legend(['train', 'test'], loc='best')\npyplot.show()\n# summarize history for mean IOU\npyplot.plot(unetSavedHistoryDF['mean_iou'])\npyplot.plot(unetSavedHistoryDF['val_mean_iou'])\npyplot.plot(unetSavedHistoryDF['lr'])\npyplot.title('model IOU and Leraning rate')\npyplot.ylabel('IOU and LR')\npyplot.xlabel('epoch')\npyplot.legend(['train', 'test', 'Lerning Rate'], loc='best')\npyplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pltUtil.plot_model(UNetModel,\n                    to_file=\"model.png\",\n                    show_shapes=True,\n                    show_layer_names=True,\n                    expand_nested=False,\n                    dpi=70)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check sample ground truth masked image and predicted masked image \nimageSet0 = trainUNetDataGen[0][0]\nmaskSet0 = trainUNetDataGen[0][1]\nprint(\"Ground Truth Box/Mask\")\nshowMaskedImage(imageSet0, maskSet0, 5)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predMasks = UNetModel.predict(imageSet0)\nprint(\"Predicted Box/Mask\")\nshowMaskedImage(imageSet0, predMasks, 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 10\n\nclass UNetTestGenerator(Sequence):\n\n    def __init__(self, _imageIdPaths, _CombinedData):       \n        self.pids = _CombinedData[\"patientId\"].to_numpy()\n        self.imgIdPaths = _imageIdPaths\n        self.coords = _CombinedData[[\"x\", \"y\", \"width\", \"height\", \"Target\"]].to_numpy() #for (1024, 1024)\n        self.classes = _CombinedData[\"class\"]\n        # Resize Bounding box\n        self.coordsOrig = self.coords #for (1024, 1024)\n        self.coords = self.coords * IMAGE_SIZE / IMG_WIDTH   #for (224, 224)\n        \n\n    def __len__(self):\n        return math.ceil(len(self.coords) / BATCH_SIZE)\n    \n\n    def __getitem__(self, idx): # Get a batch\n        batch_coords = self.coords[idx * BATCH_SIZE:(idx + 1) * BATCH_SIZE] # Image coords for (224, 224)\n        batch_coordsOrig = self.coordsOrig[idx * BATCH_SIZE:(idx + 1) * BATCH_SIZE] # Image coords for (1024, 1024)\n        batch_pids = self.pids[idx * BATCH_SIZE:(idx + 1) * BATCH_SIZE] # Image pids    \n        batch_classes = self.classes[idx * BATCH_SIZE:(idx + 1) * BATCH_SIZE] # Image classes           \n        \n        batch_images = np.zeros((len(batch_pids), IMAGE_SIZE, IMAGE_SIZE, 3), dtype=np.float32)\n        batch_masks = np.zeros((len(batch_pids), IMAGE_SIZE, IMAGE_SIZE))\n        for _indx, _pid in enumerate(batch_pids):\n            _path = self.imgIdPaths[self.imgIdPaths[\"patientId\"] == _pid][\"imgPath\"].array[0]\n            _imgData = loadImage(str(_path)) # Read image\n            img = _imgData.pixel_array \n            \n            # Resize image\n            resized_img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA) #(224, 224)\n            #resized_img = cv2.resize(img[200:824, 200:824], dsize=(IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)\n    \n            #print(\"batch_images[_indx] shape :\", batch_images[_indx][:,:,0].shape)\n            # preprocess image for the batch\n            batch_images[_indx][:,:,0] = preprocess_input(np.array(resized_img[:,:], dtype=np.float32)) # Convert to float32 array\n            batch_images[_indx][:,:,1] = preprocess_input(np.array(resized_img[:,:], dtype=np.float32)) # Convert to float32 array\n            batch_images[_indx][:,:,2] = preprocess_input(np.array(resized_img[:,:], dtype=np.float32)) # Convert to float32 array  \n            \n            x = int(batch_coords[_indx, 0])\n            y = int(batch_coords[_indx, 1])\n            width = int(batch_coords[_indx, 2])\n            height = int(batch_coords[_indx, 3])\n            target = int(batch_coords[_indx, 4])\n            \n            batch_coords[_indx, 0] = x\n            batch_coords[_indx, 1] = y \n            batch_coords[_indx, 2] = width \n            batch_coords[_indx, 3] = height    \n            batch_coords[_indx, 4] = target \n            \n            batch_masks[_indx][y:y+height, x:x+width] = 1\n\n        return batch_images, batch_masks, batch_pids, batch_coords, batch_classes, batch_coordsOrig  #for (224, 224) and (1024, 1024)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def iouFromCoords(boxA, boxB) :\n    # determine the (x, y)-coordinates of the intersection rectangle\n    xA = max(boxA[0], boxB[0])\n    yA = max(boxA[1], boxB[1])\n    xB = min(boxA[2], boxB[2])\n    yB = min(boxA[3], boxB[3])\n\n    # compute the area of intersection rectangle\n    interArea = abs(max((xB - xA, 0)) * max((yB - yA), 0))\n    if interArea == 0:\n        return 0\n    # compute the area of both the prediction and ground-truth\n    # rectangles\n    boxAArea = abs((boxA[2] - boxA[0]) * (boxA[3] - boxA[1]))\n    boxBArea = abs((boxB[2] - boxB[0]) * (boxB[3] - boxB[1]))\n\n    # compute the intersection over union by taking the intersection\n    # area and dividing it by the sum of prediction + ground-truth\n    # areas - the interesection area\n    iou = interArea / float(boxAArea + boxBArea - interArea)\n\n    # return the intersection over union value\n    return iou","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictUnetModel = False\n\nif predictUnetModel == True :\n\n    print('Number of Test Samples :', len(test_CombinedData)) # about 20% of the dataset\n    # create test generator \n    testUNetDataGen = UNetTestGenerator(test_imageIdPaths, test_CombinedData) #for (224, 224)\n    # create submission dafa frame with column names\n    submissionDF = pd.DataFrame(columns=['patientId', 'x', 'y', 'width', 'height', 'Target', 'class', 'x_pred', 'y_pred', 'width_pred', \n                                         'height_pred', 'predTarget', 'iou', 'class_pred'])\n    dfIndex = 0\n    iouThreshold = 0.3\n\n    # loop through testset\n    #for imgs, filenames in test_gen:\n    print(\"Predicting Batches \", end='')\n    for batchImages, gtBatchMasks, batchPids, batchCoords, batchClasses, batchCoordsOrig in testUNetDataGen:    #for (224, 224)\n        print(\".\", end = '')    \n        # predict batch of images\n        batchPreds = UNetModel.predict(batchImages)    #for (224, 224)\n\n        prevPid = \"\"\n        # loop through batch\n        for pred, gtMask, pid, coords, gtClass, coordsOrig in zip(batchPreds, gtBatchMasks, batchPids, batchCoords, batchClasses, batchCoordsOrig):   #for (224, 224)\n\n            if prevPid != pid :\n                prevPid = pid\n\n                # resize predicted mask\n                pred = resize(pred, (1024, 1024), mode='reflect')   #for (1024, 1024)\n                # recompute coords for resized pred\n                coords = coordsOrig   #for (1024, 1024)\n\n                # threshold predicted mask\n                strongPred = pred[:, :] > 0.5   \n\n                # apply connected components\n                strongPred = measure.label(strongPred)\n\n                loopIndx = 0\n                # collect all reagions for the prediction\n                iouCoordsDF = pd.DataFrame(columns=['iou', 'x', 'y', 'width', 'height'])\n                for region in measure.regionprops(strongPred) :\n                    # retrieve x, y, height and width\n                    y, x, y2, x2 = region.bbox\n                    height = y2 - y\n                    width = x2 - x\n                    # Get IOUs\n                    coordsXYs = np.array([coords[0], coords[1], coords[2]+coords[0], coords[3]+coords[1]])\n                    regionXYs = np.array([x, y, x2, y2])\n                    IOU = iouFromCoords(coordsXYs, regionXYs)\n                    #print(\"IOU \", IOU)\n                    iouCoordsRow = [IOU, x, y, width, height]\n                    iouCoordsDF.loc[loopIndx] = iouCoordsRow\n                    loopIndx = loopIndx + 1\n\n                GTDFRow = [pid, coords[0], coords[1], coords[2], coords[3], coords[4], gtClass] # ground truth data \n                prevGTDFRow = []\n                # Get top 2 predictions based on IOU \n                iouCoordsDF.sort_values(\"iou\", ascending=False, inplace=True)\n                predIOUCoordCount = 0\n                # If predictions exist\n                if len(iouCoordsDF) > 0 :\n                    for predIOUCoordIdx in (0, len(iouCoordsDF)-1) :\n                        if iouCoordsDF.loc[predIOUCoordIdx][\"iou\"] > iouThreshold :\n                            # add row with ground truth and prediction values to data frame    \n                            submissionDFRow = [pid, coords[0], coords[1], coords[2], coords[3], coords[4],\n                                               gtClass, int(iouCoordsDF.loc[predIOUCoordIdx][\"x\"]), int(iouCoordsDF.loc[predIOUCoordIdx][\"y\"]), \n                                               int(iouCoordsDF.loc[predIOUCoordIdx][\"width\"]), int(iouCoordsDF.loc[predIOUCoordIdx][\"height\"]), \n                                               1, iouCoordsDF.loc[predIOUCoordIdx][\"iou\"], \"Lung Opacity\"]\n                            if predIOUCoordCount < 2 :\n                                if GTDFRow != prevGTDFRow : \n                                    submissionDF.loc[dfIndex] = submissionDFRow\n                                    dfIndex = dfIndex + 1 \n                                    predIOUCoordCount = predIOUCoordCount + 1\n                                    prevGTDFRow = GTDFRow\n                            else :\n                                break;\n                        else : # Normal if IOU below threshold\n                            # add row with ground truth and prediction values to data frame\n                            if GTDFRow != prevGTDFRow :  \n                                submissionDFRow = [pid, coords[0], coords[1], coords[2], coords[3], coords[4], \n                                                   gtClass, 0, 0, 0, 0, 0, iouCoordsDF.loc[predIOUCoordIdx][\"iou\"], \"Normal\"]\n                                submissionDF.loc[dfIndex] = submissionDFRow\n                                dfIndex = dfIndex + 1  \n                                prevGTDFRow = GTDFRow\n                                break;\n                            # end of if\n                        # end of if\n                    # end of for\n\n                else : # else of If predictions exist. Normal if no predictions\n                    # add row with ground truth and prediction values to data frame\n                    submissionDFRow = [pid, coords[0], coords[1], coords[2], coords[3], coords[4], \n                                       gtClass, 0, 0, 0, 0, 0, 'NA', \"Normal\"]\n                    submissionDF.loc[dfIndex] = submissionDFRow\n                    dfIndex = dfIndex + 1      \n\n        # to stop at certain count for debug\n        #     if len(submissionDF) >= 15 :\n        #         break\n\n    # save dictionary as csv file\n    submissionDF.to_csv('submission.csv', index=False)\n    print(\"Prediction Complete!\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **Confusion Matrix and Classification Report**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"REPORT_30_FILE = SAVED_FILES_ROOT + \"submission_30.csv\"\nreport30IOU = pd.read_csv(REPORT_30_FILE)\nreport30IOU.fillna(0, inplace=True) # set NA IOU values to zero","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_30_test = report30IOU[\"Target\"]\ny_30_predicted = report30IOU[\"predTarget\"]\n\nprint(\"Predictions above 30% IOU :\\n\")\nprint(\"Confusion Matrix:- \\n\", metrics.confusion_matrix(y_30_test, y_30_predicted), \"\\n\")\nprint(\"Classification Report:- \\n\", metrics.classification_report(y_30_test, y_30_predicted))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Show top predicted masks with bounding boxes.**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"topNum = 6\n\n# Sort on IOU to get higher IOUs on top\nreport30IOU.sort_values(\"iou\", ascending=False, inplace=True)\n# Get patientIds\ntopPids = report30IOU[\"patientId\"].head(topNum)\ntopPidsAry = np.array(topPids)\n# Get IOUs\ntopIOUs = report30IOU[\"iou\"].head(topNum)\ntopIOUsAry = np.array(topIOUs)\n\n# To get ground truth images for top IOU scored pids\nimageCollc = np.zeros((topNum, IMG_WIDTH, IMG_HEIGHT), np.float32) # (1024, 1024)\n\n# Get ground truth coordinates for top IOU scored rows and prepare masks\ngtCoordCollc = report30IOU[[\"x\", \"y\", \"width\", \"height\"]].to_numpy()  # (1024, 1024)\n# To get ground truth masks\ngtMaskCollc  = np.zeros((topNum, IMG_WIDTH, IMG_HEIGHT), np.int) # (1024, 1024)\n\n# Get ground truth coordinates for top IOU scored rows and prepare masks\npredCoordCollc = report30IOU[[\"x_pred\", \"y_pred\", \"width_pred\", \"height_pred\"]].to_numpy()  # (1024, 1024)\n# To get ground truth masks\npredMaskCollc  = np.zeros((topNum, IMG_WIDTH, IMG_HEIGHT), np.int)\n\n# Get ground truth and prediction masks\nfor indx in range(0, topNum) :\n    # Get images\n    path = test_imageIdPaths[test_imageIdPaths[\"patientId\"] == topPidsAry[indx]][\"imgPath\"].array[0]\n    imgData = loadImage(str(path)) # Read image\n    img = imgData.pixel_array\n    imageCollc[indx][:,:] = preprocess_input(np.array(img[:,:], dtype=np.float32)) # Convert to float32 array\n    \n    # prepare ground truth masks\n    x = int(gtCoordCollc[indx, 0])\n    y = int(gtCoordCollc[indx, 1])\n    width = int(gtCoordCollc[indx, 2])\n    height = int(gtCoordCollc[indx, 3])\n    gtMaskCollc[indx][y:y+height, x:x+width] = 1   # (1024, 1024)\n\n    # prepare predicted masks\n    x_pred = int(predCoordCollc[indx, 0])\n    y_pred = int(predCoordCollc[indx, 1])\n    width_pred = int(predCoordCollc[indx, 2])\n    height_pred = int(predCoordCollc[indx, 3])\n    predMaskCollc[indx][y_pred:y_pred+height_pred, x_pred:x_pred+width_pred] = 1   # (1024, 1024)\n    \n\n# Show images and bounding boxes\nimageArea, axesArry = pyplot.subplots(3, 2, figsize=(18,18))\naxesArry = axesArry.ravel()\nfor axidx in range(0, topNum) :\n    axesArry[axidx].imshow(imageCollc[axidx][:, :], cmap=pyplot.cm.bone)\n    \n    gtComp = gtMaskCollc[axidx][:, :] > 0.5\n    # apply connected components\n    gtComp = measure.label(gtComp)\n    # apply ground truth bounding boxes\n    for region in measure.regionprops(gtComp):\n        # retrieve x, y, height and width\n        y1, x1, y2, x2 = region.bbox\n        heightReg = y2 - y1\n        widthReg = x2 - x1\n        axesArry[axidx].add_patch(patches.Rectangle((x1, y1), widthReg, heightReg, linewidth=1, edgecolor='r', \n                                                    facecolor='none'))\n        \n    predComp = predMaskCollc[axidx][:, :] > 0.5\n    # apply connected components\n    predComp = measure.label(predComp)\n    # apply predicted bounding boxes\n    for region_pred in measure.regionprops(predComp):\n        # retrieve x, y, height and width\n        y1_pred, x1_pred, y2_pred, x2_pred = region_pred.bbox\n        heightReg_pred = y2_pred - y1_pred\n        widthReg_pred = x2_pred - x1_pred\n        axesArry[axidx].add_patch(patches.Rectangle((x1_pred, y1_pred), widthReg_pred, heightReg_pred, linewidth=1, edgecolor='b', \n                                                    facecolor='none'))\n        axesArry[axidx].set_title('IOU : '+str(topIOUsAry[axidx]))\n        \n        \npyplot.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}