{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Lung Segmentation from RANZCR Chest X-rays \n\n[radda](https://www.kaggle.com/c/ranzcr-clip-catheter-line-classification/discussion/207183) kindly provided lung masks for the RANZCR training data. Lung masks are believed to be critical in order to successfully detect intubation/catheter malpositions.\nAs suggested I build my own UNet model, that can map the x-ray chest of the competition data to lung masks.\n\nThis notebook illustrates a simple Keras model to learn the lung-mask and leverages [qubvel's segmentation models](https://github.com/qubvel/segmentation_models) . The power of Transfer learning is used for the image segmentation - the UNet network has been pretrained with ImageNet. The performance is much better than my earlier own custom network trained from scratch: [notebook](https://www.kaggle.com/philippschwarz/ranzcr-lung-mask-model-not-pretrained)"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# First, installing the dependencies\n!pip install -U ../input/kerasapplications/Keras_Applications-1.0.8-py3-none-any.whl\n!pip install ../input/qubvel/efficientnet-1.0.0-py3-none-any.whl\n!pip install ../input/qubvel/image_classifiers-1.0.0-py3-none-any.whl\n\n# Now, installing segmentation_models (short for 'sm')\n!pip install ../input/qubvel-segmentation-model-keras-v101/segmentation_models-master\n\n# sm can work with both Keras and Tensorflow.\n# By default, it look for keras.\n# But, with Keras, it's giving error during the import. \n# So, we will be using Tensorflow as the backend for sm.\n%env SM_FRAMEWORK=tf.keras","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport cv2\nimport os\nimport matplotlib.pyplot as plt\n\nfrom keras import backend as K\nfrom keras.callbacks import CSVLogger, ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\n\nimport keras\nimport json\nimport tqdm\nfrom segmentation_models.losses import bce_jaccard_loss\nfrom segmentation_models.metrics import iou_score\nimport gc\nfrom segmentation_models import Unet\nimport segmentation_models  as sm\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils import Sequence\nfrom keras.optimizers import Adam\nimport warnings\nwarnings.filterwarnings('ignore')\nprint(os.listdir('../input'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"ctr = pd.read_csv('../input/ranzcr-clip-lung-contours/RANZCR_CLiP_lung_contours.csv')\ntrain = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path\nimport ast\n\nDIMENSION =  (256, 256) #(128, 128)  #\nIMG_HEIGHT, IMG_WIDTH = DIMENSION\n\nSEED = 25\nBATCH_SIZE = 32\nEPOCHS = 12\n\nBACKBONE = 'seresnet34'\n\nTRAIN_PATH = '../input/ranzcr-clip-catheter-line-classification/train/'\nIMAGE_LIB = TRAIN_PATH\nall_images = os.listdir(TRAIN_PATH)[:3000]\nall_images = [Path(e).stem for e in all_images]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(all_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_mask(StudyInstanceUID):\n    img = cv2.imread('../input/ranzcr-clip-catheter-line-classification/train/'+StudyInstanceUID+'.jpg',-1)\n    ctr_left = ast.literal_eval(ctr.loc[ctr.StudyInstanceUID==StudyInstanceUID,'left_lung_contour'].values[0])\n    ctr_right = ast.literal_eval(ctr.loc[ctr.StudyInstanceUID==StudyInstanceUID,'right_lung_contour'].values[0])\n    img = cv2.drawContours(img, np.array([[np.array(x) for x in ctr_left]]), 0, (255), -1)\n    img = cv2.drawContours(img, np.array([[np.array(x) for x in ctr_right]]), 0, (255), -1)\n    img = np.where(img>=255, 1.0, 0.0)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine,\n    IAASharpen, IAAEmboss, RandomContrast, RandomBrightness, Flip, OneOf, Compose, RandomGamma, Rotate,IAAAffine\n)\n\naug_null = Compose([])\naug = Compose([ \n    Blur(p=0.5, blur_limit=2),\n    IAAAffine(p=0.5, shear=5),\n    HorizontalFlip(p=0.5),              \n    #VerticalFlip(p=0.5),              \n    Rotate(limit=5, p=0.3),\n    #CLAHE(p=0.3),\n    RandomContrast(p=0.2, limit=0.1),\n    RandomBrightness(p=0.2, limit=0.1),\n    #RandomGamma(p=0.2, gamma_limit=(90, 110))\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocess_input = sm.get_preprocessing(BACKBONE)\n\nclass DataGenerator(keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, image_filenames, batch_size=32,\n                 dim=DIMENSION,  shuffle=True, \n                 preprocess_input=preprocess_input, \n                 aug=aug_null, min_mask=2 ):\n        'Initialization'\n        self.image_filenames = image_filenames\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.preprocess_input = preprocess_input\n        self.aug = aug\n        self.on_epoch_end()\n        self.dim = dim\n\n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        return int(np.floor((len(self.image_filenames) / self.batch_size) / 1) )\n\n    \n    def __getitem__(self, index):\n        'Generate one batch of data'\n        # Generate indexes of the batch\n        \n        end_index = min((index+1)*self.batch_size, len(self.indexes))\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n\n\n        # Generate data\n        X, Y = self.__data_generation(indexes)\n\n        return X, Y\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange(len(self.image_filenames))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n\n    def __data_generation(self, indexes):\n        'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)\n        \n        batch_size = len(indexes)\n        \n        # Initialization\n        XX = np.empty((batch_size, self.dim[1], self.dim[0], 3), dtype='float32')\n        YY = np.empty((batch_size, self.dim[1], self.dim[0], 1), dtype='float32')\n\n            \n        # Generate data\n        for i, ID in enumerate(indexes):\n            # Read image\n            im = cv2.imread(IMAGE_LIB + all_images[ID] +'.jpg')\n            im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n            im = im.astype(np.float32) / 255.\n            im = cv2.resize(im, dsize=(IMG_WIDTH, IMG_HEIGHT), interpolation=cv2.INTER_LANCZOS4)    \n            \n            # Read mask\n            mask = load_mask(all_images[ID])\n            mask = cv2.resize(mask, dsize=(IMG_WIDTH, IMG_HEIGHT), interpolation=cv2.INTER_NEAREST)\n            \n            # Store class\n            augmented = self.aug(image=im, mask=mask)\n            aug_img = augmented['image']\n            aug_mask = augmented['mask']\n            aug_mask = np.expand_dims(aug_mask, axis=-1)\n            \n            assert (np.max(aug_mask)<= 1.0 and  np.min(aug_mask) >= 0)\n            aug_mask[aug_mask>=0.5] = 1\n            aug_mask[aug_mask<0.5] = 0\n            \n            YY[i,] = aug_mask.astype('float32')\n            XX[i,] = aug_img.astype('float32')\n    \n       \n        XX = self.preprocess_input(XX)\n            \n        return XX, YY","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trn_idx, val_idx = train_test_split(all_images, test_size = 0.2, random_state = SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator = DataGenerator(\n    trn_idx, \n    batch_size=BATCH_SIZE, \n    dim=DIMENSION,\n    aug=aug, \n    preprocess_input=preprocess_input\n)\n\nval_generator = DataGenerator(\n    val_idx, \n    batch_size=BATCH_SIZE, \n    dim=DIMENSION,\n    aug=aug_null, \n    preprocess_input=preprocess_input,\n    shuffle=False\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x, y= train_generator[7]\nnp.max(x), x.shape, y.shape, np.max(y), np.unique(y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.unique(load_mask(ctr.StudyInstanceUID.tolist()[1234]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_batch, mask_batch = train_generator[1]\nfix, ax = plt.subplots(8,2, figsize=(10,20))\nfor i in range(8):\n    ax[i,0].imshow(image_batch[i,:,:,0])\n    ax[i,1].imshow(mask_batch[i,:,:,0])\n    ax[i, 0].axis('off')\n    ax[i, 1].axis('off')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(x[10, ..., 0])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(y[10, ..., 0])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from segmentation_models import Unet\nmodel = Unet(backbone_name=BACKBONE, encoder_weights='imagenet', activation='sigmoid', classes=1, input_shape=(IMG_HEIGHT, IMG_WIDTH, 3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dice_coef(y_true, y_pred, smooth=1):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(Adam(lr = 0.001), loss=bce_jaccard_loss, metrics=[dice_coef, iou_score])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reduce_lr = ReduceLROnPlateau(factor=0.5, patience=5, min_lr=0.000001, verbose=1, monitor='val_dice_coeff', mode='max')\n\nearly_stopping = EarlyStopping(patience=10, verbose=1, monitor='val_dice_coeff', mode='max')\nmodel_checkpoint = ModelCheckpoint(\"unet_custom_128-128_{epoch:02d}-{val_loss:.3f}.hdf5\", \n                                   save_weights_only=True, \n                                   monitor='val_dice_coeff', verbose=1, mode='max', period=1)\n\nhistory = model.fit_generator( train_generator,\n                            validation_data=val_generator,\n                            epochs=EPOCHS,\n                            callbacks=[reduce_lr, early_stopping, model_checkpoint], \n                            verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyJsonEncoder(json.JSONEncoder):\n    def default(self, obj):\n        #if isinstance(obj, np.integer):\n        #    return int(obj)\n        if isinstance(obj, np.floating):\n            return float(obj)\n        #if isinstance(obj, np.ndarray):\n        #    return obj.tolist()\n        return super(MyJsonEncoder, self).default(obj)\n\n\nwith open('history.json', 'w') as f:\n    json.dump(history.history, f, cls=MyJsonEncoder)\n    \nhistory_df = pd.DataFrame(history.history)\nhistory_df.head(2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,3,figsize=(20,4))\nhistory_df.val_loss.plot(ax=ax[0], color='red', title='Validation_loss',ylim=(0,5))\nhistory_df.val_iou_score.plot(ax=ax[1], color='blue', title='Validation_IOU', )\nhistory_df.val_dice_coef.plot(ax=ax[2], color='green', title='Validation_Dice_Coef');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = 0\ny_hat = model.predict(image_batch)\nfig, ax = plt.subplots(1,3,figsize=(12,6))\nax[0].imshow(image_batch[n,:,:,0], cmap='gray')\nax[1].imshow(mask_batch[n,:,:,0])\nax[2].imshow(y_hat[n,:,:,0]);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test quality of masks on test dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_PATH = '../input/ranzcr-clip-catheter-line-classification/test/'\nname = '1.2.826.0.1.3680043.8.498.10023042737818625910026668901358652653'\nim = cv2.imread(TEST_PATH + name +'.jpg', cv2.IMREAD_UNCHANGED).astype(\"int16\").astype('float32')\nim = cv2.resize(im, dsize=(IMG_WIDTH, IMG_HEIGHT), interpolation=cv2.INTER_LANCZOS4)\nim = (im - np.min(im)) / (np.max(im) - np.min(im))\nim = im.reshape(1,IMG_WIDTH, IMG_HEIGHT, 1 )\n\ny_hat = model.predict(im)\nfig, ax = plt.subplots(1,2,figsize=(12,4))\nax[0].imshow(im[0,:,:,0], cmap='gray')\nax[1].imshow(y_hat[0,:,:,0]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}