{"cells":[{"metadata":{"id":"KBeAf8WgaeSk","colab_type":"text","_uuid":"7c1fce19a11f95416168ced03c2c70fa818b21a5"},"cell_type":"markdown","source":"**Mask-RCNN Starter Model for the SIIM-ACR Pneumothorax Segmentation with transfer learning **\n\nThis kernel uses pre-trained weights from a past medical imaging competition on pneumonia identification: https://www.kaggle.com/hmendonca/mask-rcnn-and-coco-transfer-learning-lb-0-155\n\nBasic ideas included here:\n* dataset distribution balancing\n* image augmentation\n* multi-stage training for transfer learning, and model weights analysis\n* multi-mask prediction for submission\n\nIf you please, upvote and leave questions or constructive feedback below (for me and other kagglers learning).\nCheers!"},{"metadata":{"trusted":true,"_uuid":"cdb40bf9115f53810c9e13f0a50e53ed9eb6221b","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"debug = False\n# debug = True\n\n!ls ../input/","execution_count":null,"outputs":[]},{"metadata":{"id":"4kjcC6QqywWl","colab_type":"code","colab":{},"trusted":true,"_uuid":"40c67b3ff0fa04587dec508363308adaa3ceaf34"},"cell_type":"code","source":"import warnings \nwarnings.filterwarnings(\"ignore\")\n\nimport os, gc\nimport sys\nimport random\nimport math\nimport numpy as np\nimport cv2\nimport matplotlib.pyplot as plt\nimport json\nimport pydicom\nfrom imgaug import augmenters as iaa\nfrom tqdm import tqdm, tqdm_notebook\nimport pandas as pd \nimport glob\n\nsys.path.insert(0, '/kaggle/input/siim-acr-pneumothorax-segmentation')\nfrom mask_functions import rle2mask, mask2rle","execution_count":null,"outputs":[]},{"metadata":{"id":"yP0XLJx_x_6o","colab_type":"code","colab":{},"trusted":true,"_uuid":"6e5764759e6a0a9b698b44645658f66873edd807"},"cell_type":"code","source":"DATA_DIR = '/kaggle/input/siim-acr-pneumothorax-segmentation-data/pneumothorax'\n\n# Directory to save logs and trained model\nROOT_DIR = '/kaggle/working'\n\n!ls {DATA_DIR}","execution_count":null,"outputs":[]},{"metadata":{"id":"kdYzLq1zfKL4","colab_type":"text","_uuid":"576df4c47a23d08b1bdb384245e09aa69f88bbd3"},"cell_type":"markdown","source":"## Install Matterport's Mask-RCNN\nA very popular model in github.\nSee the [Matterport's implementation of Mask-RCNN](https://github.com/matterport/Mask_RCNN)."},{"metadata":{"id":"KgllzLnDr7kF","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":143},"outputId":"6c978df7-2013-437e-acd1-5011048dfb53","trusted":true,"_uuid":"b37d22551d332f0f7b722cc7204eb614524b6c21","_kg_hide-output":true},"cell_type":"code","source":"# !pip install 'keras==2.1.6' --force-reinstall\nSTAGE_DIR = '/tmp/Mask_RCNN'\n!git clone https://www.github.com/matterport/Mask_RCNN.git {STAGE_DIR}\nos.chdir(STAGE_DIR)\n#!python setup.py -q install\n!rm .git samples images assets -rf\n!pwd; ls","execution_count":null,"outputs":[]},{"metadata":{"id":"-KZXyWwhzOVU","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"outputId":"2576cc17-7484-4311-ad72-3c5643dcb5bb","trusted":true,"_uuid":"3acbbbe055b6a409d3c50ae0f893acf51b5ae7ba"},"cell_type":"code","source":"# Import Mask RCNN\nsys.path.append(os.path.join(ROOT_DIR, 'Mask_RCNN'))  # To find local version of the library\nfrom mrcnn.config import Config\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","execution_count":null,"outputs":[]},{"metadata":{"id":"FghMmiMjzOX2","colab_type":"code","colab":{},"trusted":true,"_uuid":"50089cc61791871cdf6a5c0037dc4f28b7b7d7cc"},"cell_type":"code","source":"train_dicom_dir = os.path.join(DATA_DIR, 'dicom-images-train')\ntest_dicom_dir = os.path.join(DATA_DIR, 'dicom-images-test')\n\n# count files\n!ls -m {train_dicom_dir} | wc\n!ls -m {test_dicom_dir} | wc","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f108beef7838be8a64dd512d395c5dc0ad952790"},"cell_type":"markdown","source":"### Load Pneumonia pre-trained weights"},{"metadata":{"trusted":true,"_uuid":"c3ee0cd0ee0b1defdec97b94bc736587c1f7631f"},"cell_type":"code","source":"# get model with best validation score: https://www.kaggle.com/hmendonca/mask-rcnn-and-coco-transfer-learning-lb-0-155/\nWEIGHTS_PATH = \"mask_rcnn_pneumonia.h5\"\n!cp /kaggle/input/mask-rcnn*/pneumonia*/*0013.h5 {WEIGHTS_PATH}\n!du -sh *.h5","execution_count":null,"outputs":[]},{"metadata":{"id":"gj-tvDvEaDiC","colab_type":"text","_uuid":"032cc5fe4baa051108106675e6ca4f4fdb2846ed"},"cell_type":"markdown","source":"## Setup Mask-RCNN\n\n- dicom_fps is a list of the dicom image path and filenames \n- image_annotions is a dictionary of the annotations keyed by the filenames\n- parsing the dataset returns a list of the image filenames and the annotations dictionary"},{"metadata":{"id":"_SfzTa-1zOck","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":917},"outputId":"91ae8935-bccb-4b8e-9a7e-aa690f95fd9b","trusted":true,"_uuid":"dfcffc4eaa94a41497717851dee9f702d8a2a73b"},"cell_type":"code","source":"# The following parameters have been selected to reduce running time for demonstration purposes \n# These are not optimal\n\nIMAGE_DIM = 512\n\nclass DetectorConfig(Config):    \n    # Give the configuration a recognizable name  \n    NAME = 'Pneumothorax'\n    \n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 11\n    \n    BACKBONE = 'resnet50'\n    \n    NUM_CLASSES = 2  # background and pneumothorax classes\n    \n    IMAGE_MIN_DIM = IMAGE_DIM\n    IMAGE_MAX_DIM = IMAGE_DIM\n    RPN_ANCHOR_SCALES = (32, 64, 128, 256)\n    TRAIN_ROIS_PER_IMAGE = 32\n    MAX_GT_INSTANCES = 15\n    DETECTION_MAX_INSTANCES = 4\n    DETECTION_MIN_CONFIDENCE = 0.90\n    DETECTION_NMS_THRESHOLD = 0.1\n    WEIGHT_DECAY = 0.0005\n\n    STEPS_PER_EPOCH = 20 if debug else 350\n    VALIDATION_STEPS = 10 if debug else 120\n    \n    ## balance out losses\n    LOSS_WEIGHTS = {\n        \"rpn_class_loss\": 12.0,\n        \"rpn_bbox_loss\": 0.6,\n        \"mrcnn_class_loss\": 6.0,\n        \"mrcnn_bbox_loss\": 1.0,\n        \"mrcnn_mask_loss\": 2.4\n    }\n\nconfig = DetectorConfig()\nconfig.display()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6136132b1f1b311e297d9432772ec4a81230924f"},"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom skimage.io import imread\nimport matplotlib.pyplot as plt\nfrom matplotlib.cm import get_cmap\nfrom skimage.segmentation import mark_boundaries\nfrom skimage.util import montage\nfrom skimage.morphology import binary_opening, disk, label\nimport gc; gc.enable() # memory is tight\n\nmontage_rgb = lambda x: np.stack([montage(x[:, :, :, i]) for i in range(x.shape[3])], -1)\n\ndef multi_rle_encode(img, **kwargs):\n    ''' Encode disconnected regions as separated masks\n    '''\n    labels = label(img)\n    if img.ndim > 2:\n        return [rle2mask(np.sum(labels==k, axis=2), **kwargs) for k in np.unique(labels[labels>0])]\n    else:\n        return [rle2mask(labels==k, **kwargs) for k in np.unique(labels[labels>0])]\n\ndef masks_as_image(rle_list, shape):\n    # Take the individual masks and create a single mask array\n    all_masks = np.zeros(shape, dtype=np.uint8)\n    for mask in rle_list:\n        if isinstance(mask, str) and mask != '-1':\n            all_masks |= rle2mask(mask, shape[0], shape[1]).T.astype(bool)\n    return all_masks\n\ndef masks_as_color(rle_list, shape):\n    # Take the individual masks and create a color mask array\n    all_masks = np.zeros(shape, dtype=np.float)\n    scale = lambda x: (len(rle_list)+x+1) / (len(rle_list)*2) ## scale the heatmap image to shift \n    for i,mask in enumerate(rle_list):\n        if isinstance(mask, str) and mask != '-1':\n            all_masks[:,:] += scale(i) * rle2mask(mask, shape[0], shape[1]).T\n    return all_masks","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3e05fa1a38c637fa228acd62b92dd41117a6672"},"cell_type":"code","source":"from PIL import Image\nfrom sklearn.model_selection import train_test_split\n\ntrain_glob = f'{train_dicom_dir}/*/*/*.dcm'\ntest_glob = f'{test_dicom_dir}/*/*/*.dcm'\n\nexclude_list = []\ntrain_names = [f for f in sorted(glob.glob(train_glob)) if f not in exclude_list]\ntest_names = [f for f in sorted(glob.glob(test_glob)) if f not in exclude_list]\n\nprint(len(train_names), len(test_names))\n# print(train_names[0], test_names[0])\n# !ls -l {os.path.join(train_dicom_dir, train_names[0])}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3050fa77026411ffdc27bed4a9b667ec0467e4ce"},"cell_type":"code","source":"# training dataset\nSEGMENTATION = DATA_DIR + '/train-rle.csv'\nanns = pd.read_csv(SEGMENTATION)\nanns.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get rid of damn space in column name\nanns.columns = ['ImageId', 'EncodedPixels']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# over-sample pneumothorax\npneumothorax_anns = anns[anns.EncodedPixels != ' -1'].ImageId.unique().tolist()\nprint(f'Positive samples: {len(pneumothorax_anns)}/{len(anns.ImageId.unique())} {100*len(pneumothorax_anns)/len(anns.ImageId.unique()):.2f}%')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"# ## use only pneumothorax images\n# pneumothorax_fps_train = [fp for fp in train_names if fp.split('/')[-1][:-4] in pneumothorax_anns]\n\n# image_fps_train, image_fps_val = train_test_split(pneumothorax_fps_train, test_size=0.1, random_state=42)\n\n# test_image_fps = test_names\n\n# if debug:\n#     print('DEBUG subsampling from:', len(image_fps_train), len(image_fps_val), len(test_image_fps))\n#     image_fps_train = image_fps_train[:150] \n#     image_fps_val = image_fps_val[:150]\n# #     test_image_fps = test_names[:150]\n    \n# print(len(image_fps_train), len(image_fps_val), len(test_image_fps))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"904636402355a305f7b2ccacb8cc55d52151d2e6","_kg_hide-input":false},"cell_type":"code","source":"## split and rebalance dataset\ntest_size = config.VALIDATION_STEPS * config.IMAGES_PER_GPU\nimage_fps_train, image_fps_val = train_test_split(train_names, test_size=test_size, random_state=42)\ntest_image_fps = test_names\n\npneumothorax_fps_train = [fp for fp in image_fps_train if fp.split('/')[-1][:-4] in pneumothorax_anns]\n\nif debug:\n    print('DEBUG subsampling from:', len(image_fps_train), len(image_fps_val), len(test_image_fps))\n    image_fps_train = image_fps_train[:100] + pneumothorax_fps_train[:50] \n    image_fps_val = image_fps_val[:150]\n#     test_image_fps = test_names[:150]\nelse:\n    image_fps_train += pneumothorax_fps_train*3  # oversample positive cases\n    random.shuffle(image_fps_train)\n    \nprint(len(image_fps_train), len(image_fps_val), len(test_image_fps))\npos, total = len([fp for fp in image_fps_train if fp in pneumothorax_fps_train]), len(image_fps_train)\nprint(f'Positive samples in training: {pos}/{total} {100*pos/total:.2f}%')","execution_count":null,"outputs":[]},{"metadata":{"id":"8EBVA1M60yAj","colab_type":"code","colab":{},"trusted":true,"_uuid":"52bd3ffbdde0173a363055482d675da51c2aba99"},"cell_type":"code","source":"class DetectorDataset(utils.Dataset):\n    \"\"\"Dataset class for training our dataset.\n    \"\"\"\n\n    def __init__(self, image_fps, image_annotations, orig_height, orig_width):\n        super().__init__(self)\n        \n        # Add classes\n        self.add_class('pneumothorax', 1, 'Pneumothorax')\n        \n        # add images \n        for i, fp in enumerate(image_fps):\n            image_id = fp.split('/')[-1][:-4]\n            annotations = image_annotations.query(f\"ImageId=='{image_id}'\")['EncodedPixels']\n            self.add_image('pneumothorax', image_id=i, path=fp, \n                           annotations=annotations, orig_height=orig_height, orig_width=orig_width)\n            \n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path']\n\n    def load_image(self, image_id):\n        info = self.image_info[image_id]\n        fp = info['path']\n        ds = pydicom.read_file(fp)\n        image = ds.pixel_array\n        # If grayscale. Convert to RGB for consistency.\n        if len(image.shape) != 3 or image.shape[2] != 3:\n            image = np.stack((image,) * 3, -1)\n        return image\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n        annotations = info['annotations']\n#         print(image_id, annotations)\n        count = len(annotations)\n        if count == 0 or (count == 1 and annotations.values[0] == ' -1'): # empty annotation\n            mask = np.zeros((info['orig_height'], info['orig_width'], 1), dtype=np.uint8)\n            class_ids = np.zeros((1,), dtype=np.int32)\n        else:\n            mask = np.zeros((info['orig_height'], info['orig_width'], count), dtype=np.uint8)\n            class_ids = np.zeros((count,), dtype=np.int32)\n            for i, a in enumerate(annotations):\n                mask[:, :, i] = rle2mask(a, info['orig_height'], info['orig_width']).T\n                class_ids[i] = 1\n        return mask.astype(np.bool), class_ids.astype(np.int32)","execution_count":null,"outputs":[]},{"metadata":{"id":"9RlMo04ckd98","colab_type":"text","_uuid":"1cb852e262b69d348743767d675573368ab672c9"},"cell_type":"markdown","source":"## Examine the annotation data\n..., parse the dataset, and view dicom fields"},{"metadata":{"id":"Mxz-pNbt5txY","colab_type":"code","colab":{},"trusted":true,"_uuid":"7aebc88f910b232e3b8759421914a007c6ffed94"},"cell_type":"code","source":"image_fps, image_annotations = train_names, anns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = pydicom.read_file(image_fps[0]) # read dicom image from filepath \n# image = ds.pixel_array # get image array\nprint(ds)\ndel ds; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"gYNSd1AhRqOV","colab_type":"code","colab":{},"trusted":true,"_uuid":"74277ae9af4a3b044e62b664d10d76b23848bb43"},"cell_type":"code","source":"# Original image size: 1024 x 1024\nORIG_SIZE = 1024","execution_count":null,"outputs":[]},{"metadata":{"id":"9KUvacUbgiEX","colab_type":"text","_uuid":"a5143c19dc22bc00d318a3b28cb7e13c7fbacc8a"},"cell_type":"markdown","source":"## Create and prepare the training dataset"},{"metadata":{"id":"jwMkhotP0yFf","colab_type":"code","colab":{},"trusted":true,"_uuid":"86c3333d4dfb8b7d00ce1f401693d0df4e6254e1"},"cell_type":"code","source":"%%time\n# prepare the training dataset\ndataset_train = DetectorDataset(image_fps_train, image_annotations, ORIG_SIZE, ORIG_SIZE)\ndataset_train.prepare()","execution_count":null,"outputs":[]},{"metadata":{"id":"K1TkWuGP0yHl","colab_type":"code","colab":{},"trusted":true,"_uuid":"313347d838fa8321a714858c8073f98c50c5be26"},"cell_type":"code","source":"%%time\n# prepare the validation dataset\ndataset_val = DetectorDataset(image_fps_val, image_annotations, ORIG_SIZE, ORIG_SIZE)\ndataset_val.prepare()","execution_count":null,"outputs":[]},{"metadata":{"id":"pEXEt8fygWuC","colab_type":"text","_uuid":"600a8135d4e382f62797d69e9358f5697873c8f9"},"cell_type":"markdown","source":"## Display a random image with bounding boxes"},{"metadata":{"id":"4xwsrf9G1lHR","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":359},"outputId":"a13386d3-a918-41fe-8824-13625c9d7b08","trusted":true,"_uuid":"491b78ec96d28fcdbbf8e2d7f9320a05d64c9249"},"cell_type":"code","source":"# Load and display random sample and their bounding boxes\n\nclass_ids = [0]\nwhile class_ids[0] == 0:  ## look for a mask\n    image_id = random.choice(dataset_val.image_ids)\n    image_fp = dataset_val.image_reference(image_id)\n    image = dataset_val.load_image(image_id)\n    mask, class_ids = dataset_val.load_mask(image_id)\n\nprint(image.shape)\n\nplt.figure(figsize=(10, 10))\nplt.subplot(1, 2, 1)\nplt.imshow(image)\nplt.axis('off')\n\nplt.subplot(1, 2, 2)\nmasked = np.zeros(image.shape[:2])\nfor i in range(mask.shape[2]):\n    masked += image[:, :, 0] * mask[:, :, i]\nplt.imshow(masked, cmap='gray')\nplt.axis('off')\n\nprint(image_fp)\nprint(class_ids)\n\ndel masked","execution_count":null,"outputs":[]},{"metadata":{"id":"ustAIH78hZI_","colab_type":"text","_uuid":"342b6008873fe7a6a0870a712ee47a87f0d2828d"},"cell_type":"markdown","source":"## Image Augmentation\nTry finetuning some variables to custom values"},{"metadata":{"id":"STZnQTE61lME","colab_type":"code","colab":{},"trusted":true,"_uuid":"4ab9d6086ce611a46f189c047956c43b29783e6d"},"cell_type":"code","source":"# Image augmentation (light but constant)\naugmentation = iaa.Sequential([\n    iaa.OneOf([ ## geometric transform\n        iaa.Affine(\n            scale={\"x\": (0.98, 1.02), \"y\": (0.98, 1.04)},\n            translate_percent={\"x\": (-0.02, 0.02), \"y\": (-0.04, 0.04)},\n            rotate=(-2, 2),\n            shear=(-1, 1),\n        ),\n        iaa.PiecewiseAffine(scale=(0.001, 0.025)),\n    ]),\n    iaa.OneOf([ ## brightness or contrast\n        iaa.Multiply((0.9, 1.1)),\n        iaa.ContrastNormalization((0.9, 1.1)),\n    ]),\n    iaa.OneOf([ ## blur or sharpen\n        iaa.GaussianBlur(sigma=(0.0, 0.1)),\n        iaa.Sharpen(alpha=(0.0, 0.1)),\n    ]),\n])\n\n# test on the same image as above\nimggrid = augmentation.draw_grid(image[:, :, 0], cols=5, rows=2)\nplt.figure(figsize=(30, 12))\n_ = plt.imshow(imggrid[:, :, 0], cmap='gray')\ndel imggrid; del image","execution_count":null,"outputs":[]},{"metadata":{"id":"M4kt7LKuc78e","colab_type":"text","_uuid":"7e65d2cecb283f446f34cdde19b663a8a8e9590f"},"cell_type":"markdown","source":"## Training\nNow it's time to train the model. Note that training even a basic model can take a few hours. \n\nNote: the following model is for demonstration purpose only. We have limited the training to a few epochs, and have set nominal values for the Detector Configuration to reduce run-time. \n\n- dataset_train and dataset_val are derived from DetectorDataset \n- DetectorDataset loads images from image filenames and  masks from the annotation data\n- model is Mask-RCNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# get pixel statistics\nimage_stats = []\nfor image_id in dataset_val.image_ids[:4]:\n    image = dataset_val.load_image(image_id)\n    image_stats.append(image.mean(axis=(0,1)))\n\nconfig.MEAN_PIXEL = np.mean(image_stats, axis=0).tolist()\n# VAR_PIXEL = images.var()\ndel image; del image_stats\ngc.collect()\n\nprint(config.MEAN_PIXEL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"138d6197fc8dce9f1f8a7b5a6c27aa2069698e03"},"cell_type":"code","source":"model = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)\n\n# load all weights as number of classes matches the pre-trained model\nmodel.load_weights(WEIGHTS_PATH, by_name=True)\n\n# # Exclude the last layers because they require a matching number of classes\n# model.load_weights(WEIGHTS_PATH, by_name=True,\n#                    exclude=[\"mrcnn_class_logits\", \"mrcnn_bbox_fc\", \"mrcnn_bbox\", \"mrcnn_mask\"])","execution_count":null,"outputs":[]},{"metadata":{"id":"RVgNhHjl1lOS","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":2575},"outputId":"2cba9efc-eeea-472d-d155-3c3d856585bf","trusted":true,"_uuid":"64cce2581ffdb8c2b1cb07948ada4a93f64874b0"},"cell_type":"code","source":"# Train the Mask-RCNN Model\nLEARNING_RATE = 0.0006","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf339a499519d174bcdf2311a1802f0e3acb1758"},"cell_type":"code","source":"%%time\n## train heads with higher lr to speedup the learning\nmodel.train(dataset_train, dataset_val,\n            learning_rate=LEARNING_RATE*2,\n            epochs=1,\n            layers='heads',\n            augmentation=None)  ## no need to augment yet\n\nhistory = model.keras_model.history.history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8004790d27f041793562e994bbe95edf67f8978b"},"cell_type":"code","source":"%%time\nmodel.train(dataset_train, dataset_val,\n            learning_rate=LEARNING_RATE,\n            epochs=3 if debug else 13,\n            layers='all',\n            augmentation=augmentation)\n\nnew_history = model.keras_model.history.history\nfor k in new_history: history[k] = history[k] + new_history[k]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0f55437aaa49e58ae60225a035fa8a3f6b604d3","_kg_hide-input":false},"cell_type":"code","source":"# %%time\n# model.train(dataset_train, dataset_val,\n#             learning_rate=LEARNING_RATE/2,\n#             epochs=4 if debug else 18,\n#             layers='all',\n#             augmentation=augmentation)\n\n# new_history = model.keras_model.history.history\n# for k in new_history: history[k] = history[k] + new_history[k]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71abf32327a102e1c22e944b24d98690c71d9560"},"cell_type":"code","source":"epochs = range(1, len(history['loss'])+1)\npd.DataFrame(history, index=epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb3b69242b91dcc49697ff076ceeb957347372e1"},"cell_type":"code","source":"plt.figure(figsize=(21,11))\n\nplt.subplot(231)\nplt.plot(epochs, history[\"loss\"], label=\"Train loss\")\nplt.plot(epochs, history[\"val_loss\"], label=\"Valid loss\")\nplt.legend()\nplt.subplot(232)\nplt.plot(epochs, history[\"rpn_class_loss\"], label=\"Train RPN class ce\")\nplt.plot(epochs, history[\"val_rpn_class_loss\"], label=\"Valid RPN class ce\")\nplt.legend()\nplt.subplot(233)\nplt.plot(epochs, history[\"rpn_bbox_loss\"], label=\"Train RPN box loss\")\nplt.plot(epochs, history[\"val_rpn_bbox_loss\"], label=\"Valid RPN box loss\")\nplt.legend()\nplt.subplot(234)\nplt.plot(epochs, history[\"mrcnn_class_loss\"], label=\"Train MRCNN class ce\")\nplt.plot(epochs, history[\"val_mrcnn_class_loss\"], label=\"Valid MRCNN class ce\")\nplt.legend()\nplt.subplot(235)\nplt.plot(epochs, history[\"mrcnn_bbox_loss\"], label=\"Train MRCNN box loss\")\nplt.plot(epochs, history[\"val_mrcnn_bbox_loss\"], label=\"Valid MRCNN box loss\")\nplt.legend()\nplt.subplot(236)\nplt.plot(epochs, history[\"mrcnn_mask_loss\"], label=\"Train Mask loss\")\nplt.plot(epochs, history[\"val_mrcnn_mask_loss\"], label=\"Valid Mask loss\")\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c2b38ecbc84575295dd62657ed175c5a0b72021"},"cell_type":"code","source":"best_epoch = np.argmin(history[\"val_loss\"])\nscore = history[\"val_loss\"][best_epoch]\nprint(f'Best Epoch:{best_epoch+1} val_loss:{score}')","execution_count":null,"outputs":[]},{"metadata":{"id":"eraRlzgPmmIZ","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"outputId":"de9e688c-ba4f-4b62-f842-dbcf00ce397c","trusted":true,"_uuid":"db5c10d3f7da099e5751a04a6e6d49819882ecd4"},"cell_type":"code","source":"# select trained model \ndir_names = next(os.walk(model.model_dir))[1]\nkey = config.NAME.lower()\ndir_names = filter(lambda f: f.startswith(key), dir_names)\ndir_names = sorted(dir_names)\n\nif not dir_names:\n    import errno\n    raise FileNotFoundError(\n        errno.ENOENT,\n        \"Could not find model directory under {}\".format(self.model_dir))\n\nfps = []\n# Pick last directory\nfor d in dir_names: \n    dir_name = os.path.join(model.model_dir, d)\n    # Find checkpoints\n    checkpoints = next(os.walk(dir_name))[2]\n    checkpoints = filter(lambda f: f.startswith(\"mask_rcnn\"), checkpoints)\n    checkpoints = sorted(checkpoints)\n    if not checkpoints:\n        raise Exception(f'No weight files in {dir_name}')\n    if best_epoch < len(checkpoints):\n        checkpoint = checkpoints[best_epoch]\n    else:\n        checkpoint = checkpoints[-1]\n    fps.append(os.path.join(dir_name, checkpoint))\n\nmodel_path = sorted(fps)[-1]\nprint('Found model {}'.format(model_path))","execution_count":null,"outputs":[]},{"metadata":{"id":"TgpT9AzC2Bgz","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":53},"outputId":"60f5a175-4666-497d-b4e8-0bdab39a92d0","trusted":true,"_uuid":"52138636b2ae5bf444bba808518cd8313bde65cd"},"cell_type":"code","source":"class InferenceConfig(DetectorConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\ninference_config = InferenceConfig()\n\n# Recreate the model in inference mode\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=ROOT_DIR)\n\n# Load trained weights (fill in path to trained weights here)\nassert model_path != \"\", \"Provide path to trained weights\"\nprint(\"Loading weights from \", model_path)\nmodel.load_weights(model_path, by_name=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"9mTBig7D2BjU","colab_type":"code","colab":{},"trusted":true,"_uuid":"e13c61bee23b791c61ecf1256f7512295cd4d9ab"},"cell_type":"code","source":"# set color for class\ndef get_colors_for_class_ids(class_ids):\n    colors = []\n    for class_id in class_ids:\n        if class_id == 1:\n            colors.append((.941, .204, .204))\n    return colors","execution_count":null,"outputs":[]},{"metadata":{"id":"A8EiL2LOiCr_","colab_type":"text","_uuid":"f99fbd3f31ff1a2bd66764835c9b646375364598"},"cell_type":"markdown","source":"## Validation\nHow does the predicted box compared to the expected value? Let's use the validation dataset to check. "},{"metadata":{"id":"irheTbrW2Bl0","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1394},"outputId":"56041ad4-173d-45ab-af67-f54e8333511e","trusted":true,"_uuid":"186412199e25b98719f71cfe5e8869abcce516c4"},"cell_type":"code","source":"# Show few example of ground truth vs. predictions on the validation dataset \ndataset = dataset_val\npneumothorax_ids_val = [fp.split('/')[-1][:-4] for fp in image_fps_val]\npneumothorax_ids_val = [i for i,id in enumerate(pneumothorax_ids_val) if id in pneumothorax_anns]\nfig = plt.figure(figsize=(10, 40))\n\nfor i in range(8):\n    image_id = random.choice(pneumothorax_ids_val)\n    \n    original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n        modellib.load_image_gt(dataset_val, inference_config, \n                               image_id, use_mini_mask=False)\n    \n#     print(original_image.shape)\n    plt.subplot(8, 2, 2*i + 1)\n    visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, \n                                dataset.class_names,\n                                colors=get_colors_for_class_ids(gt_class_id), ax=fig.axes[-1])\n    \n    plt.subplot(8, 2, 2*i + 2)\n    results = model.detect([original_image]) #, verbose=1)\n    r = results[0]\n    visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], \n                                dataset.class_names, r['scores'], \n                                colors=get_colors_for_class_ids(r['class_ids']), ax=fig.axes[-1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Basic Model Weigths Analysis"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# Show stats of all trainable weights    \nvisualize.display_weight_stats(model)\n### Click to expand output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from https://github.com/matterport/Mask_RCNN/blob/master/samples/coco/inspect_weights.ipynb\n# Pick layer types to display\nLAYER_TYPES = ['Conv2D', 'Dense', 'Conv2DTranspose']\n# Get layers\nlayers = model.get_trainable_layers()\nlayers = list(filter(lambda l: l.__class__.__name__ in LAYER_TYPES, \n                     layers))\n# Display Histograms\nfig, ax = plt.subplots(len(layers), 2, figsize=(10, 3*len(layers)), gridspec_kw={\"hspace\":1})\nfor l, layer in enumerate(layers):\n    weights = layer.get_weights()\n    for w, weight in enumerate(weights):\n        tensor = layer.weights[w]\n        ax[l, w].set_title(f'Layer:{l}.{w} {tensor.name}')\n        _ = ax[l, w].hist(weight[w].flatten(), 50)","execution_count":null,"outputs":[]},{"metadata":{"id":"WcV1cL_aiSc4","colab_type":"text","_uuid":"164e18701a830bc6c42a791feea13549de37289b"},"cell_type":"markdown","source":"## Final steps\nCreate the submission file"},{"metadata":{"id":"C6UWVrbM2Bob","colab_type":"code","colab":{},"trusted":true,"_uuid":"4a5c0c6134408ddbf5a34496d7e9d7be5692e9a1"},"cell_type":"code","source":"# Make predictions on test images, write out submission file\ndef predict(image_fps, filepath='submission.csv', min_conf=0.97):\n    # assume square image\n    resize_factor = ORIG_SIZE / config.IMAGE_SHAPE[0]\n    with open(filepath, 'w') as file:\n        file.write(\"ImageId,EncodedPixels\\n\")\n\n        for fp in tqdm_notebook(image_fps):\n            image_id = fp.split('/')[-1][:-4]\n            maks_written = 0\n            \n            ds = pydicom.read_file(fp)\n            image = ds.pixel_array\n            # If grayscale. Convert to RGB for consistency.\n            if len(image.shape) != 3 or image.shape[2] != 3:\n                image = np.stack((image,) * 3, -1)\n            image, window, scale, padding, crop = utils.resize_image(\n                image,\n                min_dim=config.IMAGE_MIN_DIM,\n                min_scale=config.IMAGE_MIN_SCALE,\n                max_dim=config.IMAGE_MAX_DIM,\n                mode=config.IMAGE_RESIZE_MODE)\n\n            results = model.detect([image])\n            r = results[0]\n\n#             assert( len(r['rois']) == len(r['class_ids']) == len(r['scores']) )\n            num_instances = len(r['rois'])\n\n            for i in range(num_instances):\n                if r['scores'][i] > min_conf and np.sum(r['masks'][...,i]) > 1:\n                    mask = r['masks'][...,i].T*255\n                    mask, _,_,_,_ = utils.resize_image(\n                        np.stack((mask,) * 3, -1), # requires 3 channels\n                        min_dim=ORIG_SIZE,\n                        min_scale=config.IMAGE_MIN_SCALE,\n                        max_dim=ORIG_SIZE,\n                        mode=config.IMAGE_RESIZE_MODE)\n                    mask = (mask[...,0] > 0)*255\n#                     print(mask.shape)\n#                     plt.imshow(mask, cmap=get_cmap('jet'))\n                    file.write(image_id + \",\" + mask2rle(mask, ORIG_SIZE, ORIG_SIZE) + \"\\n\")\n                    maks_written += 1\n\n            if maks_written == 0:\n                file.write(image_id + \",-1\\n\")  ## no pneumothorax","execution_count":null,"outputs":[]},{"metadata":{"id":"C5cBpNka2Bsv","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"outputId":"a2af9176-d9d6-49f6-f22a-5a1c455d144f","trusted":true,"_uuid":"0406e7f5aaa4867782c4f9c064f90bba386128e7"},"cell_type":"code","source":"submission_fp = os.path.join(ROOT_DIR, 'submission.csv')\npredict(test_image_fps, filepath=submission_fp)\nprint(submission_fp)","execution_count":null,"outputs":[]},{"metadata":{"id":"_BjPE_Ee9rbA","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1599},"outputId":"67b5f053-112b-494a-9ab3-d017bfb440c2","trusted":true,"_uuid":"3fd8d178fc51ef0bca94fbb3f423160f08a77edc"},"cell_type":"code","source":"sub = pd.read_csv(submission_fp, dtype={'ImageId':str, 'EncodedPixels':str})\nprint((sub.EncodedPixels != '-1').sum(), sub.ImageId.size, sub.ImageId.nunique())\nprint(sub.EncodedPixels.nunique(), (sub.EncodedPixels != '-1').sum()/sub.ImageId.nunique())\n\nprint('Unique samples:\\n', sub.EncodedPixels.drop_duplicates()[:6])\nsub.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea110f197abc2acb1c3435383f7259079dc0eb0e"},"cell_type":"code","source":"# show a few test image detection example\ndef visualize_test():\n    ids_with_mask = sub[sub.EncodedPixels != '-1'].ImageId.values\n    fp = random.choice([fp for fp in test_image_fps if fp.split('/')[-1][:-4] in ids_with_mask])\n#     import pdb; pdb.set_trace()\n    \n    # original image\n    image_id = fp.split('/')[-1][:-4]\n    ds = pydicom.read_file(fp)\n    image = ds.pixel_array\n    \n    # If grayscale. Convert to RGB for consistency.\n    if len(image.shape) != 3 or image.shape[2] != 3:\n        image = np.stack((image,) * 3, -1)\n    \n    # assume square image \n    resize_factor = 1 ## ORIG_SIZE / config.IMAGE_SHAPE[0]\n\n    # Detect on full size test images (without resizing)\n    results = model.detect([image])\n    r = results[0]\n    for bbox in r['rois']: \n#         print(bbox)\n        x1 = int(bbox[1] * resize_factor)\n        y1 = int(bbox[0] * resize_factor)\n        x2 = int(bbox[3] * resize_factor)\n        y2 = int(bbox[2]  * resize_factor)\n        cv2.rectangle(image, (x1,y1), (x2,y2), (77, 255, 9), 3, 1)\n        width = x2 - x1\n        height = y2 - y1\n#         print(\"x {} y {} h {} w {}\".format(x1, y1, width, height))\n    fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(15, 5))\n    ax1.set_title(image_id)\n    ax1.imshow(image)\n    ax2.set_title(f\"{len(r['rois'])} masks predicted again\")\n    if len(r['rois']) > 0:\n        ax2.imshow(r['masks'].max(-1))  # get max (overlap) between all masks in this prediction\n    ax3.set_title(f\"{np.count_nonzero(image_id == ids_with_mask)} masks in csv\")\n    ax3.imshow(masks_as_color(sub.query(f\"ImageId=='{image_id}'\")['EncodedPixels'].values, (ORIG_SIZE, ORIG_SIZE)))\n#     print(f\"ImageId=='{image_id}'\", sub.query(f\"ImageId=='{image_id}'\")['EncodedPixels'])\n\nfor i in range(8):\n    visualize_test()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"os.chdir(ROOT_DIR)\n!ls *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"835a15c9d018acd5deb16e9e02f9b765f68d0e78","_kg_hide-input":true},"cell_type":"code","source":"# remove files to allow committing (hit files limit otherwise)\n!rm -rf {STAGE_DIR}  /kaggle/working/*/events*","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"lesson-3-rsna-pneumonia-detection-challenge-kaggle","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"accelerator":"GPU","language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}