{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os \nimport sys\nimport random\nimport math\nimport numpy as np\nimport cv2\nimport matplotlib.pyplot as plt\nimport json\nimport pydicom\nfrom imgaug import augmenters as iaa\nfrom tqdm import tqdm\nimport pandas as pd \nimport glob ","metadata":{"id":"4kjcC6QqywWl","_uuid":"40c67b3ff0fa04587dec508363308adaa3ceaf34","execution":{"iopub.status.busy":"2022-03-13T09:32:07.836291Z","iopub.execute_input":"2022-03-13T09:32:07.836518Z","iopub.status.idle":"2022-03-13T09:32:08.820559Z","shell.execute_reply.started":"2022-03-13T09:32:07.836471Z","shell.execute_reply":"2022-03-13T09:32:08.819859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = '/kaggle/input'\n\n# Directory to save logs and trained model\nROOT_DIR = '/kaggle/working'","metadata":{"id":"yP0XLJx_x_6o","_uuid":"6e5764759e6a0a9b698b44645658f66873edd807","execution":{"iopub.status.busy":"2022-03-13T09:32:08.822969Z","iopub.execute_input":"2022-03-13T09:32:08.823391Z","iopub.status.idle":"2022-03-13T09:32:08.826906Z","shell.execute_reply.started":"2022-03-13T09:32:08.823343Z","shell.execute_reply":"2022-03-13T09:32:08.826259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Install Matterport's Mask-RCNN model from github.\nSee the [Matterport's implementation of Mask-RCNN](https://github.com/matterport/Mask_RCNN).","metadata":{"id":"kdYzLq1zfKL4","_uuid":"576df4c47a23d08b1bdb384245e09aa69f88bbd3"}},{"cell_type":"code","source":"!git clone https://www.github.com/matterport/Mask_RCNN.git\nos.chdir('Mask_RCNN')\n#!python setup.py -q install","metadata":{"id":"KgllzLnDr7kF","outputId":"6c978df7-2013-437e-acd1-5011048dfb53","_uuid":"b37d22551d332f0f7b722cc7204eb614524b6c21","execution":{"iopub.status.busy":"2022-03-13T09:32:08.828027Z","iopub.execute_input":"2022-03-13T09:32:08.828543Z","iopub.status.idle":"2022-03-13T09:32:18.161183Z","shell.execute_reply.started":"2022-03-13T09:32:08.828497Z","shell.execute_reply":"2022-03-13T09:32:18.160135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import Mask RCNN\nsys.path.append(os.path.join(ROOT_DIR, 'Mask_RCNN'))  # To find local version of the library\nfrom mrcnn.config import Config\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","metadata":{"id":"-KZXyWwhzOVU","outputId":"2576cc17-7484-4311-ad72-3c5643dcb5bb","_uuid":"3acbbbe055b6a409d3c50ae0f893acf51b5ae7ba","execution":{"iopub.status.busy":"2022-03-13T09:32:18.162717Z","iopub.execute_input":"2022-03-13T09:32:18.163196Z","iopub.status.idle":"2022-03-13T09:32:18.38199Z","shell.execute_reply.started":"2022-03-13T09:32:18.163121Z","shell.execute_reply":"2022-03-13T09:32:18.381087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dicom_dir = os.path.join(DATA_DIR, 'stage_2_train_images')\ntest_dicom_dir = os.path.join(DATA_DIR, 'stage_2_test_images')","metadata":{"id":"FghMmiMjzOX2","_uuid":"50089cc61791871cdf6a5c0037dc4f28b7b7d7cc","execution":{"iopub.status.busy":"2022-03-13T09:32:18.386007Z","iopub.execute_input":"2022-03-13T09:32:18.388202Z","iopub.status.idle":"2022-03-13T09:32:18.394314Z","shell.execute_reply.started":"2022-03-13T09:32:18.388117Z","shell.execute_reply":"2022-03-13T09:32:18.393487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Download COCO pre-trained weights\n!wget --quiet https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5\n!ls -lh mask_rcnn_coco.h5\n\nCOCO_WEIGHTS_PATH = \"mask_rcnn_coco.h5\"","metadata":{"_uuid":"c3ee0cd0ee0b1defdec97b94bc736587c1f7631f","execution":{"iopub.status.busy":"2022-03-13T09:32:18.399041Z","iopub.execute_input":"2022-03-13T09:32:18.401631Z","iopub.status.idle":"2022-03-13T09:32:29.557481Z","shell.execute_reply.started":"2022-03-13T09:32:18.40158Z","shell.execute_reply":"2022-03-13T09:32:29.55663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Some setup functions and classes for Mask-RCNN\n\n- dicom_fps is a list of the dicom image path and filenames \n- image_annotions is a dictionary of the annotations keyed by the filenames\n- parsing the dataset returns a list of the image filenames and the annotations dictionary","metadata":{"id":"gj-tvDvEaDiC","_uuid":"032cc5fe4baa051108106675e6ca4f4fdb2846ed"}},{"cell_type":"code","source":"def get_dicom_fps(dicom_dir):\n    dicom_fps = glob.glob(dicom_dir+'/'+'*.dcm')\n    return list(set(dicom_fps))\n\ndef parse_dataset(dicom_dir, anns): \n    image_fps = get_dicom_fps(dicom_dir)\n    image_annotations = {fp: [] for fp in image_fps}\n    for index, row in anns.iterrows(): \n        fp = os.path.join(dicom_dir, row['patientId']+'.dcm')\n        image_annotations[fp].append(row)\n    return image_fps, image_annotations ","metadata":{"id":"ivqC4cnszOaM","_uuid":"778cb19865d7cc63440491aef9202b71c61e8bb2","execution":{"iopub.status.busy":"2022-03-13T09:32:29.558827Z","iopub.execute_input":"2022-03-13T09:32:29.559104Z","iopub.status.idle":"2022-03-13T09:32:29.56617Z","shell.execute_reply.started":"2022-03-13T09:32:29.559054Z","shell.execute_reply":"2022-03-13T09:32:29.565031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The following parameters have been selected to reduce running time for demonstration purposes \n# These are not optimal \n\nclass DetectorConfig(Config):\n    \"\"\"Configuration for training pneumonia detection on the RSNA pneumonia dataset.\n    Overrides values in the base Config class.\n    \"\"\"\n    \n    # Give the configuration a recognizable name  \n    NAME = 'pneumonia'\n    \n    # Train on 1 GPU and 8 images per GPU. We can put multiple images on each\n    # GPU because the images are small. Batch size is 8 (GPUs * images/GPU).\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 8\n    \n    BACKBONE = 'resnet50'\n    \n    NUM_CLASSES = 2  # background + 1 pneumonia classes\n    \n    IMAGE_MIN_DIM = 256\n    IMAGE_MAX_DIM = 256\n    RPN_ANCHOR_SCALES = (32, 64, 128, 256)\n    TRAIN_ROIS_PER_IMAGE = 32\n    MAX_GT_INSTANCES = 3\n    DETECTION_MAX_INSTANCES = 3\n    DETECTION_MIN_CONFIDENCE = 0.7\n    DETECTION_NMS_THRESHOLD = 0.1\n\n    STEPS_PER_EPOCH = 200\n    \nconfig = DetectorConfig()\nconfig.display()","metadata":{"id":"_SfzTa-1zOck","outputId":"91ae8935-bccb-4b8e-9a7e-aa690f95fd9b","_uuid":"dfcffc4eaa94a41497717851dee9f702d8a2a73b","execution":{"iopub.status.busy":"2022-03-13T09:32:29.568184Z","iopub.execute_input":"2022-03-13T09:32:29.568797Z","iopub.status.idle":"2022-03-13T09:32:29.586886Z","shell.execute_reply.started":"2022-03-13T09:32:29.568415Z","shell.execute_reply":"2022-03-13T09:32:29.586077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DetectorDataset(utils.Dataset):\n    \"\"\"Dataset class for training pneumonia detection on the RSNA pneumonia dataset.\n    \"\"\"\n\n    def __init__(self, image_fps, image_annotations, orig_height, orig_width):\n        super().__init__(self)\n        \n        # Add classes\n        self.add_class('pneumonia', 1, 'Lung Opacity')\n        \n        # add images \n        for i, fp in enumerate(image_fps):\n            annotations = image_annotations[fp]\n            self.add_image('pneumonia', image_id=i, path=fp, \n                           annotations=annotations, orig_height=orig_height, orig_width=orig_width)\n            \n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path']\n\n    def load_image(self, image_id):\n        info = self.image_info[image_id]\n        fp = info['path']\n        ds = pydicom.read_file(fp)\n        image = ds.pixel_array\n        # If grayscale. Convert to RGB for consistency.\n        if len(image.shape) != 3 or image.shape[2] != 3:\n            image = np.stack((image,) * 3, -1)\n        return image\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n        annotations = info['annotations']\n        count = len(annotations)\n        if count == 0:\n            mask = np.zeros((info['orig_height'], info['orig_width'], 1), dtype=np.uint8)\n            class_ids = np.zeros((1,), dtype=np.int32)\n        else:\n            mask = np.zeros((info['orig_height'], info['orig_width'], count), dtype=np.uint8)\n            class_ids = np.zeros((count,), dtype=np.int32)\n            for i, a in enumerate(annotations):\n                if a['Target'] == 1:\n                    x = int(a['x'])\n                    y = int(a['y'])\n                    w = int(a['width'])\n                    h = int(a['height'])\n                    mask_instance = mask[:, :, i].copy()\n                    cv2.rectangle(mask_instance, (x, y), (x+w, y+h), 255, -1)\n                    mask[:, :, i] = mask_instance\n                    class_ids[i] = 1\n        return mask.astype(np.bool), class_ids.astype(np.int32)","metadata":{"id":"8EBVA1M60yAj","_uuid":"52bd3ffbdde0173a363055482d675da51c2aba99","execution":{"iopub.status.busy":"2022-03-13T09:32:29.587904Z","iopub.execute_input":"2022-03-13T09:32:29.588136Z","iopub.status.idle":"2022-03-13T09:32:29.600756Z","shell.execute_reply.started":"2022-03-13T09:32:29.588093Z","shell.execute_reply":"2022-03-13T09:32:29.599558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Examine the annotation data, parse the dataset, and view dicom fields","metadata":{"id":"9RlMo04ckd98","_uuid":"1cb852e262b69d348743767d675573368ab672c9"}},{"cell_type":"code","source":"# training dataset\nanns = pd.read_csv(os.path.join(DATA_DIR, 'stage_2_train_labels.csv'))\nanns.head()","metadata":{"id":"EdhUEFDr0yDA","outputId":"1715a5df-a577-41fd-bf20-f1a27aadb28c","_uuid":"793b1c6c6ba4e5f0d51e130080aa799f230b5ef6","execution":{"iopub.status.busy":"2022-03-13T09:32:29.601768Z","iopub.execute_input":"2022-03-13T09:32:29.602635Z","iopub.status.idle":"2022-03-13T09:32:29.707278Z","shell.execute_reply.started":"2022-03-13T09:32:29.601998Z","shell.execute_reply":"2022-03-13T09:32:29.706598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_fps, image_annotations = parse_dataset(train_dicom_dir, anns=anns)","metadata":{"id":"Mxz-pNbt5txY","_uuid":"7aebc88f910b232e3b8759421914a007c6ffed94","execution":{"iopub.status.busy":"2022-03-13T09:32:29.708482Z","iopub.execute_input":"2022-03-13T09:32:29.708733Z","iopub.status.idle":"2022-03-13T09:32:32.742123Z","shell.execute_reply.started":"2022-03-13T09:32:29.708688Z","shell.execute_reply":"2022-03-13T09:32:32.741443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = pydicom.read_file(image_fps[0]) # read dicom image from filepath \nimage = ds.pixel_array # get image array","metadata":{"id":"YPqjEIXWRhSf","_uuid":"6c386dcef041b972f6209dd19e247d547c3c349f","execution":{"iopub.status.busy":"2022-03-13T09:32:32.743297Z","iopub.execute_input":"2022-03-13T09:32:32.743559Z","iopub.status.idle":"2022-03-13T09:32:32.779567Z","shell.execute_reply.started":"2022-03-13T09:32:32.743516Z","shell.execute_reply":"2022-03-13T09:32:32.778942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show dicom fields \nds","metadata":{"id":"81lovwF2Ro5R","outputId":"e2263fe2-1a32-432a-ec75-b9220a24e697","_uuid":"0ef68a41cf1a5e842e86a219b6392e3695004720","execution":{"iopub.status.busy":"2022-03-13T09:32:32.782441Z","iopub.execute_input":"2022-03-13T09:32:32.782657Z","iopub.status.idle":"2022-03-13T09:32:32.788778Z","shell.execute_reply.started":"2022-03-13T09:32:32.782611Z","shell.execute_reply":"2022-03-13T09:32:32.788056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Original DICOM image size: 1024 x 1024\nORIG_SIZE = 1024","metadata":{"id":"gYNSd1AhRqOV","_uuid":"74277ae9af4a3b044e62b664d10d76b23848bb43","execution":{"iopub.status.busy":"2022-03-13T09:32:32.790229Z","iopub.execute_input":"2022-03-13T09:32:32.790821Z","iopub.status.idle":"2022-03-13T09:32:32.795798Z","shell.execute_reply.started":"2022-03-13T09:32:32.79077Z","shell.execute_reply":"2022-03-13T09:32:32.795043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Split the data into training and validation datasets","metadata":{"id":"4FlRu8ML-ceg","_uuid":"6563bbca143e4bceb1ea850714d7b43bb1e1178d"}},{"cell_type":"code","source":"# split dataset into training vs. validation dataset \n# split ratio is set to 0.9 vs. 0.1 (train vs. validation, respectively)\nimage_fps_list = list(image_fps)\nrandom.seed(42)\nrandom.shuffle(image_fps_list)\n\nval_size = 1500\nimage_fps_val = image_fps_list[:val_size]\nimage_fps_train = image_fps_list[val_size:]\n\nprint(len(image_fps_train), len(image_fps_val))","metadata":{"id":"7jByVCZt-ZOC","outputId":"f1aa267d-7530-4620-ffc5-2f7aa39083bb","_uuid":"6175c72e73639e3190e127f67783988eadced9ba","execution":{"iopub.status.busy":"2022-03-13T09:32:32.797187Z","iopub.execute_input":"2022-03-13T09:32:32.797755Z","iopub.status.idle":"2022-03-13T09:32:32.835481Z","shell.execute_reply.started":"2022-03-13T09:32:32.797703Z","shell.execute_reply":"2022-03-13T09:32:32.83491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create and prepare the training dataset using the DetectorDataset class.","metadata":{"id":"9KUvacUbgiEX","_uuid":"a5143c19dc22bc00d318a3b28cb7e13c7fbacc8a"}},{"cell_type":"code","source":"# prepare the training dataset\ndataset_train = DetectorDataset(image_fps_train, image_annotations, ORIG_SIZE, ORIG_SIZE)\ndataset_train.prepare()","metadata":{"id":"jwMkhotP0yFf","_uuid":"86c3333d4dfb8b7d00ce1f401693d0df4e6254e1","execution":{"iopub.status.busy":"2022-03-13T09:32:32.836588Z","iopub.execute_input":"2022-03-13T09:32:32.836829Z","iopub.status.idle":"2022-03-13T09:32:32.911423Z","shell.execute_reply.started":"2022-03-13T09:32:32.836783Z","shell.execute_reply":"2022-03-13T09:32:32.910725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Let's look at a sample annotation. We see a bounding box with (x, y) of the the top left corner as well as the width and height.","metadata":{"id":"wPDQ9EVDgxa6","_uuid":"4f69286e6b0b640827a3de166326d157a6d86668"}},{"cell_type":"code","source":"# Show annotation(s) for a DICOM image \ntest_fp = random.choice(image_fps_train)\nimage_annotations[test_fp]","metadata":{"id":"0xEc47Jz59x5","outputId":"129edfbc-cf9d-46c7-b569-d804a50cd12d","_uuid":"93da5a58731ad483a4bd2b20543f2b1df4b8ad74","execution":{"iopub.status.busy":"2022-03-13T09:32:32.912908Z","iopub.execute_input":"2022-03-13T09:32:32.91333Z","iopub.status.idle":"2022-03-13T09:32:32.919994Z","shell.execute_reply.started":"2022-03-13T09:32:32.913283Z","shell.execute_reply":"2022-03-13T09:32:32.919265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare the validation dataset\ndataset_val = DetectorDataset(image_fps_val, image_annotations, ORIG_SIZE, ORIG_SIZE)\ndataset_val.prepare()","metadata":{"id":"K1TkWuGP0yHl","_uuid":"313347d838fa8321a714858c8073f98c50c5be26","execution":{"iopub.status.busy":"2022-03-13T09:32:32.92129Z","iopub.execute_input":"2022-03-13T09:32:32.921705Z","iopub.status.idle":"2022-03-13T09:32:32.930881Z","shell.execute_reply.started":"2022-03-13T09:32:32.921658Z","shell.execute_reply":"2022-03-13T09:32:32.930268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Display a random image with bounding boxes","metadata":{"id":"pEXEt8fygWuC","_uuid":"600a8135d4e382f62797d69e9358f5697873c8f9"}},{"cell_type":"code","source":"# Load and display random sample and their bounding boxes\n\nclass_ids = [0]\nwhile class_ids[0] == 0:  ## look for a mask\n    image_id = random.choice(dataset_train.image_ids)\n    image_fp = dataset_train.image_reference(image_id)\n    image = dataset_train.load_image(image_id)\n    mask, class_ids = dataset_train.load_mask(image_id)\n\nprint(image.shape)\n\nplt.figure(figsize=(10, 10))\nplt.subplot(1, 2, 1)\nplt.imshow(image)\nplt.axis('off')\n\nplt.subplot(1, 2, 2)\nmasked = np.zeros(image.shape[:2])\nfor i in range(mask.shape[2]):\n    masked += image[:, :, 0] * mask[:, :, i]\nplt.imshow(masked, cmap='gray')\nplt.axis('off')\n\nprint(image_fp)\nprint(class_ids)","metadata":{"id":"4xwsrf9G1lHR","outputId":"a13386d3-a918-41fe-8824-13625c9d7b08","_uuid":"491b78ec96d28fcdbbf8e2d7f9320a05d64c9249","execution":{"iopub.status.busy":"2022-03-13T09:32:32.932933Z","iopub.execute_input":"2022-03-13T09:32:32.933173Z","iopub.status.idle":"2022-03-13T09:32:33.263827Z","shell.execute_reply.started":"2022-03-13T09:32:32.933118Z","shell.execute_reply":"2022-03-13T09:32:33.263056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image Augmentation. Try finetuning some variables to custom values","metadata":{"id":"ustAIH78hZI_","_uuid":"342b6008873fe7a6a0870a712ee47a87f0d2828d"}},{"cell_type":"code","source":"# Image augmentation (light but constant)\naugmentation = iaa.Sequential([\n    iaa.OneOf([ ## geometric transform\n        iaa.Affine(\n            scale={\"x\": (0.98, 1.02), \"y\": (0.98, 1.02)},\n            translate_percent={\"x\": (-0.02, 0.02), \"y\": (-0.04, 0.04)},\n            rotate=(-2, 2),\n            shear=(-1, 1),\n        ),\n        iaa.PiecewiseAffine(scale=(0.001, 0.025)),\n    ]),\n    iaa.OneOf([ ## brightness or contrast\n        iaa.Multiply((0.9, 1.1)),\n        iaa.ContrastNormalization((0.9, 1.1)),\n    ]),\n    iaa.OneOf([ ## blur or sharpen\n        iaa.GaussianBlur(sigma=(0.0, 0.1)),\n        iaa.Sharpen(alpha=(0.0, 0.1)),\n    ]),\n])\n\n# test on the same image as above\nimggrid = augmentation.draw_grid(image[:, :, 0], cols=5, rows=2)\nplt.figure(figsize=(30, 12))\n_ = plt.imshow(imggrid[:, :, 0], cmap='gray')","metadata":{"id":"STZnQTE61lME","_uuid":"4ab9d6086ce611a46f189c047956c43b29783e6d","execution":{"iopub.status.busy":"2022-03-13T09:32:33.2649Z","iopub.execute_input":"2022-03-13T09:32:33.265211Z","iopub.status.idle":"2022-03-13T09:32:38.369492Z","shell.execute_reply.started":"2022-03-13T09:32:33.265147Z","shell.execute_reply":"2022-03-13T09:32:38.366103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Now it's time to train the model. Note that training even a basic model can take a few hours. \n\n- dataset_train and dataset_val are derived from DetectorDataset \n- DetectorDataset loads images from image filenames and  masks from the annotation data\n- model is Mask-RCNN","metadata":{"id":"M4kt7LKuc78e","_uuid":"7e65d2cecb283f446f34cdde19b663a8a8e9590f"}},{"cell_type":"code","source":"model = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)\n\n# Exclude the last layers because they require a matching\n# number of classes\nmodel.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[\n    \"mrcnn_class_logits\", \"mrcnn_bbox_fc\",\n    \"mrcnn_bbox\", \"mrcnn_mask\"])","metadata":{"_uuid":"138d6197fc8dce9f1f8a7b5a6c27aa2069698e03","execution":{"iopub.status.busy":"2022-03-13T09:32:38.370845Z","iopub.execute_input":"2022-03-13T09:32:38.371356Z","iopub.status.idle":"2022-03-13T09:32:49.118437Z","shell.execute_reply.started":"2022-03-13T09:32:38.371297Z","shell.execute_reply":"2022-03-13T09:32:49.117727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LEARNING_RATE = 0.005\n\n# Train Mask-RCNN Model \nimport warnings \nwarnings.filterwarnings(\"ignore\")","metadata":{"id":"RVgNhHjl1lOS","outputId":"2cba9efc-eeea-472d-d155-3c3d856585bf","_uuid":"64cce2581ffdb8c2b1cb07948ada4a93f64874b0","execution":{"iopub.status.busy":"2022-03-13T09:32:49.119782Z","iopub.execute_input":"2022-03-13T09:32:49.120048Z","iopub.status.idle":"2022-03-13T09:32:49.125484Z","shell.execute_reply.started":"2022-03-13T09:32:49.120001Z","shell.execute_reply":"2022-03-13T09:32:49.124518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n## train heads with higher lr to speedup the learning\nmodel.train(dataset_train, dataset_val,\n            learning_rate=LEARNING_RATE*2,\n            epochs=2,\n            layers='heads',\n            augmentation=None)  ## no need to augment yet\nhistory = model.keras_model.history.history.copy()","metadata":{"_uuid":"cf339a499519d174bcdf2311a1802f0e3acb1758","execution":{"iopub.status.busy":"2022-03-13T09:32:49.126796Z","iopub.execute_input":"2022-03-13T09:32:49.127327Z","iopub.status.idle":"2022-03-13T09:58:10.478791Z","shell.execute_reply.started":"2022-03-13T09:32:49.127279Z","shell.execute_reply":"2022-03-13T09:58:10.477652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.train(dataset_train, dataset_val,\n            learning_rate=LEARNING_RATE,\n            epochs=2,\n            layers='all',\n            augmentation=augmentation)\nnews = model.keras_model.history.history\nfor k in news: history[k] = history[k] + news[k]","metadata":{"_uuid":"8004790d27f041793562e994bbe95edf67f8978b","execution":{"iopub.status.busy":"2022-03-13T09:58:10.48167Z","iopub.execute_input":"2022-03-13T09:58:10.483346Z","iopub.status.idle":"2022-03-13T09:58:34.443984Z","shell.execute_reply.started":"2022-03-13T09:58:10.483292Z","shell.execute_reply":"2022-03-13T09:58:34.442819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.train(dataset_train, dataset_val,\n            learning_rate=LEARNING_RATE/5,\n            epochs=5,\n            layers='all',\n            augmentation=augmentation)\nnews = model.keras_model.history.history\nfor k in news: history[k] = history[k] + news[k]","metadata":{"_uuid":"ccea214a520c686735e138f64977dcd7f3e3330a","execution":{"iopub.status.busy":"2022-03-13T09:58:34.638023Z","iopub.execute_input":"2022-03-13T09:58:34.639219Z","iopub.status.idle":"2022-03-13T11:17:19.704862Z","shell.execute_reply.started":"2022-03-13T09:58:34.639166Z","shell.execute_reply":"2022-03-13T11:17:19.700589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = range(1,len(next(iter(history.values())))+1)\npd.DataFrame(history, index=epochs)","metadata":{"_uuid":"cfce1fa95369d39f958d269e27b07332b9d36f89","execution":{"iopub.status.busy":"2022-03-13T11:17:19.736966Z","iopub.execute_input":"2022-03-13T11:17:19.73883Z","iopub.status.idle":"2022-03-13T11:17:19.810555Z","shell.execute_reply.started":"2022-03-13T11:17:19.738772Z","shell.execute_reply":"2022-03-13T11:17:19.809624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.subplot(111)\nplt.plot(epochs, history[\"loss\"], label=\"Train loss\")\nplt.plot(epochs, history[\"val_loss\"], label=\"Valid loss\")\nplt.legend()\nplt.show()","metadata":{"_uuid":"5edee0adf833e2ff6ff86c6ee65d782a8486d3c9","execution":{"iopub.status.busy":"2022-03-13T11:17:19.813567Z","iopub.execute_input":"2022-03-13T11:17:19.815298Z","iopub.status.idle":"2022-03-13T11:17:20.204322Z","shell.execute_reply.started":"2022-03-13T11:17:19.815237Z","shell.execute_reply":"2022-03-13T11:17:20.203331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select trained model \ndir_names = next(os.walk(model.model_dir))[1]\nkey = config.NAME.lower()\ndir_names = filter(lambda f: f.startswith(key), dir_names)\ndir_names = sorted(dir_names)\n\nif not dir_names:\n    import errno\n    raise FileNotFoundError(\n        errno.ENOENT,\n        \"Could not find model directory under {}\".format(self.model_dir))\n    \nfps = []\n# Pick last directory\nfor d in dir_names: \n    dir_name = os.path.join(model.model_dir, d)\n    # Find the last checkpoint\n    checkpoints = next(os.walk(dir_name))[2]\n    checkpoints = filter(lambda f: f.startswith(\"mask_rcnn\"), checkpoints)\n    checkpoints = sorted(checkpoints)\n    if not checkpoints:\n        print('No weight files in {}'.format(dir_name))\n    else:\n        checkpoint = os.path.join(dir_name, checkpoints[-1])\n        fps.append(checkpoint)\n\nmodel_path = sorted(fps)[-1]\nprint('Found model {}'.format(model_path))","metadata":{"id":"eraRlzgPmmIZ","outputId":"de9e688c-ba4f-4b62-f842-dbcf00ce397c","_uuid":"db5c10d3f7da099e5751a04a6e6d49819882ecd4","execution":{"iopub.status.busy":"2022-03-13T11:17:20.207262Z","iopub.execute_input":"2022-03-13T11:17:20.208757Z","iopub.status.idle":"2022-03-13T11:17:20.235989Z","shell.execute_reply.started":"2022-03-13T11:17:20.208697Z","shell.execute_reply":"2022-03-13T11:17:20.234385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InferenceConfig(DetectorConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\ninference_config = InferenceConfig()\n\n# Recreate the model in inference mode\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=ROOT_DIR)\n\n# Load trained weights (fill in path to trained weights here)\nassert model_path != \"\", \"Provide path to trained weights\"\nprint(\"Loading weights from \", model_path)\nmodel.load_weights(model_path, by_name=True)","metadata":{"id":"TgpT9AzC2Bgz","outputId":"60f5a175-4666-497d-b4e8-0bdab39a92d0","_uuid":"52138636b2ae5bf444bba808518cd8313bde65cd","execution":{"iopub.status.busy":"2022-03-13T11:17:20.239333Z","iopub.execute_input":"2022-03-13T11:17:20.241894Z","iopub.status.idle":"2022-03-13T11:17:31.229906Z","shell.execute_reply.started":"2022-03-13T11:17:20.241812Z","shell.execute_reply":"2022-03-13T11:17:31.229022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set color for class\ndef get_colors_for_class_ids(class_ids):\n    colors = []\n    for class_id in class_ids:\n        if class_id == 1:\n            colors.append((.941, .204, .204))\n    return colors","metadata":{"id":"9mTBig7D2BjU","_uuid":"e13c61bee23b791c61ecf1256f7512295cd4d9ab","execution":{"iopub.status.busy":"2022-03-13T11:17:31.231273Z","iopub.execute_input":"2022-03-13T11:17:31.231747Z","iopub.status.idle":"2022-03-13T11:17:31.237246Z","shell.execute_reply.started":"2022-03-13T11:17:31.231695Z","shell.execute_reply":"2022-03-13T11:17:31.236436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### How does the predicted box compared to the expected value? Let's use the validation dataset to check. \n\nNote that we trained only one epoch for **demonstration purposes ONLY**. You might be able to improve performance running more epochs. ","metadata":{"id":"A8EiL2LOiCr_","_uuid":"f99fbd3f31ff1a2bd66764835c9b646375364598"}},{"cell_type":"code","source":"# Show few example of ground truth vs. predictions on the validation dataset \ndataset = dataset_val\nfig = plt.figure(figsize=(10, 30))\n\nfor i in range(6):\n\n    image_id = random.choice(dataset.image_ids)\n    \n    original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n        modellib.load_image_gt(dataset_val, inference_config, \n                               image_id, use_mini_mask=False)\n    \n    print(original_image.shape)\n    plt.subplot(6, 2, 2*i + 1)\n    visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, \n                                dataset.class_names,\n                                colors=get_colors_for_class_ids(gt_class_id), ax=fig.axes[-1])\n    \n    plt.subplot(6, 2, 2*i + 2)\n    results = model.detect([original_image]) #, verbose=1)\n    r = results[0]\n    visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], \n                                dataset.class_names, r['scores'], \n                                colors=get_colors_for_class_ids(r['class_ids']), ax=fig.axes[-1])","metadata":{"id":"irheTbrW2Bl0","outputId":"56041ad4-173d-45ab-af67-f54e8333511e","_uuid":"186412199e25b98719f71cfe5e8869abcce516c4","execution":{"iopub.status.busy":"2022-03-13T11:17:31.238558Z","iopub.execute_input":"2022-03-13T11:17:31.239034Z","iopub.status.idle":"2022-03-13T11:17:36.789748Z","shell.execute_reply.started":"2022-03-13T11:17:31.238983Z","shell.execute_reply":"2022-03-13T11:17:36.782298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get filenames of test dataset DICOM images\ntest_image_fps = get_dicom_fps(test_dicom_dir)","metadata":{"id":"qRWBVJKYNdWM","_uuid":"fd9f53fa319a425693e07fe4898ddeeaa5d07f99","execution":{"iopub.status.busy":"2022-03-13T11:17:36.791421Z","iopub.execute_input":"2022-03-13T11:17:36.792077Z","iopub.status.idle":"2022-03-13T11:17:37.107139Z","shell.execute_reply.started":"2022-03-13T11:17:36.79201Z","shell.execute_reply":"2022-03-13T11:17:37.106273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Final steps - Create the submission file","metadata":{"id":"WcV1cL_aiSc4","_uuid":"164e18701a830bc6c42a791feea13549de37289b"}},{"cell_type":"code","source":"# Make predictions on test images, write out sample submission\ndef predict(image_fps, filepath='submission.csv', min_conf=0.95):\n    # assume square image\n    resize_factor = ORIG_SIZE / config.IMAGE_SHAPE[0]\n    #resize_factor = ORIG_SIZE\n    with open(filepath, 'w') as file:\n        file.write(\"patientId,PredictionString\\n\")\n\n        for image_id in tqdm(image_fps):\n            ds = pydicom.read_file(image_id)\n            image = ds.pixel_array\n            # If grayscale. Convert to RGB for consistency.\n            if len(image.shape) != 3 or image.shape[2] != 3:\n                image = np.stack((image,) * 3, -1)\n            image, window, scale, padding, crop = utils.resize_image(\n                image,\n                min_dim=config.IMAGE_MIN_DIM,\n                min_scale=config.IMAGE_MIN_SCALE,\n                max_dim=config.IMAGE_MAX_DIM,\n                mode=config.IMAGE_RESIZE_MODE)\n\n            patient_id = os.path.splitext(os.path.basename(image_id))[0]\n\n            results = model.detect([image])\n            r = results[0]\n\n            out_str = \"\"\n            out_str += patient_id\n            out_str += \",\"\n            assert( len(r['rois']) == len(r['class_ids']) == len(r['scores']) )\n            if len(r['rois']) == 0:\n                pass\n            else:\n                num_instances = len(r['rois'])\n\n                for i in range(num_instances):\n                    if r['scores'][i] > min_conf:\n                        out_str += ' '\n                        out_str += str(round(r['scores'][i], 2))\n                        out_str += ' '\n\n                        # x1, y1, width, height\n                        x1 = r['rois'][i][1]\n                        y1 = r['rois'][i][0]\n                        width = r['rois'][i][3] - x1\n                        height = r['rois'][i][2] - y1\n                        bboxes_str = \"{} {} {} {}\".format(x1*resize_factor, y1*resize_factor, \\\n                                                           width*resize_factor, height*resize_factor)\n                        out_str += bboxes_str\n\n            file.write(out_str+\"\\n\")","metadata":{"id":"C6UWVrbM2Bob","_uuid":"4a5c0c6134408ddbf5a34496d7e9d7be5692e9a1","execution":{"iopub.status.busy":"2022-03-13T11:17:37.108631Z","iopub.execute_input":"2022-03-13T11:17:37.109111Z","iopub.status.idle":"2022-03-13T11:17:37.128141Z","shell.execute_reply.started":"2022-03-13T11:17:37.109059Z","shell.execute_reply":"2022-03-13T11:17:37.127169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_fp = os.path.join(ROOT_DIR, 'submission.csv')\npredict(test_image_fps, filepath=submission_fp)\nprint(submission_fp)","metadata":{"id":"C5cBpNka2Bsv","outputId":"a2af9176-d9d6-49f6-f22a-5a1c455d144f","_uuid":"0406e7f5aaa4867782c4f9c064f90bba386128e7","execution":{"iopub.status.busy":"2022-03-13T11:17:37.129646Z","iopub.execute_input":"2022-03-13T11:17:37.130239Z","iopub.status.idle":"2022-03-13T11:27:15.948233Z","shell.execute_reply.started":"2022-03-13T11:17:37.130159Z","shell.execute_reply":"2022-03-13T11:27:15.947353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = pd.read_csv(submission_fp, names=['patientId', 'PredictionString'])\noutput.head(60)","metadata":{"id":"_BjPE_Ee9rbA","outputId":"67b5f053-112b-494a-9ab3-d017bfb440c2","_uuid":"3fd8d178fc51ef0bca94fbb3f423160f08a77edc","execution":{"iopub.status.busy":"2022-03-13T11:27:15.951116Z","iopub.execute_input":"2022-03-13T11:27:15.952835Z","iopub.status.idle":"2022-03-13T11:27:16.050234Z","shell.execute_reply.started":"2022-03-13T11:27:15.952768Z","shell.execute_reply":"2022-03-13T11:27:16.049355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show a few test image detection example\ndef visualize(): \n    image_id = random.choice(test_image_fps)\n    ds = pydicom.read_file(image_id)\n    \n    # original image \n    image = ds.pixel_array\n    \n    # assume square image \n    resize_factor = ORIG_SIZE / config.IMAGE_SHAPE[0]\n    \n    # If grayscale. Convert to RGB for consistency.\n    if len(image.shape) != 3 or image.shape[2] != 3:\n        image = np.stack((image,) * 3, -1) \n    resized_image, window, scale, padding, crop = utils.resize_image(\n        image,\n        min_dim=config.IMAGE_MIN_DIM,\n        min_scale=config.IMAGE_MIN_SCALE,\n        max_dim=config.IMAGE_MAX_DIM,\n        mode=config.IMAGE_RESIZE_MODE)\n\n    patient_id = os.path.splitext(os.path.basename(image_id))[0]\n    print(patient_id)\n\n    results = model.detect([resized_image])\n    r = results[0]\n    for bbox in r['rois']: \n        print(bbox)\n        x1 = int(bbox[1] * resize_factor)\n        y1 = int(bbox[0] * resize_factor)\n        x2 = int(bbox[3] * resize_factor)\n        y2 = int(bbox[2]  * resize_factor)\n        cv2.rectangle(image, (x1,y1), (x2,y2), (77, 255, 9), 3, 1)\n        width = x2 - x1 \n        height = y2 - y1 \n        print(\"x {} y {} h {} w {}\".format(x1, y1, width, height))\n    plt.figure() \n    plt.imshow(image, cmap=plt.cm.gist_gray)\n\nvisualize()\nvisualize()\nvisualize()\nvisualize()","metadata":{"_uuid":"ea110f197abc2acb1c3435383f7259079dc0eb0e","execution":{"iopub.status.busy":"2022-03-13T11:27:16.053035Z","iopub.execute_input":"2022-03-13T11:27:16.05483Z","iopub.status.idle":"2022-03-13T11:27:18.278176Z","shell.execute_reply.started":"2022-03-13T11:27:16.054758Z","shell.execute_reply":"2022-03-13T11:27:18.277392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove files to allow committing (hit files limit otherwise)\n!rm -rf /kaggle/working/Mask_RCNN","metadata":{"_uuid":"835a15c9d018acd5deb16e9e02f9b765f68d0e78","execution":{"iopub.status.busy":"2022-03-13T11:27:18.280656Z","iopub.execute_input":"2022-03-13T11:27:18.282141Z","iopub.status.idle":"2022-03-13T11:27:19.094544Z","shell.execute_reply.started":"2022-03-13T11:27:18.282085Z","shell.execute_reply":"2022-03-13T11:27:19.093233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}