{"cells":[{"metadata":{"id":"KBeAf8WgaeSk","colab_type":"text","_uuid":"7c1fce19a11f95416168ced03c2c70fa818b21a5"},"cell_type":"markdown","source":"I have forked the great work from [Henrique Mendonça](https://www.kaggle.com/hmendonca) and added some annotations (pun not-intended :p)\nmainly for myself since I am new to the [Mask R-CNN](https://github.com/matterport/Mask_RCNN) model.\n\nDon't forget to check (and upvote) the original notebook [here](https://www.kaggle.com/hmendonca/airbus-mask-rcnn-and-coco-transfer-learning).\n\nEnjoy!"},{"metadata":{"_uuid":"9c20b4c3981d252a0d14dbbc3724a77f95bd6095"},"cell_type":"markdown","source":"Before you start exploring this notebook, here are some useful resources:\n    \n- What is the Mask R-CNN loss function? https://stackoverflow.com/questions/46272841/what-is-the-loss-function-of-the-mask-rcnn. \n- Mask R-CNN original paper: https://arxiv.org/pdf/1703.06870.pdf. \n- A good blog post explaining the history behin Mask R-CNN: https://blog.athelas.com/a-brief-history-of-cnns-in-image-segmentation-from-r-cnn-to-mask-r-cnn-34ea83205de4. \n- Mask R-CNN video presentation by [Kaiming He](http://kaiminghe.com/): https://www.youtube.com/watch?v=g7z4mkfRjI4"},{"metadata":{"id":"4kjcC6QqywWl","colab_type":"code","colab":{},"trusted":true,"_uuid":"40c67b3ff0fa04587dec508363308adaa3ceaf34"},"cell_type":"code","source":"# The usual imports\nimport os \nimport sys\nimport random\nimport math\nimport numpy as np\nimport cv2\nimport matplotlib.pyplot as plt\nimport json\nfrom imgaug import augmenters as iaa\nfrom tqdm import tqdm\nimport pandas as pd \nimport glob ","execution_count":null,"outputs":[]},{"metadata":{"id":"yP0XLJx_x_6o","colab_type":"code","colab":{},"trusted":true,"_uuid":"6e5764759e6a0a9b698b44645658f66873edd807"},"cell_type":"code","source":"# Some constants\ndebug = False\nDATA_DIR = '/kaggle/input/airbus-ship-detection'\n\n# Directory to save logs and trained model\nROOT_DIR = '/kaggle/working'\nSEED = 314\n# Probably corrupted images\nIMGS_TO_EXCLUDE = ['6384c3e78.jpg','13703f040.jpg', '14715c06d.jpg',  '33e0ff2d5.jpg',\n                   '4d4e09f2a.jpg', '877691df8.jpg', '8b909bb20.jpg', 'a8d99130e.jpg', \n                   'ad55c3143.jpg', 'c8260c541.jpg', 'd6c7f17c7.jpg', 'dc3e7c901.jpg',\n                   'e44dffe88.jpg', 'ef87bad36.jpg', 'f083256d8.jpg']\n# The image size (before augmentation)\nIMG_SIZE = (768, 768)","execution_count":null,"outputs":[]},{"metadata":{"id":"kdYzLq1zfKL4","colab_type":"text","_uuid":"576df4c47a23d08b1bdb384245e09aa69f88bbd3"},"cell_type":"markdown","source":"### Install Matterport's Mask-RCNN model from github.\nSee the [Matterport's implementation of Mask-RCNN](https://github.com/matterport/Mask_RCNN)."},{"metadata":{"id":"KgllzLnDr7kF","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":143},"outputId":"6c978df7-2013-437e-acd1-5011048dfb53","trusted":true,"_uuid":"b37d22551d332f0f7b722cc7204eb614524b6c21"},"cell_type":"code","source":"!git clone https://www.github.com/matterport/Mask_RCNN.git\nos.chdir('Mask_RCNN')","execution_count":null,"outputs":[]},{"metadata":{"id":"-KZXyWwhzOVU","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"outputId":"2576cc17-7484-4311-ad72-3c5643dcb5bb","trusted":true,"_uuid":"3acbbbe055b6a409d3c50ae0f893acf51b5ae7ba"},"cell_type":"code","source":"# Import Mask RCNN\nsys.path.append(os.path.join(ROOT_DIR, 'Mask_RCNN'))  # To find local version of the library\nfrom mrcnn.config import Config\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","execution_count":null,"outputs":[]},{"metadata":{"id":"FghMmiMjzOX2","colab_type":"code","colab":{},"trusted":true,"_uuid":"50089cc61791871cdf6a5c0037dc4f28b7b7d7cc"},"cell_type":"code","source":"train_dicom_dir = os.path.join(DATA_DIR, 'train_v2')\ntest_dicom_dir = os.path.join(DATA_DIR, 'test_v2')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f108beef7838be8a64dd512d395c5dc0ad952790"},"cell_type":"markdown","source":"### Download COCO pre-trained weights"},{"metadata":{"trusted":true,"_uuid":"c3ee0cd0ee0b1defdec97b94bc736587c1f7631f"},"cell_type":"code","source":"!wget --quiet https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5\n!ls -lh mask_rcnn_coco.h5\n\nCOCO_WEIGHTS_PATH = \"mask_rcnn_coco.h5\"","execution_count":null,"outputs":[]},{"metadata":{"id":"gj-tvDvEaDiC","colab_type":"text","_uuid":"032cc5fe4baa051108106675e6ca4f4fdb2846ed"},"cell_type":"markdown","source":"### Some setup functions and classes for Mask-RCNN\n\n- dicom_fps is a list of the dicom image path and filenames \n- image_annotions is a dictionary of the annotations keyed by the filenames\n- parsing the dataset returns a list of the image filenames and the annotations dictionary"},{"metadata":{"id":"_SfzTa-1zOck","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":917},"outputId":"91ae8935-bccb-4b8e-9a7e-aa690f95fd9b","trusted":true,"_uuid":"dfcffc4eaa94a41497717851dee9f702d8a2a73b"},"cell_type":"code","source":"# TODO: Tweak these and see what happens.\n\nclass DetectorConfig(Config):    \n    # Give the configuration a recognizable name  \n    NAME = 'airbus'\n    \n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 8\n    \n    BACKBONE = 'resnet50'\n    \n    NUM_CLASSES = 2  # background and ship classes\n    \n    IMAGE_MIN_DIM = 384\n    IMAGE_MAX_DIM = 384\n    RPN_ANCHOR_SCALES = (8, 16, 32, 64)\n    TRAIN_ROIS_PER_IMAGE = 64\n    MAX_GT_INSTANCES = 14\n    DETECTION_MAX_INSTANCES = 14\n    DETECTION_MIN_CONFIDENCE = 0.95\n    DETECTION_NMS_THRESHOLD = 0.0\n\n    STEPS_PER_EPOCH = 12 if debug else 120\n    VALIDATION_STEPS = 10 if debug else 100\n\nconfig = DetectorConfig()\nconfig.display()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6136132b1f1b311e297d9432772ec4a81230924f"},"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom skimage.io import imread\nimport matplotlib.pyplot as plt\nfrom matplotlib.cm import get_cmap\nfrom skimage.segmentation import mark_boundaries\nfrom skimage.util import montage\nfrom skimage.morphology import binary_opening, disk, label\nimport gc; gc.enable() # memory is tight\n\nmontage_rgb = lambda x: np.stack([montage(x[:, :, :, i]) for i in range(x.shape[3])], -1)\n\ndef multi_rle_encode(img, **kwargs):\n    '''\n    Encode connected regions as separated masks\n    '''\n    labels = label(img)\n    if img.ndim > 2:\n        return [rle_encode(np.sum(labels==k, axis=2), **kwargs) for k in np.unique(labels[labels>0])]\n    else:\n        return [rle_encode(labels==k, **kwargs) for k in np.unique(labels[labels>0])]\n\n# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\ndef rle_encode(img, min_max_threshold=1e-3, max_mean_threshold=None):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    if np.max(img) < min_max_threshold:\n        return '' ## no need to encode if it's all zeros\n    if max_mean_threshold and np.mean(img) > max_mean_threshold:\n        return '' ## ignore overfilled mask\n    pixels = img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef rle_decode(mask_rle, shape=(768, 768)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T  # Needed to align to RLE direction\n\ndef masks_as_image(in_mask_list):\n    # Take the individual ship masks and create a single mask array for all ships\n    all_masks = np.zeros((768, 768), dtype = np.uint8)\n    for mask in in_mask_list:\n        if isinstance(mask, str):\n            all_masks |= rle_decode(mask)\n    return all_masks\n\ndef masks_as_color(in_mask_list):\n    # Take the individual ship masks and create a color mask array for each ships\n    all_masks = np.zeros((768, 768), dtype = np.float)\n    scale = lambda x: (len(in_mask_list)+x+1) / (len(in_mask_list)*2) ## scale the heatmap image to shift \n    for i,mask in enumerate(in_mask_list):\n        if isinstance(mask, str):\n            all_masks[:,:] += scale(i) * rle_decode(mask)\n    return all_masks","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3422d00bd4ed2a98e608c690954da8e9d8ba1b8d"},"cell_type":"markdown","source":"# Train, test, and segmentation files exploration"},{"metadata":{"trusted":true,"_uuid":"d3e05fa1a38c637fa228acd62b92dd41117a6672"},"cell_type":"code","source":"from PIL import Image\nfrom sklearn.model_selection import train_test_split\n\ntrain_names = [f for f in os.listdir(train_dicom_dir)]\ntest_names = [f for f in os.listdir(test_dicom_dir)]\nfor el in IMGS_TO_EXCLUDE:\n    if(el in train_names): train_names.remove(el)\n    if(el in test_names): test_names.remove(el)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"9a1087db78370d02463e6804407c5403dc23cd1d"},"cell_type":"code","source":"f'There are {len(train_names)} train images, and {len(test_names)} test ones'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3050fa77026411ffdc27bed4a9b667ec0467e4ce","scrolled":false},"cell_type":"code","source":"# Segmentation train dataset: link between train images and train RLE masks.\nSEGMENTATION_PATH = os.path.join(DATA_DIR, 'train_ship_segmentations_v2.csv')\nsegmentation_df = pd.read_csv(SEGMENTATION_PATH)\nsegmentation_df.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"bdb9217d1332f1c60d139ba5fb4fec03c09f0991"},"cell_type":"code","source":"segmentation_df.groupby('ImageId')['EncodedPixels'].count().hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"857c8f036aa6e6a47c5c17ab1559ff680b33d516"},"cell_type":"markdown","source":"As you can see, most train images are empty. What are the percentages?"},{"metadata":{"trusted":true,"_uuid":"2d47be1fc371b57ea10217d599af0f60a21adf62"},"cell_type":"code","source":"(segmentation_df.groupby('ImageId')['EncodedPixels']\n               .count()\n               .value_counts(normalize=True)\n               .mul(100)\n               .to_dict())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1b3e9dbaa17024e23081acf150797b0fdf404da"},"cell_type":"code","source":"# Select unique train files with at least one ship mask.\nships_train_names = (segmentation_df.loc[segmentation_df.EncodedPixels.notnull(), \n                                        'ImageId']\n                                    .unique())\nno_ships_train_names = (segmentation_df.loc[segmentation_df.EncodedPixels.isnull(), \n                                        'ImageId']\n                                    .unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"ca7173cbbbea921b3e45c7b270bf1beac1fa40e6"},"cell_type":"code","source":"(f'There are {len(ships_train_names)} unique train images with at least one shipe and '\n  f'{len(no_ships_train_names)} without any')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"904636402355a305f7b2ccacb8cc55d52151d2e6"},"cell_type":"code","source":"# TODO: Add some comments.\ntest_size = config.VALIDATION_STEPS * config.IMAGES_PER_GPU\nimage_fps_train, image_fps_val = train_test_split(ships_train_names, \n                                                  test_size=test_size, \n                                                  random_state=SEED)\n\nprint(len(image_fps_train), len(image_fps_val), len(test_names))","execution_count":null,"outputs":[]},{"metadata":{"id":"8EBVA1M60yAj","colab_type":"code","colab":{},"trusted":true,"_uuid":"52bd3ffbdde0173a363055482d675da51c2aba99"},"cell_type":"code","source":"# TODO: Add some documentation\n\nclass DetectorDataset(utils.Dataset):\n    \"\"\"Dataset class for training our dataset.\n    \"\"\"\n\n    def __init__(self, image_fps, image_annotations, orig_height, orig_width):\n        super().__init__(self)\n        \n        # Add classes\n        self.add_class('ship', 1, 'Ship')\n        \n        # add images \n        for i, fp in enumerate(image_fps):\n            annotations = image_annotations.query('ImageId==\"' + fp + '\"')['EncodedPixels']\n            self.add_image('ship', image_id=i, path=os.path.join(train_dicom_dir, fp), \n                           annotations=annotations, orig_height=orig_height, orig_width=orig_width)\n            \n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path']\n\n    def load_image(self, image_id):\n        info = self.image_info[image_id]\n        fp = info['path']\n        image = imread(fp)\n        # If grayscale. Convert to RGB for consistency.\n        if len(image.shape) != 3 or image.shape[2] != 3:\n            image = np.stack((image,) * 3, -1)\n        return image\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n        annotations = info['annotations']\n        count = len(annotations)\n        if count == 0:\n            mask = np.zeros((info['orig_height'], info['orig_width'], 1), dtype=np.uint8)\n            class_ids = np.zeros((1,), dtype=np.int32)\n        else:\n            mask = np.zeros((info['orig_height'], info['orig_width'], count), dtype=np.uint8)\n            class_ids = np.zeros((count,), dtype=np.int32)\n            for i, a in enumerate(annotations):\n                mask[:, :, i] = rle_decode(a)\n                class_ids[i] = 1\n        return mask.astype(np.bool), class_ids.astype(np.int32)","execution_count":null,"outputs":[]},{"metadata":{"id":"9RlMo04ckd98","colab_type":"text","_uuid":"1cb852e262b69d348743767d675573368ab672c9"},"cell_type":"markdown","source":"### Examine the annotation data, parse the dataset, and view dicom fields"},{"metadata":{"id":"Mxz-pNbt5txY","colab_type":"code","colab":{},"trusted":true,"_uuid":"7aebc88f910b232e3b8759421914a007c6ffed94"},"cell_type":"code","source":"# Reassign variables for Mask R-CNN\nimage_fps, image_annotations = train_names, segmentation_df","execution_count":null,"outputs":[]},{"metadata":{"id":"YPqjEIXWRhSf","colab_type":"code","colab":{},"trusted":true,"_uuid":"6c386dcef041b972f6209dd19e247d547c3c349f"},"cell_type":"code","source":"ds = imread(os.path.join(train_dicom_dir, train_names[10])) \n_ = plt.imshow(ds)","execution_count":null,"outputs":[]},{"metadata":{"id":"9KUvacUbgiEX","colab_type":"text","_uuid":"a5143c19dc22bc00d318a3b28cb7e13c7fbacc8a"},"cell_type":"markdown","source":"### Create and prepare the training dataset using the DetectorDataset class."},{"metadata":{"id":"jwMkhotP0yFf","colab_type":"code","colab":{},"trusted":true,"_uuid":"86c3333d4dfb8b7d00ce1f401693d0df4e6254e1"},"cell_type":"code","source":"%%time\n# prepare the training dataset\ndataset_train = DetectorDataset(image_fps_train, image_annotations, *IMG_SIZE)\ndataset_train.prepare()","execution_count":null,"outputs":[]},{"metadata":{"id":"K1TkWuGP0yHl","colab_type":"code","colab":{},"trusted":true,"_uuid":"313347d838fa8321a714858c8073f98c50c5be26"},"cell_type":"code","source":"%%time\n# prepare the validation dataset\ndataset_val = DetectorDataset(image_fps_val, image_annotations, *IMG_SIZE)\ndataset_val.prepare()","execution_count":null,"outputs":[]},{"metadata":{"id":"pEXEt8fygWuC","colab_type":"text","_uuid":"600a8135d4e382f62797d69e9358f5697873c8f9"},"cell_type":"markdown","source":"### Display a random image with bounding boxes"},{"metadata":{"id":"4xwsrf9G1lHR","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":359},"outputId":"a13386d3-a918-41fe-8824-13625c9d7b08","trusted":true,"_uuid":"491b78ec96d28fcdbbf8e2d7f9320a05d64c9249"},"cell_type":"code","source":"# Load and display random sample and their bounding boxes\n\nclass_ids = [0]\nwhile class_ids[0] == 0:  ## look for a mask\n    image_id = random.choice(dataset_val.image_ids)\n    image_fp = dataset_val.image_reference(image_id)\n    image = dataset_val.load_image(image_id)\n    mask, class_ids = dataset_val.load_mask(image_id)\n\nprint(image.shape)\n\nplt.figure(figsize=(10, 10))\nplt.subplot(1, 2, 1)\nplt.imshow(image)\nplt.axis('off')\n\nplt.subplot(1, 2, 2)\nmasked = np.zeros(image.shape[:2])\nfor i in range(mask.shape[2]):\n    masked += mask[:, :, i] ## * image[:, :, 0]\nplt.imshow(masked, cmap='gray')\nplt.axis('off')\n\nprint(image_fp)\nprint(class_ids)","execution_count":null,"outputs":[]},{"metadata":{"id":"ustAIH78hZI_","colab_type":"text","_uuid":"342b6008873fe7a6a0870a712ee47a87f0d2828d"},"cell_type":"markdown","source":"### Image Augmentation. Try finetuning some variables to custom values"},{"metadata":{"id":"STZnQTE61lME","colab_type":"code","colab":{},"trusted":true,"_uuid":"4ab9d6086ce611a46f189c047956c43b29783e6d"},"cell_type":"code","source":"# Image augmentation (light but constant)\naugmentation = iaa.Sequential([\n    iaa.OneOf([ ## rotate\n        iaa.Affine(rotate=0),\n        iaa.Affine(rotate=90),\n        iaa.Affine(rotate=180),\n        iaa.Affine(rotate=270),\n    ]),\n    iaa.Fliplr(0.5),\n    iaa.Flipud(0.5),\n    iaa.OneOf([ ## brightness or contrast\n        iaa.Multiply((0.9, 1.1)),\n        iaa.ContrastNormalization((0.9, 1.1)),\n    ]),\n    iaa.OneOf([ ## blur or sharpen\n        iaa.GaussianBlur(sigma=(0.0, 0.1)),\n        iaa.Sharpen(alpha=(0.0, 0.1)),\n    ]),\n])\n\n# test on the same image as above\nimggrid = augmentation.draw_grid(image, cols=5, rows=2)\nplt.figure(figsize=(30, 12))\n_ = plt.imshow(imggrid.astype(int))","execution_count":null,"outputs":[]},{"metadata":{"id":"M4kt7LKuc78e","colab_type":"text","_uuid":"7e65d2cecb283f446f34cdde19b663a8a8e9590f"},"cell_type":"markdown","source":"### Now it's time to train the model. Note that training even a basic model can take a few hours. \n\nNote: the following model is for demonstration purpose only. We have limited the training to one epoch, and have set nominal values for the Detector Configuration to reduce run-time. \n\n- dataset_train and dataset_val are derived from DetectorDataset \n- DetectorDataset loads images from image filenames and  masks from the annotation data\n- model is Mask-RCNN"},{"metadata":{"trusted":true,"_uuid":"138d6197fc8dce9f1f8a7b5a6c27aa2069698e03"},"cell_type":"code","source":"model = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)\n\n# Exclude the last layers because they require a matching\n# number of classes\nmodel.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[\n    \"mrcnn_class_logits\", \"mrcnn_bbox_fc\",\n    \"mrcnn_bbox\", \"mrcnn_mask\"])","execution_count":null,"outputs":[]},{"metadata":{"id":"RVgNhHjl1lOS","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":2575},"outputId":"2cba9efc-eeea-472d-d155-3c3d856585bf","trusted":true,"_uuid":"64cce2581ffdb8c2b1cb07948ada4a93f64874b0"},"cell_type":"code","source":"# TODO: Is this the optimal one? Investigate...\nSTART_LEARNING_RATE = 0.006\n\n# Train Mask-RCNN Model \nimport warnings \nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d98ea11b49d3a73502bb7a1cba1634c68d09167"},"cell_type":"markdown","source":"The strategy here is to train the model in three phases: \n    \n1. Bigger learning rate with only the heads layers and no augmentation for few epochs (for 2 epochs here)\n2. Smaller learning rate (half the previous one) with all the layers (for 12 epochs here)\n3. Even smaller learning rate (half the previous one) with all the layers (for 20 epochs here)\n\nNotice that this strategy could be refined of course and the various \nhyperparamters could be improved."},{"metadata":{"_uuid":"7ae7fdf706350f201e40e8fd52da30b8710142f0"},"cell_type":"markdown","source":"TODO: Group the various training variations into a single function."},{"metadata":{"trusted":true,"_uuid":"cf339a499519d174bcdf2311a1802f0e3acb1758"},"cell_type":"code","source":"def training_strategy(start_learning_rate):\n\n    model.train(dataset_train, dataset_val,\n                learning_rate=start_learning_rate,\n                epochs=2,\n                layers='heads',\n                augmentation=None)  ## no need to augment yet\n\n\n    model.train(dataset_train, dataset_val,\n                learning_rate=start_learning_rate / 2,\n                epochs=12,\n                layers='all',\n                augmentation=augmentation)\n\n\n    model.train(dataset_train, dataset_val,\n                learning_rate= start_learning_rate / 4,\n                epochs=20,\n                layers='all',\n                augmentation=augmentation)\n    \n\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"f5da7bbcd3bea72ea540de43b71677ec96f25b42"},"cell_type":"code","source":"%time model = training_strategy(START_LEARNING_RATE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae882d5ab3c46ce917b1b5f3d0638b977ac5a613"},"cell_type":"code","source":"history = model.keras_model.history.history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb3b69242b91dcc49697ff076ceeb957347372e1"},"cell_type":"code","source":"plt.figure(figsize=(17,5))\n\nplt.subplot(131)\nplt.plot(epochs, history[\"loss\"], label=\"Train loss\")\nplt.plot(epochs, history[\"val_loss\"], label=\"Valid loss\")\nplt.legend()\nplt.subplot(132)\nplt.plot(epochs, history[\"mrcnn_class_loss\"], label=\"Train class ce\")\nplt.plot(epochs, history[\"val_mrcnn_class_loss\"], label=\"Valid class ce\")\nplt.legend()\nplt.subplot(133)\nplt.plot(epochs, history[\"mrcnn_bbox_loss\"], label=\"Train box loss\")\nplt.plot(epochs, history[\"val_mrcnn_bbox_loss\"], label=\"Valid box loss\")\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c2b38ecbc84575295dd62657ed175c5a0b72021"},"cell_type":"code","source":"best_epoch = np.argmin(history[\"val_loss\"])\nscore = history[\"val_loss\"][best_epoch]\nprint(f'Best Epoch:{best_epoch+1} val_loss:{score}')","execution_count":null,"outputs":[]},{"metadata":{"id":"eraRlzgPmmIZ","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"outputId":"de9e688c-ba4f-4b62-f842-dbcf00ce397c","trusted":true,"_uuid":"db5c10d3f7da099e5751a04a6e6d49819882ecd4"},"cell_type":"code","source":"# select trained model \ndir_names = next(os.walk(model.model_dir))[1]\nkey = config.NAME.lower()\ndir_names = filter(lambda f: f.startswith(key), dir_names)\ndir_names = sorted(dir_names)\n\nif not dir_names:\n    import errno\n    raise FileNotFoundError(\n        errno.ENOENT,\n        \"Could not find model directory under {}\".format(self.model_dir))\n    \nfps = []\n# Pick last directory\nfor d in dir_names: \n    dir_name = os.path.join(model.model_dir, d)\n    # Find the last checkpoint\n    checkpoints = next(os.walk(dir_name))[2]\n    checkpoints = filter(lambda f: f.startswith(\"mask_rcnn\"), checkpoints)\n    checkpoints = sorted(checkpoints)\n    if not checkpoints:\n        print('No weight files in {}'.format(dir_name))\n    else:\n        checkpoint = os.path.join(dir_name, checkpoints[best_epoch])\n        fps.append(checkpoint)\n\nmodel_path = sorted(fps)[-1]\nprint('Found model {}'.format(model_path))","execution_count":null,"outputs":[]},{"metadata":{"id":"TgpT9AzC2Bgz","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":53},"outputId":"60f5a175-4666-497d-b4e8-0bdab39a92d0","trusted":true,"_uuid":"52138636b2ae5bf444bba808518cd8313bde65cd"},"cell_type":"code","source":"class InferenceConfig(DetectorConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\ninference_config = InferenceConfig()\n\n# Recreate the model in inference mode\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=ROOT_DIR)\n\n# Load trained weights (fill in path to trained weights here)\nassert model_path != \"\", \"Provide path to trained weights\"\nprint(\"Loading weights from \", model_path)\nmodel.load_weights(model_path, by_name=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"9mTBig7D2BjU","colab_type":"code","colab":{},"trusted":true,"_uuid":"e13c61bee23b791c61ecf1256f7512295cd4d9ab"},"cell_type":"code","source":"# set color for class\ndef get_colors_for_class_ids(class_ids):\n    colors = []\n    for class_id in class_ids:\n        if class_id == 1:\n            colors.append((.941, .204, .204))\n    return colors","execution_count":null,"outputs":[]},{"metadata":{"id":"A8EiL2LOiCr_","colab_type":"text","_uuid":"f99fbd3f31ff1a2bd66764835c9b646375364598"},"cell_type":"markdown","source":"# Check the trained model's predictions"},{"metadata":{"id":"irheTbrW2Bl0","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1394},"outputId":"56041ad4-173d-45ab-af67-f54e8333511e","trusted":true,"_uuid":"186412199e25b98719f71cfe5e8869abcce516c4"},"cell_type":"code","source":"# Show few example of ground truth vs. predictions on the validation dataset \ndataset = dataset_val\nfig = plt.figure(figsize=(10, 40))\n\nfor i in range(8):\n\n    image_id = random.choice(dataset.image_ids)\n    \n    original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n        modellib.load_image_gt(dataset_val, inference_config, \n                               image_id, use_mini_mask=False)\n    \n    print(original_image.shape)\n    plt.subplot(8, 2, 2*i + 1)\n    visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, \n                                dataset.class_names,\n                                colors=get_colors_for_class_ids(gt_class_id), ax=fig.axes[-1])\n    \n    plt.subplot(8, 2, 2*i + 2)\n    results = model.detect([original_image]) #, verbose=1)\n    r = results[0]\n    visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], \n                                dataset.class_names, r['scores'], \n                                colors=get_colors_for_class_ids(r['class_ids']), ax=fig.axes[-1])","execution_count":null,"outputs":[]},{"metadata":{"id":"WcV1cL_aiSc4","colab_type":"text","_uuid":"164e18701a830bc6c42a791feea13549de37289b"},"cell_type":"markdown","source":"# Create the submission file"},{"metadata":{"trusted":true,"_uuid":"5a124f21c2918ac4cb40ce99c852b86ea223d7e4","scrolled":true},"cell_type":"code","source":"# Get filenames of test dataset images\ntest_image_fps = test_names","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ff03a1817f862df32e2ef576dfad324f809d0c4"},"cell_type":"markdown","source":"## Load ship vs no-ship classification result first"},{"metadata":{"trusted":true,"_uuid":"a5b7afbfbcde9afe9ef3d80263ad7f55a85531f0"},"cell_type":"code","source":"DETECTION_TEST_PRED = '/kaggle/input/fine-tuning-resnet34-on-ship-detection-new-data/ship_detection.csv'\nship_detection = pd.read_csv(DETECTION_TEST_PRED)\nship_detection.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02321d63a0610d91f9dae621002dc7d4cce57034"},"cell_type":"code","source":"SHIP_THRESHOLD = 0.5\ntest_names = ship_detection.loc[ship_detection['p_ship'] > SHIP_THRESHOLD, \n                                ['id']]['id'].values.tolist()\ntest_names_nothing = ship_detection.loc[ship_detection['p_ship'] <= SHIP_THRESHOLD, \n                                        ['id']]['id'].values.tolist()\n\n\nprint(f'The classification model predicted {len(test_names)} images with ships and ' \n      f'{len(test_names_nothing)} without')","execution_count":null,"outputs":[]},{"metadata":{"id":"C6UWVrbM2Bob","colab_type":"code","colab":{},"trusted":true,"_uuid":"4a5c0c6134408ddbf5a34496d7e9d7be5692e9a1"},"cell_type":"code","source":"# TODO: Check if it is possible to make this run faster...\n# Make predictions on test images, write out sample submission\ndef predict(image_fps, filepath='submission.csv', \n            min_conf=config.DETECTION_MIN_CONFIDENCE):\n    # assume square image\n    resize_factor = IMG_SIZE[0] / config.IMAGE_SHAPE[0]\n    with open(filepath, 'w') as file:\n        file.write(\"ImageId,EncodedPixels\\n\")\n\n        for image_id in tqdm(image_fps):\n            found = False\n            \n            image = imread(os.path.join(test_dicom_dir, image_id))\n            # If grayscale. Convert to RGB for consistency.\n            if len(image.shape) != 3 or image.shape[2] != 3:\n                image = np.stack((image,) * 3, -1)\n            image, window, scale, padding, crop = utils.resize_image(\n                image,\n                min_dim=config.IMAGE_MIN_DIM,\n                min_scale=config.IMAGE_MIN_SCALE,\n                max_dim=config.IMAGE_MAX_DIM,\n                mode=config.IMAGE_RESIZE_MODE)\n\n            results = model.detect([image])\n            r = results[0]\n\n            assert( len(r['rois']) == len(r['class_ids']) == len(r['scores']) )\n            if len(r['rois']) == 0 or image_id in test_names_nothing:\n                pass  ## no ship\n            else:\n                num_instances = len(r['rois'])\n\n                for i in range(num_instances):\n                    if r['scores'][i] > min_conf:\n                        file.write(image_id + \",\" + rle_encode(r['masks'][...,i]) + \"\\n\")\n                        found = True\n\n            if not found:\n                file.write(image_id + \",\\n\")  ## no ship","execution_count":null,"outputs":[]},{"metadata":{"id":"C5cBpNka2Bsv","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"outputId":"a2af9176-d9d6-49f6-f22a-5a1c455d144f","trusted":true,"_uuid":"0406e7f5aaa4867782c4f9c064f90bba386128e7","scrolled":false},"cell_type":"code","source":"submission_fp = os.path.join(ROOT_DIR, 'submission.csv')\npredict(test_image_fps, filepath=submission_fp)\nprint(submission_fp)","execution_count":null,"outputs":[]},{"metadata":{"id":"_BjPE_Ee9rbA","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1599},"outputId":"67b5f053-112b-494a-9ab3-d017bfb440c2","trusted":true,"_uuid":"3fd8d178fc51ef0bca94fbb3f423160f08a77edc","scrolled":true},"cell_type":"code","source":"submission_df = pd.read_csv(submission_fp)\nsubmission_df.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea110f197abc2acb1c3435383f7259079dc0eb0e"},"cell_type":"code","source":"# show a few test image detection example\ndef visualize_test(): \n    image_id = random.choice(test_names)\n    \n    # original image\n    image = imread(os.path.join(test_dicom_dir, image_id))\n    \n    # assume square image \n    resize_factor = IMG_SIZE[0] / config.IMAGE_SHAPE[0]\n    \n    # If grayscale. Convert to RGB for consistency.\n    if len(image.shape) != 3 or image.shape[2] != 3:\n        image = np.stack((image,) * 3, -1) \n    resized_image, window, scale, padding, crop = utils.resize_image(\n        image,\n        min_dim=config.IMAGE_MIN_DIM,\n        min_scale=config.IMAGE_MIN_SCALE,\n        max_dim=config.IMAGE_MAX_DIM,\n        mode=config.IMAGE_RESIZE_MODE)\n\n    results = model.detect([resized_image])\n    r = results[0]\n    for bbox in r['rois']: \n        x1 = int(bbox[1] * resize_factor)\n        y1 = int(bbox[0] * resize_factor)\n        x2 = int(bbox[3] * resize_factor)\n        y2 = int(bbox[2]  * resize_factor)\n        cv2.rectangle(image, (x1,y1), (x2,y2), (77, 255, 9), 3, 1)\n        width = x2 - x1 \n        height = y2 - y1 \n    fig, ax = plt.subplots()\n    ax.set_title(f\"{len(r['rois'])}: {image_id}\")\n    plt.imshow(image)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"328b19a99bc5f5d534da3778d650435dfd0cd381"},"cell_type":"code","source":"for i in range(10):\n    visualize_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"835a15c9d018acd5deb16e9e02f9b765f68d0e78"},"cell_type":"code","source":"# remove files to allow committing (hit files limit otherwise)\n!rm -rf /kaggle/working/Mask_RCNN","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"lesson-3-rsna-pneumonia-detection-challenge-kaggle","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"accelerator":"GPU","language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}