{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Mask-RCNN Starter Model for the Airbus Ship Detection Challenge with transfer learning **\n\nUsing pre-trained COCO weights trained on http://cocodataset.org as in https://github.com/matterport/Mask_RCNN/tree/master/samples/balloon\n\nWe get some amazing performance training only within the 6hrs kaggle kernel limit.","metadata":{"id":"KBeAf8WgaeSk","_uuid":"7c1fce19a11f95416168ced03c2c70fa818b21a5"}},{"cell_type":"code","source":"debug = False\n# debug = True","metadata":{"_uuid":"cdb40bf9115f53810c9e13f0a50e53ed9eb6221b","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-02-20T07:48:33.371315Z","iopub.execute_input":"2023-02-20T07:48:33.371627Z","iopub.status.idle":"2023-02-20T07:48:33.375829Z","shell.execute_reply.started":"2023-02-20T07:48:33.371567Z","shell.execute_reply":"2023-02-20T07:48:33.374717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os \nimport sys\nimport random\nimport math\nimport numpy as np\nimport cv2\nimport matplotlib.pyplot as plt\nimport json\nfrom imgaug import augmenters as iaa\nfrom tqdm import tqdm\nimport pandas as pd \nimport glob ","metadata":{"id":"4kjcC6QqywWl","_uuid":"40c67b3ff0fa04587dec508363308adaa3ceaf34","execution":{"iopub.status.busy":"2023-02-20T07:48:33.911944Z","iopub.execute_input":"2023-02-20T07:48:33.91233Z","iopub.status.idle":"2023-02-20T07:48:34.772193Z","shell.execute_reply.started":"2023-02-20T07:48:33.912279Z","shell.execute_reply":"2023-02-20T07:48:34.77117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = '/kaggle/input/airbus-ship-detection'\n\n# Directory to save logs and trained model\nROOT_DIR = '/kaggle/working'","metadata":{"id":"yP0XLJx_x_6o","_uuid":"6e5764759e6a0a9b698b44645658f66873edd807","execution":{"iopub.status.busy":"2023-02-20T07:48:34.773857Z","iopub.execute_input":"2023-02-20T07:48:34.774168Z","iopub.status.idle":"2023-02-20T07:48:34.778542Z","shell.execute_reply.started":"2023-02-20T07:48:34.774111Z","shell.execute_reply":"2023-02-20T07:48:34.777757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Install Matterport's Mask-RCNN model from github.\nSee the [Matterport's implementation of Mask-RCNN](https://github.com/matterport/Mask_RCNN).","metadata":{"id":"kdYzLq1zfKL4","_uuid":"576df4c47a23d08b1bdb384245e09aa69f88bbd3"}},{"cell_type":"code","source":"!git clone https://www.github.com/matterport/Mask_RCNN.git\nos.chdir('Mask_RCNN')\n#!python setup.py -q install","metadata":{"id":"KgllzLnDr7kF","outputId":"6c978df7-2013-437e-acd1-5011048dfb53","_uuid":"b37d22551d332f0f7b722cc7204eb614524b6c21","execution":{"iopub.status.busy":"2023-02-20T07:48:35.300644Z","iopub.execute_input":"2023-02-20T07:48:35.30096Z","iopub.status.idle":"2023-02-20T07:48:53.276863Z","shell.execute_reply.started":"2023-02-20T07:48:35.300892Z","shell.execute_reply":"2023-02-20T07:48:53.275934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import Mask RCNN\nsys.path.append(os.path.join(ROOT_DIR, 'Mask_RCNN'))  # To find local version of the library\nfrom mrcnn.config import Config\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","metadata":{"id":"-KZXyWwhzOVU","outputId":"2576cc17-7484-4311-ad72-3c5643dcb5bb","_uuid":"3acbbbe055b6a409d3c50ae0f893acf51b5ae7ba","execution":{"iopub.status.busy":"2023-02-20T07:48:53.280936Z","iopub.execute_input":"2023-02-20T07:48:53.281201Z","iopub.status.idle":"2023-02-20T07:48:53.618397Z","shell.execute_reply.started":"2023-02-20T07:48:53.281148Z","shell.execute_reply":"2023-02-20T07:48:53.617609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dicom_dir = os.path.join(DATA_DIR, 'train_v2')\ntest_dicom_dir = os.path.join(DATA_DIR, 'test_v2')","metadata":{"id":"FghMmiMjzOX2","_uuid":"50089cc61791871cdf6a5c0037dc4f28b7b7d7cc","execution":{"iopub.status.busy":"2023-02-20T07:48:53.620542Z","iopub.execute_input":"2023-02-20T07:48:53.620831Z","iopub.status.idle":"2023-02-20T07:48:53.628018Z","shell.execute_reply.started":"2023-02-20T07:48:53.620782Z","shell.execute_reply":"2023-02-20T07:48:53.627323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Download COCO pre-trained weights","metadata":{"_uuid":"f108beef7838be8a64dd512d395c5dc0ad952790"}},{"cell_type":"code","source":"!wget --quiet https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5\n!ls -lh mask_rcnn_coco.h5\n\nCOCO_WEIGHTS_PATH = \"mask_rcnn_coco.h5\"","metadata":{"_uuid":"c3ee0cd0ee0b1defdec97b94bc736587c1f7631f","execution":{"iopub.status.busy":"2023-02-20T07:48:53.633057Z","iopub.execute_input":"2023-02-20T07:48:53.6334Z","iopub.status.idle":"2023-02-20T07:48:59.279891Z","shell.execute_reply.started":"2023-02-20T07:48:53.633318Z","shell.execute_reply":"2023-02-20T07:48:59.278818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Some setup functions and classes for Mask-RCNN\n\n- dicom_fps is a list of the dicom image path and filenames \n- image_annotions is a dictionary of the annotations keyed by the filenames\n- parsing the dataset returns a list of the image filenames and the annotations dictionary","metadata":{"id":"gj-tvDvEaDiC","_uuid":"032cc5fe4baa051108106675e6ca4f4fdb2846ed"}},{"cell_type":"code","source":"# The following parameters have been selected to reduce running time for demonstration purposes \n# These are not optimal \n\nclass DetectorConfig(Config):    \n    # Give the configuration a recognizable name  \n    NAME = 'airbus'\n    \n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 9\n    \n    BACKBONE = 'resnet50'\n    \n    NUM_CLASSES = 2  # background and ship classes\n    \n    IMAGE_MIN_DIM = 384\n    IMAGE_MAX_DIM = 384\n    RPN_ANCHOR_SCALES = (8, 16, 32, 64)\n    TRAIN_ROIS_PER_IMAGE = 64\n    MAX_GT_INSTANCES = 14\n    DETECTION_MAX_INSTANCES = 10\n    DETECTION_MIN_CONFIDENCE = 0.95\n    DETECTION_NMS_THRESHOLD = 0.0\n\n    STEPS_PER_EPOCH = 15 if debug else 150\n    VALIDATION_STEPS = 10 if debug else 125\n    \n    ## balance out losses\n    LOSS_WEIGHTS = {\n        \"rpn_class_loss\": 30.0,\n        \"rpn_bbox_loss\": 0.8,\n        \"mrcnn_class_loss\": 6.0,\n        \"mrcnn_bbox_loss\": 1.0,\n        \"mrcnn_mask_loss\": 1.2\n    }\n\nconfig = DetectorConfig()\nconfig.display()","metadata":{"id":"_SfzTa-1zOck","outputId":"91ae8935-bccb-4b8e-9a7e-aa690f95fd9b","_uuid":"dfcffc4eaa94a41497717851dee9f702d8a2a73b","execution":{"iopub.status.busy":"2023-02-20T07:48:59.281252Z","iopub.execute_input":"2023-02-20T07:48:59.281523Z","iopub.status.idle":"2023-02-20T07:48:59.298365Z","shell.execute_reply.started":"2023-02-20T07:48:59.281471Z","shell.execute_reply":"2023-02-20T07:48:59.297615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom skimage.io import imread\nimport matplotlib.pyplot as plt\nfrom matplotlib.cm import get_cmap\nfrom skimage.segmentation import mark_boundaries\nfrom skimage.util import montage\nfrom skimage.morphology import binary_opening, disk, label\nimport gc; gc.enable() # memory is tight\n\nmontage_rgb = lambda x: np.stack([montage(x[:, :, :, i]) for i in range(x.shape[3])], -1)\n\ndef multi_rle_encode(img, **kwargs):\n    '''\n    Encode connected regions as separated masks\n    '''\n    labels = label(img)\n    if img.ndim > 2:\n        return [rle_encode(np.sum(labels==k, axis=2), **kwargs) for k in np.unique(labels[labels>0])]\n    else:\n        return [rle_encode(labels==k, **kwargs) for k in np.unique(labels[labels>0])]\n\n# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\ndef rle_encode(img, min_max_threshold=1e-3, max_mean_threshold=None):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    if np.max(img) < min_max_threshold:\n        return '' ## no need to encode if it's all zeros\n    if max_mean_threshold and np.mean(img) > max_mean_threshold:\n        return '' ## ignore overfilled mask\n    pixels = img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef rle_decode(mask_rle, shape=(768, 768)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T  # Needed to align to RLE direction\n\ndef masks_as_image(in_mask_list):\n    # Take the individual ship masks and create a single mask array for all ships\n    all_masks = np.zeros((768, 768), dtype = np.uint8)\n    for mask in in_mask_list:\n        if isinstance(mask, str):\n            all_masks |= rle_decode(mask)\n    return all_masks\n\ndef masks_as_color(in_mask_list):\n    # Take the individual ship masks and create a color mask array for each ships\n    all_masks = np.zeros((768, 768), dtype = np.float)\n    scale = lambda x: (len(in_mask_list)+x+1) / (len(in_mask_list)*2) ## scale the heatmap image to shift \n    for i,mask in enumerate(in_mask_list):\n        if isinstance(mask, str):\n            all_masks[:,:] += scale(i) * rle_decode(mask)\n    return all_masks","metadata":{"_uuid":"6136132b1f1b311e297d9432772ec4a81230924f","execution":{"iopub.status.busy":"2023-02-20T07:48:59.299438Z","iopub.execute_input":"2023-02-20T07:48:59.299768Z","iopub.status.idle":"2023-02-20T07:48:59.324229Z","shell.execute_reply.started":"2023-02-20T07:48:59.299718Z","shell.execute_reply":"2023-02-20T07:48:59.322954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nfrom sklearn.model_selection import train_test_split\n\nexclude_list = ['6384c3e78.jpg','13703f040.jpg', '14715c06d.jpg',  '33e0ff2d5.jpg',\n                '4d4e09f2a.jpg', '877691df8.jpg', '8b909bb20.jpg', 'a8d99130e.jpg', \n                'ad55c3143.jpg', 'c8260c541.jpg', 'd6c7f17c7.jpg', 'dc3e7c901.jpg',\n                'e44dffe88.jpg', 'ef87bad36.jpg', 'f083256d8.jpg'] #corrupted images\n\ntrain_names = [f for f in os.listdir(train_dicom_dir) if f not in exclude_list]\ntest_names = [f for f in os.listdir(test_dicom_dir) if f not in exclude_list]\n\nprint(len(train_names), len(test_names))","metadata":{"_uuid":"d3e05fa1a38c637fa228acd62b92dd41117a6672","execution":{"iopub.status.busy":"2023-02-20T07:48:59.325195Z","iopub.execute_input":"2023-02-20T07:48:59.325432Z","iopub.status.idle":"2023-02-20T07:49:02.003487Z","shell.execute_reply.started":"2023-02-20T07:48:59.325386Z","shell.execute_reply":"2023-02-20T07:49:02.002675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training dataset\nSEGMENTATION = DATA_DIR + '/train_ship_segmentations_v2.csv'\nanns = pd.read_csv(SEGMENTATION)\nanns.head()","metadata":{"_uuid":"3050fa77026411ffdc27bed4a9b667ec0467e4ce","execution":{"iopub.status.busy":"2023-02-20T07:49:02.004504Z","iopub.execute_input":"2023-02-20T07:49:02.004783Z","iopub.status.idle":"2023-02-20T07:49:03.428765Z","shell.execute_reply.started":"2023-02-20T07:49:02.004735Z","shell.execute_reply":"2023-02-20T07:49:03.427967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_names = anns[anns.EncodedPixels.notnull()].ImageId.unique().tolist()  ## override with ships\n\ntest_size = config.VALIDATION_STEPS * config.IMAGES_PER_GPU\nimage_fps_train, image_fps_val = train_test_split(train_names, test_size=test_size, random_state=42)\n\nif debug:\n    image_fps_train = image_fps_train[:100]\n    image_fps_val = image_fps_val[:100]\n    test_names = test_names[:100]\n    \nprint(len(image_fps_train), len(image_fps_val), len(test_names))","metadata":{"_uuid":"904636402355a305f7b2ccacb8cc55d52151d2e6","execution":{"iopub.status.busy":"2023-02-20T07:49:03.42976Z","iopub.execute_input":"2023-02-20T07:49:03.430017Z","iopub.status.idle":"2023-02-20T07:49:03.476029Z","shell.execute_reply.started":"2023-02-20T07:49:03.429971Z","shell.execute_reply":"2023-02-20T07:49:03.475228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DetectorDataset(utils.Dataset):\n    \"\"\"Dataset class for training our dataset.\n    \"\"\"\n\n    def __init__(self, image_fps, image_annotations, orig_height, orig_width):\n        super().__init__(self)\n        \n        # Add classes\n        self.add_class('ship', 1, 'Ship')\n        \n        # add images \n        for i, fp in enumerate(image_fps):\n            annotations = image_annotations.query('ImageId==\"' + fp + '\"')['EncodedPixels']\n            self.add_image('ship', image_id=i, path=os.path.join(train_dicom_dir, fp), \n                           annotations=annotations, orig_height=orig_height, orig_width=orig_width)\n            \n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path']\n\n    def load_image(self, image_id):\n        info = self.image_info[image_id]\n        fp = info['path']\n        image = imread(fp)\n        # If grayscale. Convert to RGB for consistency.\n        if len(image.shape) != 3 or image.shape[2] != 3:\n            image = np.stack((image,) * 3, -1)\n        return image\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n        annotations = info['annotations']\n#         print(image_id, annotations)\n        count = len(annotations)\n        if count == 0:\n            mask = np.zeros((info['orig_height'], info['orig_width'], 1), dtype=np.uint8)\n            class_ids = np.zeros((1,), dtype=np.int32)\n        else:\n            mask = np.zeros((info['orig_height'], info['orig_width'], count), dtype=np.uint8)\n            class_ids = np.zeros((count,), dtype=np.int32)\n            for i, a in enumerate(annotations):\n                mask[:, :, i] = rle_decode(a)\n                class_ids[i] = 1\n        return mask.astype(np.bool), class_ids.astype(np.int32)","metadata":{"id":"8EBVA1M60yAj","_uuid":"52bd3ffbdde0173a363055482d675da51c2aba99","execution":{"iopub.status.busy":"2023-02-20T07:49:03.477263Z","iopub.execute_input":"2023-02-20T07:49:03.477762Z","iopub.status.idle":"2023-02-20T07:49:03.492535Z","shell.execute_reply.started":"2023-02-20T07:49:03.477706Z","shell.execute_reply":"2023-02-20T07:49:03.491659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Examine the annotation data, parse the dataset, and view dicom fields","metadata":{"id":"9RlMo04ckd98","_uuid":"1cb852e262b69d348743767d675573368ab672c9"}},{"cell_type":"code","source":"image_fps, image_annotations = train_names, anns","metadata":{"id":"Mxz-pNbt5txY","_uuid":"7aebc88f910b232e3b8759421914a007c6ffed94","execution":{"iopub.status.busy":"2023-02-20T07:49:03.500307Z","iopub.execute_input":"2023-02-20T07:49:03.500814Z","iopub.status.idle":"2023-02-20T07:49:03.505339Z","shell.execute_reply.started":"2023-02-20T07:49:03.500608Z","shell.execute_reply":"2023-02-20T07:49:03.504567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = imread(os.path.join(train_dicom_dir, image_fps[0])) # read  image from filepath \n_ = plt.imshow(ds)","metadata":{"id":"YPqjEIXWRhSf","_uuid":"6c386dcef041b972f6209dd19e247d547c3c349f","execution":{"iopub.status.busy":"2023-02-20T07:49:03.506901Z","iopub.execute_input":"2023-02-20T07:49:03.507359Z","iopub.status.idle":"2023-02-20T07:49:03.871163Z","shell.execute_reply.started":"2023-02-20T07:49:03.507304Z","shell.execute_reply":"2023-02-20T07:49:03.870012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Original image size: 768 x 768\nORIG_SIZE = ds.shape[0]\nORIG_SIZE","metadata":{"id":"gYNSd1AhRqOV","_uuid":"74277ae9af4a3b044e62b664d10d76b23848bb43","execution":{"iopub.status.busy":"2023-02-20T07:49:03.87255Z","iopub.execute_input":"2023-02-20T07:49:03.872933Z","iopub.status.idle":"2023-02-20T07:49:03.885042Z","shell.execute_reply.started":"2023-02-20T07:49:03.872861Z","shell.execute_reply":"2023-02-20T07:49:03.883989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create and prepare the training dataset using the DetectorDataset class.","metadata":{"id":"9KUvacUbgiEX","_uuid":"a5143c19dc22bc00d318a3b28cb7e13c7fbacc8a"}},{"cell_type":"code","source":"%%time\n# prepare the training dataset\ndataset_train = DetectorDataset(image_fps_train, image_annotations, ORIG_SIZE, ORIG_SIZE)\ndataset_train.prepare()","metadata":{"id":"jwMkhotP0yFf","_uuid":"86c3333d4dfb8b7d00ce1f401693d0df4e6254e1","execution":{"iopub.status.busy":"2023-02-20T07:49:03.88652Z","iopub.execute_input":"2023-02-20T07:49:03.886978Z","iopub.status.idle":"2023-02-20T07:53:34.341736Z","shell.execute_reply.started":"2023-02-20T07:49:03.886903Z","shell.execute_reply":"2023-02-20T07:53:34.340705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# prepare the validation dataset\ndataset_val = DetectorDataset(image_fps_val, image_annotations, ORIG_SIZE, ORIG_SIZE)\ndataset_val.prepare()","metadata":{"id":"K1TkWuGP0yHl","_uuid":"313347d838fa8321a714858c8073f98c50c5be26","execution":{"iopub.status.busy":"2023-02-20T07:53:34.342684Z","iopub.execute_input":"2023-02-20T07:53:34.342953Z","iopub.status.idle":"2023-02-20T07:53:41.968288Z","shell.execute_reply.started":"2023-02-20T07:53:34.342891Z","shell.execute_reply":"2023-02-20T07:53:41.967094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Display a random image with bounding boxes","metadata":{"id":"pEXEt8fygWuC","_uuid":"600a8135d4e382f62797d69e9358f5697873c8f9"}},{"cell_type":"code","source":"# Load and display random sample and their bounding boxes\n\nclass_ids = [0]\nwhile class_ids[0] == 0:  ## look for a mask\n    image_id = random.choice(dataset_val.image_ids)\n    image_fp = dataset_val.image_reference(image_id)\n    image = dataset_val.load_image(image_id)\n    mask, class_ids = dataset_val.load_mask(image_id)\n\nprint(image.shape)\n\nplt.figure(figsize=(10, 10))\nplt.subplot(1, 2, 1)\nplt.imshow(image)\nplt.axis('off')\n\nplt.subplot(1, 2, 2)\nmasked = np.zeros(image.shape[:2])\nfor i in range(mask.shape[2]):\n    masked += mask[:, :, i] ## * image[:, :, 0]\nplt.imshow(masked, cmap='gray')\nplt.axis('off')\n\nprint(image_fp)\nprint(class_ids)","metadata":{"id":"4xwsrf9G1lHR","outputId":"a13386d3-a918-41fe-8824-13625c9d7b08","_uuid":"491b78ec96d28fcdbbf8e2d7f9320a05d64c9249","execution":{"iopub.status.busy":"2023-02-20T07:53:41.971537Z","iopub.execute_input":"2023-02-20T07:53:41.973431Z","iopub.status.idle":"2023-02-20T07:53:42.403688Z","shell.execute_reply.started":"2023-02-20T07:53:41.971775Z","shell.execute_reply":"2023-02-20T07:53:42.402631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image Augmentation. Try finetuning some variables to custom values","metadata":{"id":"ustAIH78hZI_","_uuid":"342b6008873fe7a6a0870a712ee47a87f0d2828d"}},{"cell_type":"code","source":"# Image augmentation (light but constant)\naugmentation = iaa.Sequential([\n    iaa.OneOf([ ## rotate\n        iaa.Affine(rotate=0),\n        iaa.Affine(rotate=90),\n        iaa.Affine(rotate=180),\n        iaa.Affine(rotate=270),\n    ]),\n    iaa.Fliplr(0.5),\n    iaa.Flipud(0.5),\n    iaa.OneOf([ ## brightness or contrast\n        iaa.Multiply((0.9, 1.1)),\n        iaa.ContrastNormalization((0.9, 1.1)),\n    ]),\n    iaa.OneOf([ ## blur or sharpen\n        iaa.GaussianBlur(sigma=(0.0, 0.1)),\n        iaa.Sharpen(alpha=(0.0, 0.1)),\n    ]),\n])\n\n# test on the same image as above\nimggrid = augmentation.draw_grid(image, cols=5, rows=2)\nplt.figure(figsize=(30, 12))\n_ = plt.imshow(imggrid.astype(int))","metadata":{"id":"STZnQTE61lME","_uuid":"4ab9d6086ce611a46f189c047956c43b29783e6d","execution":{"iopub.status.busy":"2023-02-20T07:53:42.404951Z","iopub.execute_input":"2023-02-20T07:53:42.405414Z","iopub.status.idle":"2023-02-20T07:53:45.132442Z","shell.execute_reply.started":"2023-02-20T07:53:42.40536Z","shell.execute_reply":"2023-02-20T07:53:45.131672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Now it's time to train the model. Note that training even a basic model can take a few hours. \n\nNote: the following model is for demonstration purpose only. We have limited the training to one epoch, and have set nominal values for the Detector Configuration to reduce run-time. \n\n- dataset_train and dataset_val are derived from DetectorDataset \n- DetectorDataset loads images from image filenames and  masks from the annotation data\n- model is Mask-RCNN","metadata":{"id":"M4kt7LKuc78e","_uuid":"7e65d2cecb283f446f34cdde19b663a8a8e9590f"}},{"cell_type":"code","source":"model = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)\n\n# Exclude the last layers because they require a matching\n# number of classes\nmodel.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[\n    \"mrcnn_class_logits\", \"mrcnn_bbox_fc\",\n    \"mrcnn_bbox\", \"mrcnn_mask\"])","metadata":{"_uuid":"138d6197fc8dce9f1f8a7b5a6c27aa2069698e03","execution":{"iopub.status.busy":"2023-02-20T07:53:45.13355Z","iopub.execute_input":"2023-02-20T07:53:45.133982Z","iopub.status.idle":"2023-02-20T07:53:56.767277Z","shell.execute_reply.started":"2023-02-20T07:53:45.133917Z","shell.execute_reply":"2023-02-20T07:53:56.766293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LEARNING_RATE = 0.003\n\n# Train Mask-RCNN Model \nimport warnings \nwarnings.filterwarnings(\"ignore\")","metadata":{"id":"RVgNhHjl1lOS","outputId":"2cba9efc-eeea-472d-d155-3c3d856585bf","_uuid":"64cce2581ffdb8c2b1cb07948ada4a93f64874b0","execution":{"iopub.status.busy":"2023-02-20T07:53:56.768267Z","iopub.execute_input":"2023-02-20T07:53:56.76851Z","iopub.status.idle":"2023-02-20T07:53:56.776874Z","shell.execute_reply.started":"2023-02-20T07:53:56.768463Z","shell.execute_reply":"2023-02-20T07:53:56.775994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.train(dataset_train, dataset_val,\n            learning_rate=LEARNING_RATE/2,\n            epochs=5,\n            layers='all',\n            augmentation=augmentation)\n\nnew_history = model.keras_model.history.history\nfor k in new_history: history[k] = history[k] + new_history[k]","metadata":{"_uuid":"5c2b38ecbc84575295dd62657ed175c5a0b72021","execution":{"iopub.status.busy":"2023-02-20T08:39:07.361054Z","iopub.execute_input":"2023-02-20T08:39:07.36144Z","iopub.status.idle":"2023-02-20T09:18:06.034772Z","shell.execute_reply.started":"2023-02-20T08:39:07.361382Z","shell.execute_reply":"2023-02-20T09:18:06.029135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.keras_model.history.history","metadata":{"execution":{"iopub.status.busy":"2023-02-20T09:18:46.03024Z","iopub.execute_input":"2023-02-20T09:18:46.030581Z","iopub.status.idle":"2023-02-20T09:18:46.037561Z","shell.execute_reply.started":"2023-02-20T09:18:46.03052Z","shell.execute_reply":"2023-02-20T09:18:46.036706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = range(1, len(history['loss'])+1)\npd.DataFrame(history, index=epochs)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T09:18:48.409856Z","iopub.execute_input":"2023-02-20T09:18:48.410185Z","iopub.status.idle":"2023-02-20T09:18:48.478243Z","shell.execute_reply.started":"2023-02-20T09:18:48.410122Z","shell.execute_reply":"2023-02-20T09:18:48.477276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(21,11))\n\nplt.subplot(231)\nplt.plot(epochs, history[\"loss\"], label=\"Train loss\")\nplt.plot(epochs, history[\"val_loss\"], label=\"Valid loss\")\nplt.legend()\nplt.subplot(232)\nplt.plot(epochs, history[\"rpn_class_loss\"], label=\"Train RPN class ce\")\nplt.plot(epochs, history[\"val_rpn_class_loss\"], label=\"Valid RPN class ce\")\nplt.legend()\nplt.subplot(233)\nplt.plot(epochs, history[\"rpn_bbox_loss\"], label=\"Train RPN box loss\")\nplt.plot(epochs, history[\"val_rpn_bbox_loss\"], label=\"Valid RPN box loss\")\nplt.legend()\nplt.subplot(234)\nplt.plot(epochs, history[\"mrcnn_class_loss\"], label=\"Train MRCNN class ce\")\nplt.plot(epochs, history[\"val_mrcnn_class_loss\"], label=\"Valid MRCNN class ce\")\nplt.legend()\nplt.subplot(235)\nplt.plot(epochs, history[\"mrcnn_bbox_loss\"], label=\"Train MRCNN box loss\")\nplt.plot(epochs, history[\"val_mrcnn_bbox_loss\"], label=\"Valid MRCNN box loss\")\nplt.legend()\nplt.subplot(236)\nplt.plot(epochs, history[\"mrcnn_mask_loss\"], label=\"Train Mask loss\")\nplt.plot(epochs, history[\"val_mrcnn_mask_loss\"], label=\"Valid Mask loss\")\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-20T09:18:50.598137Z","iopub.execute_input":"2023-02-20T09:18:50.598466Z","iopub.status.idle":"2023-02-20T09:18:51.849679Z","shell.execute_reply.started":"2023-02-20T09:18:50.598408Z","shell.execute_reply":"2023-02-20T09:18:51.848957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_epoch = np.argmin(history[\"val_loss\"])\nscore = history[\"val_loss\"][best_epoch]\nprint(f'Best Epoch:{best_epoch+1} val_loss:{score}')","metadata":{"execution":{"iopub.status.busy":"2023-02-20T09:20:20.603226Z","iopub.execute_input":"2023-02-20T09:20:20.603548Z","iopub.status.idle":"2023-02-20T09:20:20.610583Z","shell.execute_reply.started":"2023-02-20T09:20:20.603487Z","shell.execute_reply":"2023-02-20T09:20:20.6097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select trained model \ndir_names = next(os.walk(model.model_dir))[1]\nkey = config.NAME.lower()\ndir_names = filter(lambda f: f.startswith(key), dir_names)\ndir_names = sorted(dir_names)\n\nif not dir_names:\n    import errno\n    raise FileNotFoundError(\n        errno.ENOENT,\n        \"Could not find model directory under {}\".format(self.model_dir))\n\nfps = []\n# Pick last directory\nfor d in dir_names: \n    dir_name = os.path.join(model.model_dir, d)\n    # Find the last checkpoint\n    checkpoints = next(os.walk(dir_name))[2]\n    checkpoints = filter(lambda f: f.startswith(\"mask_rcnn\"), checkpoints)\n    checkpoints = sorted(checkpoints)\n    if not checkpoints:\n        print('No weight files in {}'.format(dir_name))\n    else:\n        checkpoint = os.path.join(dir_name, checkpoints[best_epoch])\n        fps.append(checkpoint)\n\nmodel_path = sorted(fps)[-1]\nprint('Found model {}'.format(model_path))","metadata":{"id":"eraRlzgPmmIZ","outputId":"de9e688c-ba4f-4b62-f842-dbcf00ce397c","_uuid":"db5c10d3f7da099e5751a04a6e6d49819882ecd4","execution":{"iopub.status.busy":"2023-02-20T09:20:21.704764Z","iopub.execute_input":"2023-02-20T09:20:21.705147Z","iopub.status.idle":"2023-02-20T09:20:21.719142Z","shell.execute_reply.started":"2023-02-20T09:20:21.705074Z","shell.execute_reply":"2023-02-20T09:20:21.718066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InferenceConfig(DetectorConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\ninference_config = InferenceConfig()\n\n# Recreate the model in inference mode\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=ROOT_DIR)\n\n# Load trained weights (fill in path to trained weights here)\nassert model_path != \"\", \"Provide path to trained weights\"\nprint(\"Loading weights from \", model_path)\nmodel.load_weights(model_path, by_name=True)","metadata":{"id":"TgpT9AzC2Bgz","outputId":"60f5a175-4666-497d-b4e8-0bdab39a92d0","_uuid":"52138636b2ae5bf444bba808518cd8313bde65cd","execution":{"iopub.status.busy":"2023-02-20T09:20:23.473582Z","iopub.execute_input":"2023-02-20T09:20:23.47393Z","iopub.status.idle":"2023-02-20T09:20:34.016201Z","shell.execute_reply.started":"2023-02-20T09:20:23.473857Z","shell.execute_reply":"2023-02-20T09:20:34.014412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set color for class\ndef get_colors_for_class_ids(class_ids):\n    colors = []\n    for class_id in class_ids:\n        if class_id == 1:\n            colors.append((.941, .204, .204))\n    return colors","metadata":{"id":"9mTBig7D2BjU","_uuid":"e13c61bee23b791c61ecf1256f7512295cd4d9ab","execution":{"iopub.status.busy":"2023-02-20T09:20:34.018105Z","iopub.execute_input":"2023-02-20T09:20:34.018413Z","iopub.status.idle":"2023-02-20T09:20:34.024101Z","shell.execute_reply.started":"2023-02-20T09:20:34.01836Z","shell.execute_reply":"2023-02-20T09:20:34.023044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### How does the predicted box compared to the expected value? Let's use the validation dataset to check. ","metadata":{"id":"A8EiL2LOiCr_","_uuid":"f99fbd3f31ff1a2bd66764835c9b646375364598"}},{"cell_type":"code","source":"# Show few example of ground truth vs. predictions on the validation dataset \ndataset = dataset_val\nfig = plt.figure(figsize=(10, 40))\n\nfor i in range(8):\n\n    image_id = random.choice(dataset.image_ids)\n    \n    original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n        modellib.load_image_gt(dataset_val, inference_config, \n                               image_id, use_mini_mask=False)\n    \n#     print(original_image.shape)\n    plt.subplot(8, 2, 2*i + 1)\n    visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, \n                                dataset.class_names,\n                                colors=get_colors_for_class_ids(gt_class_id), ax=fig.axes[-1])\n    \n    plt.subplot(8, 2, 2*i + 2)\n    results = model.detect([original_image]) #, verbose=1)\n    r = results[0]\n    visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], \n                                dataset.class_names, r['scores'], \n                                colors=get_colors_for_class_ids(r['class_ids']), ax=fig.axes[-1])","metadata":{"id":"irheTbrW2Bl0","outputId":"56041ad4-173d-45ab-af67-f54e8333511e","_uuid":"186412199e25b98719f71cfe5e8869abcce516c4","execution":{"iopub.status.busy":"2023-02-20T09:20:34.026178Z","iopub.execute_input":"2023-02-20T09:20:34.026774Z","iopub.status.idle":"2023-02-20T09:20:40.832403Z","shell.execute_reply.started":"2023-02-20T09:20:34.026724Z","shell.execute_reply":"2023-02-20T09:20:40.831576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Final steps - Create the filtered submission file","metadata":{"id":"WcV1cL_aiSc4","_uuid":"164e18701a830bc6c42a791feea13549de37289b"}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}