{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Understanding cloud organization with MaskRCNN using Keras/Tensorflow\n\nThis kernel is a first attempt to use a MaskRCNN to solve the cloud organization classification problem. It also helps to evaluate how promising the results are within the allowed processing time on kernels as it is likely such model will take much longer to train properly. It is using the [Matterplot implementation of MaskRCNN](https://github.com/matterport/Mask_RCNN) and is inspired from [this kernel](https://www.kaggle.com/pednoi/training-mask-r-cnn-to-be-a-fashionista-lb-0-07) from the iMaterialist competition. If you are particularly interested in data cleaning for a MaskRCNN, I have also created [another kernel](https://www.kaggle.com/frlemarchand/keras-maskrcnn-kuzushiji-recognition) for the Kuzushiji recognition competition which may be of help.\n\n### If you find this kernel useful, please give an upvote! :)","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport time\nimport json\nimport glob\nimport random\nfrom pathlib import Path\nimport pandas as pd\n\nfrom PIL import Image\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom imgaug import augmenters as iaa\n\nimport itertools\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-05T15:02:36.266230Z","iopub.execute_input":"2022-02-05T15:02:36.266555Z","iopub.status.idle":"2022-02-05T15:02:37.898970Z","shell.execute_reply.started":"2022-02-05T15:02:36.266500Z","shell.execute_reply":"2022-02-05T15:02:37.898247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/understanding_cloud_organization/train.csv\")\ntrain_df = train_df.dropna()","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-02-05T15:02:59.762833Z","iopub.execute_input":"2022-02-05T15:02:59.763137Z","iopub.status.idle":"2022-02-05T15:03:01.556215Z","shell.execute_reply.started":"2022-02-05T15:02:59.763090Z","shell.execute_reply":"2022-02-05T15:03:01.555514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:03:05.612771Z","iopub.execute_input":"2022-02-05T15:03:05.613083Z","iopub.status.idle":"2022-02-05T15:03:05.636234Z","shell.execute_reply.started":"2022-02-05T15:03:05.613033Z","shell.execute_reply":"2022-02-05T15:03:05.635029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following cells are simply to arrange the data in such format so that it can be fed to the MaskRCNN. For each image, we can a list of masks in RLE (the \"EncodedPixels\" column) and the corresponding cloud category (\"CategoryId\")","metadata":{}},{"cell_type":"code","source":"category_list = [\"Fish\",\"Flower\",\"Gravel\",\"Sugar\"]","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:03:10.580996Z","iopub.execute_input":"2022-02-05T15:03:10.581555Z","iopub.status.idle":"2022-02-05T15:03:10.587868Z","shell.execute_reply.started":"2022-02-05T15:03:10.581317Z","shell.execute_reply":"2022-02-05T15:03:10.586495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dict = {}\ntrain_class_dict = {}\nfor idx, row in train_df.iterrows():\n    image_filename = row.Image_Label.split(\"_\")[0]\n    class_name = row.Image_Label.split(\"_\")[1]\n    class_id = category_list.index(class_name)\n    if train_dict.get(image_filename):\n        train_dict[image_filename].append(row.EncodedPixels)\n        train_class_dict[image_filename].append(class_id)\n    else:\n        train_dict[image_filename] = [row.EncodedPixels]\n        train_class_dict[image_filename] = [class_id]","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:03:18.001544Z","iopub.execute_input":"2022-02-05T15:03:18.001848Z","iopub.status.idle":"2022-02-05T15:03:19.767793Z","shell.execute_reply.started":"2022-02-05T15:03:18.001797Z","shell.execute_reply":"2022-02-05T15:03:19.767077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(columns=[\"image_id\",\"EncodedPixels\",\"CategoryId\",\"Width\",\"Height\"])\nfor key, value in train_dict.items():\n    img = Image.open(\"../input/understanding_cloud_organization/train_images/{}\".format(key))\n    width, height = img.width, img.height\n    df = df.append({\"image_id\": key, \"EncodedPixels\": value, \"CategoryId\": train_class_dict[key], \"Width\": width, \"Height\": height},ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:03:28.748985Z","iopub.execute_input":"2022-02-05T15:03:28.749332Z","iopub.status.idle":"2022-02-05T15:05:40.491712Z","shell.execute_reply.started":"2022-02-05T15:03:28.749288Z","shell.execute_reply":"2022-02-05T15:05:40.490917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:06:19.434298Z","iopub.execute_input":"2022-02-05T15:06:19.434598Z","iopub.status.idle":"2022-02-05T15:06:19.464450Z","shell.execute_reply.started":"2022-02-05T15:06:19.434549Z","shell.execute_reply":"2022-02-05T15:06:19.459587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Setting the MaskRCNN","metadata":{}},{"cell_type":"code","source":"DATA_DIR = Path('../kaggle/input/')\nROOT_DIR = \"../../working\"\n\nNUM_CATS = len(category_list)\nIMAGE_SIZE = 512","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:06:26.738447Z","iopub.execute_input":"2022-02-05T15:06:26.738792Z","iopub.status.idle":"2022-02-05T15:06:26.742816Z","shell.execute_reply.started":"2022-02-05T15:06:26.738727Z","shell.execute_reply":"2022-02-05T15:06:26.742032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://www.github.com/matterport/Mask_RCNN.git\nos.chdir('Mask_RCNN')\n\n!rm -rf .git # to prevent an error when the kernel is committed\n!rm -rf images assets # to prevent displaying images at the bottom of a kernel","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:06:31.637320Z","iopub.execute_input":"2022-02-05T15:06:31.637604Z","iopub.status.idle":"2022-02-05T15:06:41.480485Z","shell.execute_reply.started":"2022-02-05T15:06:31.637556Z","shell.execute_reply":"2022-02-05T15:06:41.479504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append(ROOT_DIR+'/Mask_RCNN')\nfrom mrcnn.config import Config\n\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:07:41.137732Z","iopub.execute_input":"2022-02-05T15:07:41.138077Z","iopub.status.idle":"2022-02-05T15:07:42.781156Z","shell.execute_reply.started":"2022-02-05T15:07:41.138025Z","shell.execute_reply":"2022-02-05T15:07:42.780322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will use the COCO weights for the MaskRCNN as a base, even though the images are from a different domain than our satellite images.","metadata":{}},{"cell_type":"code","source":"!wget --quiet https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5\n!ls -lh mask_rcnn_coco.h5\n\nCOCO_WEIGHTS_PATH = 'mask_rcnn_coco.h5'","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:07:51.529471Z","iopub.execute_input":"2022-02-05T15:07:51.529764Z","iopub.status.idle":"2022-02-05T15:08:16.021245Z","shell.execute_reply.started":"2022-02-05T15:07:51.529710Z","shell.execute_reply":"2022-02-05T15:08:16.020425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CloudConfig(Config):\n    NAME = \"cloud\"\n    NUM_CLASSES = NUM_CATS + 1 # +1 for the background class\n    \n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 4 #That is the maximum with the memory available on kernels\n    \n    BACKBONE = 'resnet50'\n    \n    IMAGE_MIN_DIM = IMAGE_SIZE\n    IMAGE_MAX_DIM = IMAGE_SIZE    \n    IMAGE_RESIZE_MODE = 'none'\n    \n    RPN_ANCHOR_SCALES = (16, 32, 64, 128, 256)\n    \n    # STEPS_PER_EPOCH should be the number of instances \n    # divided by (GPU_COUNT*IMAGES_PER_GPU), and so should VALIDATION_STEPS;\n    # however, due to the time limit, I set them so that this kernel can be run in 9 hours\n    STEPS_PER_EPOCH = 4500\n    VALIDATION_STEPS = 500\n    \nconfig = CloudConfig()\nconfig.display()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:08:31.305019Z","iopub.execute_input":"2022-02-05T15:08:31.305338Z","iopub.status.idle":"2022-02-05T15:08:31.320830Z","shell.execute_reply.started":"2022-02-05T15:08:31.305284Z","shell.execute_reply":"2022-02-05T15:08:31.320078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize_image(image_path):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)  \n    return img","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:08:39.281598Z","iopub.execute_input":"2022-02-05T15:08:39.281910Z","iopub.status.idle":"2022-02-05T15:08:39.287384Z","shell.execute_reply.started":"2022-02-05T15:08:39.281855Z","shell.execute_reply":"2022-02-05T15:08:39.286248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CloudDataset(utils.Dataset):\n\n    def __init__(self, df):\n        super().__init__(self)\n        \n        # Add classes\n        for i, name in enumerate(category_list):\n            self.add_class(\"cloud\", i+1, name)\n        \n        # Add images \n        for i, row in df.iterrows():\n            self.add_image(\"cloud\", \n                           image_id=row.name, \n                           path='../../input/understanding_cloud_organization/train_images/'+str(row.image_id), \n                           labels=row['CategoryId'],\n                           annotations=row['EncodedPixels'], \n                           height=row['Height'], width=row['Width'])\n\n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path'], [category_list[int(x)] for x in info['labels']]\n    \n    def load_image(self, image_id):\n        return resize_image(self.image_info[image_id]['path'])\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n                \n        mask = np.zeros((IMAGE_SIZE, IMAGE_SIZE, len(info['annotations'])), dtype=np.uint8)\n        labels = []\n        \n        for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):\n            sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)\n            annotation = [int(x) for x in annotation.split(' ')]\n            \n            for i, start_pixel in enumerate(annotation[::2]):\n                sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n\n            sub_mask = sub_mask.reshape((info['height'], info['width']), order='F')\n            sub_mask = cv2.resize(sub_mask, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)\n            \n            mask[:, :, m] = sub_mask\n            labels.append(int(label)+1)\n            \n        return mask, np.array(labels)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:08:45.901696Z","iopub.execute_input":"2022-02-05T15:08:45.902050Z","iopub.status.idle":"2022-02-05T15:08:45.918483Z","shell.execute_reply.started":"2022-02-05T15:08:45.901996Z","shell.execute_reply":"2022-02-05T15:08:45.917549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We define that 90% of the data will be used for training and 10% left for validation, before displaying some examples of masks.","metadata":{}},{"cell_type":"code","source":"training_percentage = 0.9\n\ntraining_set_size = int(training_percentage*len(df))\nvalidation_set_size = int((1-training_percentage)*len(df))\n\ntrain_dataset = CloudDataset(df[:training_set_size])\ntrain_dataset.prepare()\n\nvalid_dataset = CloudDataset(df[training_set_size:training_set_size+validation_set_size])\nvalid_dataset.prepare()\n\nfor i in range(5):\n    image_id = random.choice(train_dataset.image_ids)\n    print(train_dataset.image_reference(image_id))\n    \n    image = train_dataset.load_image(image_id)\n    mask, class_ids = train_dataset.load_mask(image_id)\n    visualize.display_top_masks(image, mask, class_ids, train_dataset.class_names, limit=5)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:08:58.262853Z","iopub.execute_input":"2022-02-05T15:08:58.263181Z","iopub.status.idle":"2022-02-05T15:09:02.159545Z","shell.execute_reply.started":"2022-02-05T15:08:58.263131Z","shell.execute_reply":"2022-02-05T15:09:02.158793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Start training","metadata":{}},{"cell_type":"markdown","source":"Let's hope that this 8 epochs will run within the 9 hours allowed.","metadata":{}},{"cell_type":"code","source":"LR = 1e-4\nEPOCHS = [3,9]\n\nimport warnings \nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:09:15.016868Z","iopub.execute_input":"2022-02-05T15:09:15.017160Z","iopub.status.idle":"2022-02-05T15:09:15.023832Z","shell.execute_reply.started":"2022-02-05T15:09:15.017114Z","shell.execute_reply":"2022-02-05T15:09:15.021524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Augmentation is quite important in our case as it turns out a few samples had contrast issues and differentiating between the 4 types of clouds is no easy feat. We proceed to flip the image both vertically and horizontally, before applying different techniques to crop and alter the colours.","metadata":{}},{"cell_type":"code","source":"augmentation = iaa.Sequential([\n    iaa.Fliplr(0.5),\n    iaa.Flipud(0.5)\n], random_order=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:09:22.630328Z","iopub.execute_input":"2022-02-05T15:09:22.630649Z","iopub.status.idle":"2022-02-05T15:09:22.637192Z","shell.execute_reply.started":"2022-02-05T15:09:22.630598Z","shell.execute_reply":"2022-02-05T15:09:22.636254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We initialize the model with the COCO weights even though they are quite different from the satellite imagery in the dataset provided.","metadata":{}},{"cell_type":"code","source":"model = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)\n\nmodel.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[\n    'mrcnn_class_logits', 'mrcnn_bbox_fc', 'mrcnn_bbox', 'mrcnn_mask'])","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:09:28.117171Z","iopub.execute_input":"2022-02-05T15:09:28.117463Z","iopub.status.idle":"2022-02-05T15:09:38.670345Z","shell.execute_reply.started":"2022-02-05T15:09:28.117415Z","shell.execute_reply":"2022-02-05T15:09:38.669551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will first train the heads before training the entire model.","metadata":{}},{"cell_type":"code","source":"%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR*2,\n            epochs=EPOCHS[0],\n            layers='heads',\n            augmentation=None)\n\nhistory = model.keras_model.history.history","metadata":{"execution":{"iopub.status.busy":"2022-02-05T15:09:43.996273Z","iopub.execute_input":"2022-02-05T15:09:43.996574Z","iopub.status.idle":"2022-02-05T17:47:12.488405Z","shell.execute_reply.started":"2022-02-05T15:09:43.996524Z","shell.execute_reply":"2022-02-05T17:47:12.484074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR,\n            epochs=EPOCHS[1],\n            layers='all',\n            augmentation=augmentation)\n\nnew_history = model.keras_model.history.history\nfor k in new_history: history[k] = history[k] + new_history[k]","metadata":{"execution":{"iopub.status.busy":"2022-02-05T17:47:54.168683Z","iopub.execute_input":"2022-02-05T17:47:54.171532Z","iopub.status.idle":"2022-02-05T23:14:37.805100Z","shell.execute_reply.started":"2022-02-05T17:47:54.171450Z","shell.execute_reply":"2022-02-05T23:14:37.798987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = range(EPOCHS[-1])\n\nplt.figure(figsize=(18, 6))\n\nplt.subplot(131)\nplt.plot(epochs, history['loss'], label=\"train loss\")\nplt.plot(epochs, history['val_loss'], label=\"valid loss\")\nplt.legend()\nplt.subplot(132)\nplt.plot(epochs, history['mrcnn_class_loss'], label=\"train class loss\")\nplt.plot(epochs, history['val_mrcnn_class_loss'], label=\"valid class loss\")\nplt.legend()\nplt.subplot(133)\nplt.plot(epochs, history['mrcnn_mask_loss'], label=\"train mask loss\")\nplt.plot(epochs, history['val_mrcnn_mask_loss'], label=\"valid mask loss\")\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:14:55.683471Z","iopub.execute_input":"2022-02-05T23:14:55.685069Z","iopub.status.idle":"2022-02-05T23:14:57.441551Z","shell.execute_reply.started":"2022-02-05T23:14:55.684704Z","shell.execute_reply":"2022-02-05T23:14:57.440414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_epoch = np.argmin(history[\"val_loss\"]) + 1\nprint(\"Best epoch: \", best_epoch)\nprint(\"Valid loss: \", history[\"val_loss\"][best_epoch-1])","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:15:08.158136Z","iopub.execute_input":"2022-02-05T23:15:08.158832Z","iopub.status.idle":"2022-02-05T23:15:08.169024Z","shell.execute_reply.started":"2022-02-05T23:15:08.158485Z","shell.execute_reply":"2022-02-05T23:15:08.165643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the best weights and visual assessment on small sample","metadata":{}},{"cell_type":"code","source":"class InferenceConfig(CloudConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\ninference_config = InferenceConfig()\n\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=ROOT_DIR)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:15:15.502633Z","iopub.execute_input":"2022-02-05T23:15:15.503197Z","iopub.status.idle":"2022-02-05T23:15:23.208972Z","shell.execute_reply.started":"2022-02-05T23:15:15.502977Z","shell.execute_reply":"2022-02-05T23:15:23.207975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glob_list = glob.glob(f'../../working/cloud*/mask_rcnn_cloud_{best_epoch:04d}.h5')\nmodel_path = glob_list[0] if glob_list else ''\nmodel.load_weights(model_path, by_name=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:15:33.811450Z","iopub.execute_input":"2022-02-05T23:15:33.811737Z","iopub.status.idle":"2022-02-05T23:15:39.761113Z","shell.execute_reply.started":"2022-02-05T23:15:33.811688Z","shell.execute_reply":"2022-02-05T23:15:39.760362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fix overlapping masks\ndef refine_masks(masks, rois):\n    areas = np.sum(masks.reshape(-1, masks.shape[-1]), axis=0)\n    mask_index = np.argsort(areas)\n    union_mask = np.zeros(masks.shape[:-1], dtype=bool)\n    for m in mask_index:\n        masks[:, :, m] = np.logical_and(masks[:, :, m], np.logical_not(union_mask))\n        union_mask = np.logical_or(masks[:, :, m], union_mask)\n    for m in range(masks.shape[-1]):\n        mask_pos = np.where(masks[:, :, m]==True)\n        if np.any(mask_pos):\n            y1, x1 = np.min(mask_pos, axis=1)\n            y2, x2 = np.max(mask_pos, axis=1)\n            rois[m, :] = [y1, x1, y2, x2]\n    return masks, rois","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:15:47.922996Z","iopub.execute_input":"2022-02-05T23:15:47.923319Z","iopub.status.idle":"2022-02-05T23:15:47.933338Z","shell.execute_reply.started":"2022-02-05T23:15:47.923267Z","shell.execute_reply":"2022-02-05T23:15:47.932538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = pd.read_csv(\"../../input/understanding_cloud_organization/sample_submission.csv\")\nsample_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:15:56.139569Z","iopub.execute_input":"2022-02-05T23:15:56.139908Z","iopub.status.idle":"2022-02-05T23:15:56.239740Z","shell.execute_reply.started":"2022-02-05T23:15:56.139837Z","shell.execute_reply":"2022-02-05T23:15:56.238915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.DataFrame(columns=[\"image_id\",\"EncodedPixels\",\"CategoryId\"])\nfor idx,row in sample_df.iterrows():\n    image_filename = row.Image_Label.split(\"_\")[0]\n    test_df = test_df.append({\"image_id\": image_filename},ignore_index=True)\ntest_df = test_df.drop_duplicates()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:16:02.540640Z","iopub.execute_input":"2022-02-05T23:16:02.540988Z","iopub.status.idle":"2022-02-05T23:17:07.916505Z","shell.execute_reply.started":"2022-02-05T23:16:02.540931Z","shell.execute_reply":"2022-02-05T23:17:07.915767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:17:19.996044Z","iopub.execute_input":"2022-02-05T23:17:19.996449Z","iopub.status.idle":"2022-02-05T23:17:20.007141Z","shell.execute_reply.started":"2022-02-05T23:17:19.996323Z","shell.execute_reply":"2022-02-05T23:17:20.006038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(8):\n    image_id = test_df.sample()[\"image_id\"].values[0]\n    image_path = str('../../input/understanding_cloud_organization/test_images/'+image_id)\n    print(image_path)\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    result = model.detect([resize_image(image_path)])\n    r = result[0]\n    \n    if r['masks'].size > 0:\n        masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)\n        for m in range(r['masks'].shape[-1]):\n            masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n                                        (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n        \n        y_scale = img.shape[0]/IMAGE_SIZE\n        x_scale = img.shape[1]/IMAGE_SIZE\n        rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n        \n        masks, rois = refine_masks(masks, rois)\n    else:\n        masks, rois = r['masks'], r['rois']\n        \n    visualize.display_instances(img, rois, masks, r['class_ids'], \n                                ['bg']+category_list, r['scores'],\n                                title=image_id, figsize=(12, 12))","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:17:28.166886Z","iopub.execute_input":"2022-02-05T23:17:28.167180Z","iopub.status.idle":"2022-02-05T23:17:38.316464Z","shell.execute_reply.started":"2022-02-05T23:17:28.167130Z","shell.execute_reply":"2022-02-05T23:17:38.315757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit predictions","metadata":{}},{"cell_type":"code","source":"def rle_encoding(x):\n    dots = np.where(x.T.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return ' '.join([str(x) for x in run_lengths])","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:18:07.335194Z","iopub.execute_input":"2022-02-05T23:18:07.335500Z","iopub.status.idle":"2022-02-05T23:18:07.341951Z","shell.execute_reply.started":"2022-02-05T23:18:07.335447Z","shell.execute_reply":"2022-02-05T23:18:07.340694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Due to the rectangular shape of the groundtruth, I have decided to use the ROIs and not the masks given by the MaskRCNN. The regions in images where there are no data are removed from the ROIs before generating the RLE. More experimentation will be required.","metadata":{}},{"cell_type":"code","source":"submission_df = sample_df.copy()\nsubmission_df[\"EncodedPixels\"] = \"\"\nwith tqdm(total=len(test_df)) as pbar:\n    for i,row in test_df.iterrows():\n        pbar.update(1)\n        image_id = row[\"image_id\"]\n        image_path = str('../../input/understanding_cloud_organization/test_images/'+image_id)\n        img = cv2.imread(image_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        result = model.detect([resize_image(image_path)])\n        r = result[0]\n\n        if r['masks'].size > 0:\n            masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)\n            for m in range(r['masks'].shape[-1]):\n                masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n                                            (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n\n            y_scale = img.shape[0]/IMAGE_SIZE\n            x_scale = img.shape[1]/IMAGE_SIZE\n            rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n            masks, rois, class_ids = r['masks'], r['rois'], r['class_ids']\n\n            #The following piece of code is creating rectangular masks from\n            # the ROIs instead of using the masks drawn by the MaskRCNN.\n            # It also removes any missing area from the imagery from the predicted masks.\n            # Everything is added directly to the submission dataframe.\n            rectangular_masks = []\n            mask_dict = {\"Fish\":[],\"Flower\":[],\"Gravel\":[],\"Sugar\":[]}\n            for roi, class_id in zip(rois, class_ids):\n                rectangular_mask = np.zeros((512,512))\n                rectangular_mask[roi[0]:roi[2], roi[1]:roi[3]] = 255\n                img = cv2.resize(img, dsize=(512,512), interpolation = cv2.INTER_LINEAR)\n                cropped_img = img[roi[0]:roi[2], roi[1]:roi[3]]\n                \n                kernel = np.ones((5,5),np.uint8)\n                missing_data = np.where(cropped_img[:,:,0]==0,255,0).astype('uint8')\n                contour_mask = np.zeros(missing_data.shape)\n                opening = cv2.morphologyEx(missing_data.astype('uint8'), cv2.MORPH_OPEN, kernel)\n                contours= cv2.findContours(opening,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)\n                if len(contours[0])>0:\n                    largest_contour = max(contours[0], key = cv2.contourArea)\n                    cv2.fillPoly(contour_mask, pts =[largest_contour], color=(255))\n                    kernel = np.ones((5,5),np.uint8)\n                    opening = cv2.morphologyEx(contour_mask, cv2.MORPH_OPEN, kernel)\n                    fixed_mask = np.where(opening[:,:]==255,0,255)\n                    rectangular_mask[roi[0]:roi[2], roi[1]:roi[3]] = fixed_mask.copy()\n                    \n                if mask_dict[category_list[class_id-1]]==[]:\n                    mask_dict[category_list[class_id-1]] = rectangular_mask\n                else:\n                    previous_mask = mask_dict[category_list[class_id-1]].copy()\n                    #prevents a bug where the mask is in int64\n                    previous_mask = previous_mask.astype('float64')\n                    boolean_mask = np.ma.mask_or(previous_mask, rectangular_mask)\n                    merged_mask = np.where(boolean_mask, 255, 0)\n                    mask_dict[category_list[class_id-1]] = merged_mask\n\n            \n            #Going through the masks per category and create a md mask in RLE\n            for cloud_category in mask_dict.keys():\n                if mask_dict[cloud_category]!=[]:\n                    #resizing for submission\n                    resized_mask = cv2.resize((mask_dict[cloud_category]/255).astype('uint8'), dsize=(525,350), interpolation = cv2.INTER_LINEAR)\n                    rle_str = rle_encoding(resized_mask)\n                    image_label = \"{}_{}\".format(image_id,cloud_category)\n                    submission_df.loc[submission_df['Image_Label']==image_label,'EncodedPixels'] = rle_str\n        else:\n            masks, rois = r['masks'], r['rois']","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:18:14.276215Z","iopub.execute_input":"2022-02-05T23:18:14.276528Z","iopub.status.idle":"2022-02-05T23:35:57.378979Z","shell.execute_reply.started":"2022-02-05T23:18:14.276470Z","shell.execute_reply":"2022-02-05T23:35:57.377913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.query(\"EncodedPixels!=''\").head()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:36:05.838603Z","iopub.execute_input":"2022-02-05T23:36:05.838930Z","iopub.status.idle":"2022-02-05T23:36:05.869621Z","shell.execute_reply.started":"2022-02-05T23:36:05.838873Z","shell.execute_reply":"2022-02-05T23:36:05.868844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"../../working/submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:36:34.407528Z","iopub.execute_input":"2022-02-05T23:36:34.407862Z","iopub.status.idle":"2022-02-05T23:36:34.850958Z","shell.execute_reply.started":"2022-02-05T23:36:34.407801Z","shell.execute_reply":"2022-02-05T23:36:34.849951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Thanks for reading my kernel. Feel free to post some feedback!\n### If you find this kernel helpful, please give an upvote! ","metadata":{}}]}