{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Augmentation is quite important in our case as it turns out a few samples had contrast issues and differentiating between the 4 types of clouds is no easy feat. We proceed to flip the image both vertically and horizontally, before applying different techniques to crop and alter the colours.","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport time\nimport json\nimport glob\nimport random\nfrom pathlib import Path\nimport pandas as pd\n\nfrom PIL import Image\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom imgaug import augmenters as iaa\n\nimport itertools\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:07:23.182710Z","iopub.execute_input":"2022-02-28T01:07:23.183078Z","iopub.status.idle":"2022-02-28T01:07:24.823952Z","shell.execute_reply.started":"2022-02-28T01:07:23.183025Z","shell.execute_reply":"2022-02-28T01:07:24.823117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/understanding_cloud_organization/train.csv\")\ntrain_df = train_df.dropna()","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:07:32.270272Z","iopub.execute_input":"2022-02-28T01:07:32.270571Z","iopub.status.idle":"2022-02-28T01:07:35.999119Z","shell.execute_reply.started":"2022-02-28T01:07:32.270521Z","shell.execute_reply":"2022-02-28T01:07:35.998358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:07:40.243897Z","iopub.execute_input":"2022-02-28T01:07:40.244195Z","iopub.status.idle":"2022-02-28T01:07:40.265861Z","shell.execute_reply.started":"2022-02-28T01:07:40.244147Z","shell.execute_reply":"2022-02-28T01:07:40.264729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"category_list = [\"Fish\",\"Flower\",\"Gravel\",\"Sugar\"]","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:07:45.929514Z","iopub.execute_input":"2022-02-28T01:07:45.929833Z","iopub.status.idle":"2022-02-28T01:07:45.934238Z","shell.execute_reply.started":"2022-02-28T01:07:45.929761Z","shell.execute_reply":"2022-02-28T01:07:45.933014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dict = {}\ntrain_class_dict = {}\nfor idx, row in train_df.iterrows():\n    image_filename = row.Image_Label.split(\"_\")[0]\n    class_name = row.Image_Label.split(\"_\")[1]\n    class_id = category_list.index(class_name)\n    if train_dict.get(image_filename):\n        train_dict[image_filename].append(row.EncodedPixels)\n        train_class_dict[image_filename].append(class_id)\n    else:\n        train_dict[image_filename] = [row.EncodedPixels]\n        train_class_dict[image_filename] = [class_id]","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:08:04.035721Z","iopub.execute_input":"2022-02-28T01:08:04.036058Z","iopub.status.idle":"2022-02-28T01:08:05.778550Z","shell.execute_reply.started":"2022-02-28T01:08:04.036003Z","shell.execute_reply":"2022-02-28T01:08:05.777892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(columns=[\"image_id\",\"EncodedPixels\",\"CategoryId\",\"Width\",\"Height\"])\nfor key, value in train_dict.items():\n    img = Image.open(\"../input/understanding_cloud_organization/train_images/{}\".format(key))\n    width, height = img.width, img.height\n    df = df.append({\"image_id\": key, \"EncodedPixels\": value, \"CategoryId\": train_class_dict[key], \"Width\": width, \"Height\": height},ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:08:09.881120Z","iopub.execute_input":"2022-02-28T01:08:09.881425Z","iopub.status.idle":"2022-02-28T01:09:58.754047Z","shell.execute_reply.started":"2022-02-28T01:08:09.881375Z","shell.execute_reply":"2022-02-28T01:09:58.753230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:10:04.143029Z","iopub.execute_input":"2022-02-28T01:10:04.143583Z","iopub.status.idle":"2022-02-28T01:10:04.172961Z","shell.execute_reply.started":"2022-02-28T01:10:04.143344Z","shell.execute_reply":"2022-02-28T01:10:04.172263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = Path('../kaggle/input/')\nROOT_DIR = \"../../working\"\n\nNUM_CATS = len(category_list)\nIMAGE_SIZE = 512","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:10:08.827832Z","iopub.execute_input":"2022-02-28T01:10:08.828124Z","iopub.status.idle":"2022-02-28T01:10:08.833317Z","shell.execute_reply.started":"2022-02-28T01:10:08.828078Z","shell.execute_reply":"2022-02-28T01:10:08.832391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://www.github.com/matterport/Mask_RCNN.git\nos.chdir('Mask_RCNN')\n\n!rm -rf .git # to prevent an error when the kernel is committed\n!rm -rf images assets # to prevent displaying images at the bottom of a kernel","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:10:14.520137Z","iopub.execute_input":"2022-02-28T01:10:14.520436Z","iopub.status.idle":"2022-02-28T01:10:21.971529Z","shell.execute_reply.started":"2022-02-28T01:10:14.520386Z","shell.execute_reply":"2022-02-28T01:10:21.970599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append(ROOT_DIR+'/Mask_RCNN')\nfrom mrcnn.config import Config\n\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:10:24.702096Z","iopub.execute_input":"2022-02-28T01:10:24.702413Z","iopub.status.idle":"2022-02-28T01:10:26.419567Z","shell.execute_reply.started":"2022-02-28T01:10:24.702355Z","shell.execute_reply":"2022-02-28T01:10:26.418685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wget --quiet https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5\n!ls -lh mask_rcnn_coco.h5\n\nCOCO_WEIGHTS_PATH = 'mask_rcnn_coco.h5'","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:10:32.198595Z","iopub.execute_input":"2022-02-28T01:10:32.198914Z","iopub.status.idle":"2022-02-28T01:10:36.424483Z","shell.execute_reply.started":"2022-02-28T01:10:32.198854Z","shell.execute_reply":"2022-02-28T01:10:36.423705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CloudConfig(Config):\n    NAME = \"cloud\"\n    NUM_CLASSES = NUM_CATS + 1 # +1 for the background class\n    \n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 4 #That is the maximum with the memory available on kernels\n    \n    BACKBONE = 'resnet50'\n    \n    IMAGE_MIN_DIM = IMAGE_SIZE\n    IMAGE_MAX_DIM = IMAGE_SIZE    \n    IMAGE_RESIZE_MODE = 'none'\n    \n    RPN_ANCHOR_SCALES = (16, 32, 64, 128, 256)\n    \n    # STEPS_PER_EPOCH should be the number of instances \n    # divided by (GPU_COUNT*IMAGES_PER_GPU), and so should VALIDATION_STEPS;\n    # however, due to the time limit, I set them so that this kernel can be run in 9 hours\n    STEPS_PER_EPOCH = 4500\n    VALIDATION_STEPS = 500\n    \nconfig = CloudConfig()\nconfig.display()","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:10:43.679206Z","iopub.execute_input":"2022-02-28T01:10:43.679532Z","iopub.status.idle":"2022-02-28T01:10:43.695057Z","shell.execute_reply.started":"2022-02-28T01:10:43.679475Z","shell.execute_reply":"2022-02-28T01:10:43.694026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize_image(image_path):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)  \n    return img","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:10:52.937703Z","iopub.execute_input":"2022-02-28T01:10:52.938043Z","iopub.status.idle":"2022-02-28T01:10:52.942638Z","shell.execute_reply.started":"2022-02-28T01:10:52.937991Z","shell.execute_reply":"2022-02-28T01:10:52.941917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CloudDataset(utils.Dataset):\n\n    def __init__(self, df):\n        super().__init__(self)\n        \n        # Add classes\n        for i, name in enumerate(category_list):\n            self.add_class(\"cloud\", i+1, name)\n        \n        # Add images \n        for i, row in df.iterrows():\n            self.add_image(\"cloud\", \n                           image_id=row.name, \n                           path='../../input/understanding_cloud_organization/train_images/'+str(row.image_id), \n                           labels=row['CategoryId'],\n                           annotations=row['EncodedPixels'], \n                           height=row['Height'], width=row['Width'])\n\n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path'], [category_list[int(x)] for x in info['labels']]\n    \n    def load_image(self, image_id):\n        return resize_image(self.image_info[image_id]['path'])\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n                \n        mask = np.zeros((IMAGE_SIZE, IMAGE_SIZE, len(info['annotations'])), dtype=np.uint8)\n        labels = []\n        \n        for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):\n            sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)\n            annotation = [int(x) for x in annotation.split(' ')]\n            \n            for i, start_pixel in enumerate(annotation[::2]):\n                sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n\n            sub_mask = sub_mask.reshape((info['height'], info['width']), order='F')\n            sub_mask = cv2.resize(sub_mask, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)\n            \n            mask[:, :, m] = sub_mask\n            labels.append(int(label)+1)\n            \n        return mask, np.array(labels)\n","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:10:56.361330Z","iopub.execute_input":"2022-02-28T01:10:56.361636Z","iopub.status.idle":"2022-02-28T01:10:56.377164Z","shell.execute_reply.started":"2022-02-28T01:10:56.361586Z","shell.execute_reply":"2022-02-28T01:10:56.376312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_percentage = 0.9\n\ntraining_set_size = int(training_percentage*len(df))\nvalidation_set_size = int((1-training_percentage)*len(df))\n\ntrain_dataset = CloudDataset(df[:training_set_size])\ntrain_dataset.prepare()\n\nvalid_dataset = CloudDataset(df[training_set_size:training_set_size+validation_set_size])\nvalid_dataset.prepare()\n\nfor i in range(5):\n    image_id = random.choice(train_dataset.image_ids)\n    print(train_dataset.image_reference(image_id))\n    \n    image = train_dataset.load_image(image_id)\n    mask, class_ids = train_dataset.load_mask(image_id)\n    visualize.display_top_masks(image, mask, class_ids, train_dataset.class_names, limit=5)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:11:01.910560Z","iopub.execute_input":"2022-02-28T01:11:01.910869Z","iopub.status.idle":"2022-02-28T01:11:05.142058Z","shell.execute_reply.started":"2022-02-28T01:11:01.910815Z","shell.execute_reply":"2022-02-28T01:11:05.141372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LR = 1e-4\nEPOCHS = [3,9]\n\nimport warnings \nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:11:25.009954Z","iopub.execute_input":"2022-02-28T01:11:25.010278Z","iopub.status.idle":"2022-02-28T01:11:25.016886Z","shell.execute_reply.started":"2022-02-28T01:11:25.010226Z","shell.execute_reply":"2022-02-28T01:11:25.015841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"augmentation = iaa.Sequential([\n    iaa.Fliplr(0.5),\n    iaa.Flipud(0.5)\n], random_order=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:11:30.530146Z","iopub.execute_input":"2022-02-28T01:11:30.530562Z","iopub.status.idle":"2022-02-28T01:11:30.536163Z","shell.execute_reply.started":"2022-02-28T01:11:30.530490Z","shell.execute_reply":"2022-02-28T01:11:30.535307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)\n\nmodel.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[\n    'mrcnn_class_logits', 'mrcnn_bbox_fc', 'mrcnn_bbox', 'mrcnn_mask'])","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:11:35.757022Z","iopub.execute_input":"2022-02-28T01:11:35.757326Z","iopub.status.idle":"2022-02-28T01:11:46.363296Z","shell.execute_reply.started":"2022-02-28T01:11:35.757276Z","shell.execute_reply":"2022-02-28T01:11:46.362513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR*2,\n            epochs=EPOCHS[0],\n            layers='heads',\n            augmentation=None)\n\nhistory = model.keras_model.history.history","metadata":{"execution":{"iopub.status.busy":"2022-02-28T01:11:49.277894Z","iopub.execute_input":"2022-02-28T01:11:49.278266Z","iopub.status.idle":"2022-02-28T03:48:55.885407Z","shell.execute_reply.started":"2022-02-28T01:11:49.278204Z","shell.execute_reply":"2022-02-28T03:48:55.882344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR,\n            epochs=EPOCHS[1],\n            layers='all',\n            augmentation=augmentation)\n\nnew_history = model.keras_model.history.history\nfor k in new_history: history[k] = history[k] + new_history[k]","metadata":{"execution":{"iopub.status.busy":"2022-02-28T04:42:13.566300Z","iopub.execute_input":"2022-02-28T04:42:13.566654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = range(EPOCHS[-1])\n\nplt.figure(figsize=(18, 6))\n\nplt.subplot(131)\nplt.plot(epochs, history['loss'], label=\"train loss\")\nplt.plot(epochs, history['val_loss'], label=\"valid loss\")\nplt.legend()\nplt.subplot(132)\nplt.plot(epochs, history['mrcnn_class_loss'], label=\"train class loss\")\nplt.plot(epochs, history['val_mrcnn_class_loss'], label=\"valid class loss\")\nplt.legend()\nplt.subplot(133)\nplt.plot(epochs, history['mrcnn_mask_loss'], label=\"train mask loss\")\nplt.plot(epochs, history['val_mrcnn_mask_loss'], label=\"valid mask loss\")\nplt.legend()\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_epoch = np.argmin(history[\"val_loss\"]) + 1\nprint(\"Best epoch: \", best_epoch)\nprint(\"Valid loss: \", history[\"val_loss\"][best_epoch-1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the best weights and visual assessment on small sample","metadata":{}},{"cell_type":"markdown","source":"# Load the best weights and visual assessment on small sample","metadata":{}},{"cell_type":"code","source":"class InferenceConfig(CloudConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\ninference_config = InferenceConfig()\n\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=ROOT_DIR)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:15:15.502633Z","iopub.execute_input":"2022-02-05T23:15:15.503197Z","iopub.status.idle":"2022-02-05T23:15:23.208972Z","shell.execute_reply.started":"2022-02-05T23:15:15.502977Z","shell.execute_reply":"2022-02-05T23:15:23.207975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InferenceConfig(CloudConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\ninference_config = InferenceConfig()\n\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=ROOT_DIR)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:15:15.502633Z","iopub.execute_input":"2022-02-05T23:15:15.503197Z","iopub.status.idle":"2022-02-05T23:15:23.208972Z","shell.execute_reply.started":"2022-02-05T23:15:15.502977Z","shell.execute_reply":"2022-02-05T23:15:23.207975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glob_list = glob.glob(f'../../working/cloud*/mask_rcnn_cloud_{best_epoch:04d}.h5')\nmodel_path = glob_list[0] if glob_list else ''\nmodel.load_weights(model_path, by_name=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:15:33.81145Z","iopub.execute_input":"2022-02-05T23:15:33.811737Z","iopub.status.idle":"2022-02-05T23:15:39.761113Z","shell.execute_reply.started":"2022-02-05T23:15:33.811688Z","shell.execute_reply":"2022-02-05T23:15:39.760362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glob_list = glob.glob(f'../../working/cloud*/mask_rcnn_cloud_{best_epoch:04d}.h5')\nmodel_path = glob_list[0] if glob_list else ''\nmodel.load_weights(model_path, by_name=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:15:33.81145Z","iopub.execute_input":"2022-02-05T23:15:33.811737Z","iopub.status.idle":"2022-02-05T23:15:39.761113Z","shell.execute_reply.started":"2022-02-05T23:15:33.811688Z","shell.execute_reply":"2022-02-05T23:15:39.760362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fix overlapping masks\ndef refine_masks(masks, rois):\n    areas = np.sum(masks.reshape(-1, masks.shape[-1]), axis=0)\n    mask_index = np.argsort(areas)\n    union_mask = np.zeros(masks.shape[:-1], dtype=bool)\n    for m in mask_index:\n        masks[:, :, m] = np.logical_and(masks[:, :, m], np.logical_not(union_mask))\n        union_mask = np.logical_or(masks[:, :, m], union_mask)\n    for m in range(masks.shape[-1]):\n        mask_pos = np.where(masks[:, :, m]==True)\n        if np.any(mask_pos):\n            y1, x1 = np.min(mask_pos, axis=1)\n            y2, x2 = np.max(mask_pos, axis=1)\n            rois[m, :] = [y1, x1, y2, x2]\n    return masks, rois","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:15:47.922996Z","iopub.execute_input":"2022-02-05T23:15:47.923319Z","iopub.status.idle":"2022-02-05T23:15:47.933338Z","shell.execute_reply.started":"2022-02-05T23:15:47.923267Z","shell.execute_reply":"2022-02-05T23:15:47.932538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fix overlapping masks\ndef refine_masks(masks, rois):\n    areas = np.sum(masks.reshape(-1, masks.shape[-1]), axis=0)\n    mask_index = np.argsort(areas)\n    union_mask = np.zeros(masks.shape[:-1], dtype=bool)\n    for m in mask_index:\n        masks[:, :, m] = np.logical_and(masks[:, :, m], np.logical_not(union_mask))\n        union_mask = np.logical_or(masks[:, :, m], union_mask)\n    for m in range(masks.shape[-1]):\n        mask_pos = np.where(masks[:, :, m]==True)\n        if np.any(mask_pos):\n            y1, x1 = np.min(mask_pos, axis=1)\n            y2, x2 = np.max(mask_pos, axis=1)\n            rois[m, :] = [y1, x1, y2, x2]\n    return masks, rois","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:15:47.922996Z","iopub.execute_input":"2022-02-05T23:15:47.923319Z","iopub.status.idle":"2022-02-05T23:15:47.933338Z","shell.execute_reply.started":"2022-02-05T23:15:47.923267Z","shell.execute_reply":"2022-02-05T23:15:47.932538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = pd.read_csv(\"../../input/understanding_cloud_organization/sample_submission.csv\")\nsample_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:15:56.139569Z","iopub.execute_input":"2022-02-05T23:15:56.139908Z","iopub.status.idle":"2022-02-05T23:15:56.23974Z","shell.execute_reply.started":"2022-02-05T23:15:56.139837Z","shell.execute_reply":"2022-02-05T23:15:56.238915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = pd.read_csv(\"../../input/understanding_cloud_organization/sample_submission.csv\")\nsample_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:15:56.139569Z","iopub.execute_input":"2022-02-05T23:15:56.139908Z","iopub.status.idle":"2022-02-05T23:15:56.23974Z","shell.execute_reply.started":"2022-02-05T23:15:56.139837Z","shell.execute_reply":"2022-02-05T23:15:56.238915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.DataFrame(columns=[\"image_id\",\"EncodedPixels\",\"CategoryId\"])\nfor idx,row in sample_df.iterrows():\n    image_filename = row.Image_Label.split(\"_\")[0]\n    test_df = test_df.append({\"image_id\": image_filename},ignore_index=True)\ntest_df = test_df.drop_duplicates()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:16:02.54064Z","iopub.execute_input":"2022-02-05T23:16:02.540988Z","iopub.status.idle":"2022-02-05T23:17:07.916505Z","shell.execute_reply.started":"2022-02-05T23:16:02.540931Z","shell.execute_reply":"2022-02-05T23:17:07.915767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.DataFrame(columns=[\"image_id\",\"EncodedPixels\",\"CategoryId\"])\nfor idx,row in sample_df.iterrows():\n    image_filename = row.Image_Label.split(\"_\")[0]\n    test_df = test_df.append({\"image_id\": image_filename},ignore_index=True)\ntest_df = test_df.drop_duplicates()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:16:02.54064Z","iopub.execute_input":"2022-02-05T23:16:02.540988Z","iopub.status.idle":"2022-02-05T23:17:07.916505Z","shell.execute_reply.started":"2022-02-05T23:16:02.540931Z","shell.execute_reply":"2022-02-05T23:17:07.915767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:17:19.996044Z","iopub.execute_input":"2022-02-05T23:17:19.996449Z","iopub.status.idle":"2022-02-05T23:17:20.007141Z","shell.execute_reply.started":"2022-02-05T23:17:19.996323Z","shell.execute_reply":"2022-02-05T23:17:20.006038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:17:19.996044Z","iopub.execute_input":"2022-02-05T23:17:19.996449Z","iopub.status.idle":"2022-02-05T23:17:20.007141Z","shell.execute_reply.started":"2022-02-05T23:17:19.996323Z","shell.execute_reply":"2022-02-05T23:17:20.006038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(8):\n    image_id = test_df.sample()[\"image_id\"].values[0]\n    image_path = str('../../input/understanding_cloud_organization/test_images/'+image_id)\n    print(image_path)\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    result = model.detect([resize_image(image_path)])\n    r = result[0]\n    \n    if r['masks'].size > 0:\n        masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)\n        for m in range(r['masks'].shape[-1]):\n            masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n                                        (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n        \n        y_scale = img.shape[0]/IMAGE_SIZE\n        x_scale = img.shape[1]/IMAGE_SIZE\n        rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n        \n        masks, rois = refine_masks(masks, rois)\n    else:\n        masks, rois = r['masks'], r['rois']\n        \n    visualize.display_instances(img, rois, masks, r['class_ids'], \n                                ['bg']+category_list, r['scores'],\n                                title=image_id, figsize=(12, 12))","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:17:28.166886Z","iopub.execute_input":"2022-02-05T23:17:28.16718Z","iopub.status.idle":"2022-02-05T23:17:38.316464Z","shell.execute_reply.started":"2022-02-05T23:17:28.16713Z","shell.execute_reply":"2022-02-05T23:17:38.315757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(8):\n    image_id = test_df.sample()[\"image_id\"].values[0]\n    image_path = str('../../input/understanding_cloud_organization/test_images/'+image_id)\n    print(image_path)\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    result = model.detect([resize_image(image_path)])\n    r = result[0]\n    \n    if r['masks'].size > 0:\n        masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)\n        for m in range(r['masks'].shape[-1]):\n            masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n                                        (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n        \n        y_scale = img.shape[0]/IMAGE_SIZE\n        x_scale = img.shape[1]/IMAGE_SIZE\n        rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n        \n        masks, rois = refine_masks(masks, rois)\n    else:\n        masks, rois = r['masks'], r['rois']\n        \n    visualize.display_instances(img, rois, masks, r['class_ids'], \n                                ['bg']+category_list, r['scores'],\n                                title=image_id, figsize=(12, 12))","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:17:28.166886Z","iopub.execute_input":"2022-02-05T23:17:28.16718Z","iopub.status.idle":"2022-02-05T23:17:38.316464Z","shell.execute_reply.started":"2022-02-05T23:17:28.16713Z","shell.execute_reply":"2022-02-05T23:17:38.315757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit predictions","metadata":{}},{"cell_type":"markdown","source":"# Submit predictions","metadata":{}},{"cell_type":"code","source":"def rle_encoding(x):\n    dots = np.where(x.T.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return ' '.join([str(x) for x in run_lengths])","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:18:07.335194Z","iopub.execute_input":"2022-02-05T23:18:07.3355Z","iopub.status.idle":"2022-02-05T23:18:07.341951Z","shell.execute_reply.started":"2022-02-05T23:18:07.335447Z","shell.execute_reply":"2022-02-05T23:18:07.340694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_encoding(x):\n    dots = np.where(x.T.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return ' '.join([str(x) for x in run_lengths])","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:18:07.335194Z","iopub.execute_input":"2022-02-05T23:18:07.3355Z","iopub.status.idle":"2022-02-05T23:18:07.341951Z","shell.execute_reply.started":"2022-02-05T23:18:07.335447Z","shell.execute_reply":"2022-02-05T23:18:07.340694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Due to the rectangular shape of the groundtruth, I have decided to use the ROIs and not the masks given by the MaskRCNN. The regions in images where there are no data are removed from the ROIs before generating the RLE. More experimentation will be required.","metadata":{}},{"cell_type":"markdown","source":"Due to the rectangular shape of the groundtruth, I have decided to use the ROIs and not the masks given by the MaskRCNN. The regions in images where there are no data are removed from the ROIs before generating the RLE. More experimentation will be required.","metadata":{}},{"cell_type":"code","source":"submission_df = sample_df.copy()\nsubmission_df[\"EncodedPixels\"] = \"\"\nwith tqdm(total=len(test_df)) as pbar:\n    for i,row in test_df.iterrows():\n        pbar.update(1)\n        image_id = row[\"image_id\"]\n        image_path = str('../../input/understanding_cloud_organization/test_images/'+image_id)\n        img = cv2.imread(image_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        result = model.detect([resize_image(image_path)])\n        r = result[0]\n\n        if r['masks'].size > 0:\n            masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)\n            for m in range(r['masks'].shape[-1]):\n                masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n                                            (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n\n            y_scale = img.shape[0]/IMAGE_SIZE\n            x_scale = img.shape[1]/IMAGE_SIZE\n            rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n            masks, rois, class_ids = r['masks'], r['rois'], r['class_ids']\n\n            #The following piece of code is creating rectangular masks from\n            # the ROIs instead of using the masks drawn by the MaskRCNN.\n            # It also removes any missing area from the imagery from the predicted masks.\n            # Everything is added directly to the submission dataframe.\n            rectangular_masks = []\n            mask_dict = {\"Fish\":[],\"Flower\":[],\"Gravel\":[],\"Sugar\":[]}\n            for roi, class_id in zip(rois, class_ids):\n                rectangular_mask = np.zeros((512,512))\n                rectangular_mask[roi[0]:roi[2], roi[1]:roi[3]] = 255\n                img = cv2.resize(img, dsize=(512,512), interpolation = cv2.INTER_LINEAR)\n                cropped_img = img[roi[0]:roi[2], roi[1]:roi[3]]\n                \n                kernel = np.ones((5,5),np.uint8)\n                missing_data = np.where(cropped_img[:,:,0]==0,255,0).astype('uint8')\n                contour_mask = np.zeros(missing_data.shape)\n                opening = cv2.morphologyEx(missing_data.astype('uint8'), cv2.MORPH_OPEN, kernel)\n                contours= cv2.findContours(opening,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)\n                if len(contours[0])>0:\n                    largest_contour = max(contours[0], key = cv2.contourArea)\n                    cv2.fillPoly(contour_mask, pts =[largest_contour], color=(255))\n                    kernel = np.ones((5,5),np.uint8)\n                    opening = cv2.morphologyEx(contour_mask, cv2.MORPH_OPEN, kernel)\n                    fixed_mask = np.where(opening[:,:]==255,0,255)\n                    rectangular_mask[roi[0]:roi[2], roi[1]:roi[3]] = fixed_mask.copy()\n                    \n                if mask_dict[category_list[class_id-1]]==[]:\n                    mask_dict[category_list[class_id-1]] = rectangular_mask\n                else:\n                    previous_mask = mask_dict[category_list[class_id-1]].copy()\n                    #prevents a bug where the mask is in int64\n                    previous_mask = previous_mask.astype('float64')\n                    boolean_mask = np.ma.mask_or(previous_mask, rectangular_mask)\n                    merged_mask = np.where(boolean_mask, 255, 0)\n                    mask_dict[category_list[class_id-1]] = merged_mask\n\n            \n            #Going through the masks per category and create a md mask in RLE\n            for cloud_category in mask_dict.keys():\n                if mask_dict[cloud_category]!=[]:\n                    #resizing for submission\n                    resized_mask = cv2.resize((mask_dict[cloud_category]/255).astype('uint8'), dsize=(525,350), interpolation = cv2.INTER_LINEAR)\n                    rle_str = rle_encoding(resized_mask)\n                    image_label = \"{}_{}\".format(image_id,cloud_category)\n                    submission_df.loc[submission_df['Image_Label']==image_label,'EncodedPixels'] = rle_str\n        else:\n            masks, rois = r['masks'], r['rois']","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:18:14.276215Z","iopub.execute_input":"2022-02-05T23:18:14.276528Z","iopub.status.idle":"2022-02-05T23:35:57.378979Z","shell.execute_reply.started":"2022-02-05T23:18:14.27647Z","shell.execute_reply":"2022-02-05T23:35:57.377913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = sample_df.copy()\nsubmission_df[\"EncodedPixels\"] = \"\"\nwith tqdm(total=len(test_df)) as pbar:\n    for i,row in test_df.iterrows():\n        pbar.update(1)\n        image_id = row[\"image_id\"]\n        image_path = str('../../input/understanding_cloud_organization/test_images/'+image_id)\n        img = cv2.imread(image_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        result = model.detect([resize_image(image_path)])\n        r = result[0]\n\n        if r['masks'].size > 0:\n            masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)\n            for m in range(r['masks'].shape[-1]):\n                masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n                                            (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n\n            y_scale = img.shape[0]/IMAGE_SIZE\n            x_scale = img.shape[1]/IMAGE_SIZE\n            rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n            masks, rois, class_ids = r['masks'], r['rois'], r['class_ids']\n\n            #The following piece of code is creating rectangular masks from\n            # the ROIs instead of using the masks drawn by the MaskRCNN.\n            # It also removes any missing area from the imagery from the predicted masks.\n            # Everything is added directly to the submission dataframe.\n            rectangular_masks = []\n            mask_dict = {\"Fish\":[],\"Flower\":[],\"Gravel\":[],\"Sugar\":[]}\n            for roi, class_id in zip(rois, class_ids):\n                rectangular_mask = np.zeros((512,512))\n                rectangular_mask[roi[0]:roi[2], roi[1]:roi[3]] = 255\n                img = cv2.resize(img, dsize=(512,512), interpolation = cv2.INTER_LINEAR)\n                cropped_img = img[roi[0]:roi[2], roi[1]:roi[3]]\n                \n                kernel = np.ones((5,5),np.uint8)\n                missing_data = np.where(cropped_img[:,:,0]==0,255,0).astype('uint8')\n                contour_mask = np.zeros(missing_data.shape)\n                opening = cv2.morphologyEx(missing_data.astype('uint8'), cv2.MORPH_OPEN, kernel)\n                contours= cv2.findContours(opening,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)\n                if len(contours[0])>0:\n                    largest_contour = max(contours[0], key = cv2.contourArea)\n                    cv2.fillPoly(contour_mask, pts =[largest_contour], color=(255))\n                    kernel = np.ones((5,5),np.uint8)\n                    opening = cv2.morphologyEx(contour_mask, cv2.MORPH_OPEN, kernel)\n                    fixed_mask = np.where(opening[:,:]==255,0,255)\n                    rectangular_mask[roi[0]:roi[2], roi[1]:roi[3]] = fixed_mask.copy()\n                    \n                if mask_dict[category_list[class_id-1]]==[]:\n                    mask_dict[category_list[class_id-1]] = rectangular_mask\n                else:\n                    previous_mask = mask_dict[category_list[class_id-1]].copy()\n                    #prevents a bug where the mask is in int64\n                    previous_mask = previous_mask.astype('float64')\n                    boolean_mask = np.ma.mask_or(previous_mask, rectangular_mask)\n                    merged_mask = np.where(boolean_mask, 255, 0)\n                    mask_dict[category_list[class_id-1]] = merged_mask\n\n            \n            #Going through the masks per category and create a md mask in RLE\n            for cloud_category in mask_dict.keys():\n                if mask_dict[cloud_category]!=[]:\n                    #resizing for submission\n                    resized_mask = cv2.resize((mask_dict[cloud_category]/255).astype('uint8'), dsize=(525,350), interpolation = cv2.INTER_LINEAR)\n                    rle_str = rle_encoding(resized_mask)\n                    image_label = \"{}_{}\".format(image_id,cloud_category)\n                    submission_df.loc[submission_df['Image_Label']==image_label,'EncodedPixels'] = rle_str\n        else:\n            masks, rois = r['masks'], r['rois']","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:18:14.276215Z","iopub.execute_input":"2022-02-05T23:18:14.276528Z","iopub.status.idle":"2022-02-05T23:35:57.378979Z","shell.execute_reply.started":"2022-02-05T23:18:14.27647Z","shell.execute_reply":"2022-02-05T23:35:57.377913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.query(\"EncodedPixels!=''\").head()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:36:05.838603Z","iopub.execute_input":"2022-02-05T23:36:05.83893Z","iopub.status.idle":"2022-02-05T23:36:05.869621Z","shell.execute_reply.started":"2022-02-05T23:36:05.838873Z","shell.execute_reply":"2022-02-05T23:36:05.868844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.query(\"EncodedPixels!=''\").head()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:36:05.838603Z","iopub.execute_input":"2022-02-05T23:36:05.83893Z","iopub.status.idle":"2022-02-05T23:36:05.869621Z","shell.execute_reply.started":"2022-02-05T23:36:05.838873Z","shell.execute_reply":"2022-02-05T23:36:05.868844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"../../working/submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:36:34.407528Z","iopub.execute_input":"2022-02-05T23:36:34.407862Z","iopub.status.idle":"2022-02-05T23:36:34.850958Z","shell.execute_reply.started":"2022-02-05T23:36:34.407801Z","shell.execute_reply":"2022-02-05T23:36:34.849951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"../../working/submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T23:36:34.407528Z","iopub.execute_input":"2022-02-05T23:36:34.407862Z","iopub.status.idle":"2022-02-05T23:36:34.850958Z","shell.execute_reply.started":"2022-02-05T23:36:34.407801Z","shell.execute_reply":"2022-02-05T23:36:34.849951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Thanks for reading my kernel. Feel free to post some feedback!\n### If you find this kernel helpful, please give an upvote! ","metadata":{}},{"cell_type":"markdown","source":"### Thanks for reading my kernel. Feel free to post some feedback!\n### If you find this kernel helpful, please give an upvote! ","metadata":{}}]}