{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"When i try to use the [centermask2](https://github.com/youngwanLEE/centermask2/blob/master/configs/centermask/centermask_V_39_eSE_FPN_ms_3x.yaml) to train my model, an error occur \"AttributeError: 'BitMasks' object has no attribute 'polygons'\". So there is a need of converting original RLE mask to polygons to run the code.\n\nFirst we need to install detectron2 for subsequent use","metadata":{}},{"cell_type":"code","source":"! python -m pip -q install 'git+https://github.com/facebookresearch/detectron2.git'","metadata":{"execution":{"iopub.status.busy":"2021-11-01T10:30:05.681259Z","iopub.execute_input":"2021-11-01T10:30:05.681535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# RLE to Polygon\nThe basic idea is to first convert RLE into bitmask, and then obtain the corresponding polygons by looking for contours on the bitmask.","metadata":{}},{"cell_type":"code","source":"def polygonFromMask(maskedArr): # https://github.com/hazirbas/coco-json-converter/blob/master/generate_coco_json.py\n\n    contours, _ = cv2.findContours(maskedArr, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n\n    segmentation = []\n    for contour in contours:\n        # Valid polygons have >= 6 coordinates (3 points)\n        if contour.size >= 6:\n            segmentation.append(contour.flatten().tolist())\n    RLEs = mask_util.frPyObjects(segmentation, maskedArr.shape[0], maskedArr.shape[1])\n    RLE = mask_util.merge(RLEs)\n    # RLE = mask.encode(np.asfortranarray(maskedArr))\n    area = mask_util.area(RLE)\n    [x, y, w, h] = cv2.boundingRect(maskedArr)\n\n    return segmentation[0] #, [x, y, w, h], area","metadata":{"execution":{"iopub.status.busy":"2021-11-01T09:57:31.732042Z","iopub.execute_input":"2021-11-01T09:57:31.732383Z","iopub.status.idle":"2021-11-01T09:57:31.739618Z","shell.execute_reply.started":"2021-11-01T09:57:31.73234Z","shell.execute_reply":"2021-11-01T09:57:31.739107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Polygon to RLE\nUsing the opposite method, we can also convert polygons to RLE. Through this inverse process, we can observe whether there is information loss during the conversion process.","metadata":{}},{"cell_type":"code","source":"from detectron2.structures import polygons_to_bitmask\ndef polygon_to_rle(polygon: list, shape=(520, 704)):\n    '''\n    polygon: a list of [x1, y1, x2, y2,....]\n    shape: shape of bitmask\n    Return: RLE type of mask\n    '''\n    mask = polygons_to_bitmask([np.asarray(polygon) + 0.25], shape[0], shape[1]) # add 0.25 can keep the pixels before and after the conversion unchanged\n    rle = mask_util.encode(np.asfortranarray(mask))\n    return rle","metadata":{"execution":{"iopub.status.busy":"2021-11-01T09:57:31.740399Z","iopub.execute_input":"2021-11-01T09:57:31.740584Z","iopub.status.idle":"2021-11-01T09:57:32.435017Z","shell.execute_reply.started":"2021-11-01T09:57:31.740561Z","shell.execute_reply":"2021-11-01T09:57:32.434177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Experiment","metadata":{}},{"cell_type":"code","source":"import detectron2\nfrom pathlib import Path\nimport random, cv2, os\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pycocotools.mask as mask_util\n# import some common detectron2 utilities\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer, default_setup, hooks, launch\nfrom detectron2.utils.visualizer import Visualizer, ColorMode\nfrom detectron2.data import MetadataCatalog, DatasetCatalog\nfrom detectron2.data.datasets import register_coco_instances\nfrom detectron2.utils.logger import setup_logger\nfrom detectron2.evaluation.evaluator import DatasetEvaluator\nfrom detectron2.checkpoint import DetectionCheckpointer\nfrom detectron2.modeling import GeneralizedRCNNWithTTA\nsetup_logger()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T09:57:32.437897Z","iopub.execute_input":"2021-11-01T09:57:32.438474Z","iopub.status.idle":"2021-11-01T09:57:32.601925Z","shell.execute_reply.started":"2021-11-01T09:57:32.438426Z","shell.execute_reply":"2021-11-01T09:57:32.60108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"dataDir=Path('/kaggle/input/sartorius-cell-instance-segmentation/')\ntrain_json_file = \"/kaggle/input/sartorius-cell-instance-segmentation-coco/annotations_train.json\"\nval_json_file = \"/kaggle/input/sartorius-cell-instance-segmentation-coco/annotations_val.json\"\nregister_coco_instances(\"sar_train\", {}, train_json_file, dataDir)\nregister_coco_instances(\"sar_val\", {}, val_json_file, dataDir)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T09:57:32.6033Z","iopub.execute_input":"2021-11-01T09:57:32.603817Z","iopub.status.idle":"2021-11-01T09:57:32.609393Z","shell.execute_reply.started":"2021-11-01T09:57:32.603772Z","shell.execute_reply":"2021-11-01T09:57:32.608557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata = MetadataCatalog.get('sar_train')\ntrain_ds = DatasetCatalog.get('sar_train')","metadata":{"execution":{"iopub.status.busy":"2021-11-01T09:57:32.610842Z","iopub.execute_input":"2021-11-01T09:57:32.611058Z","iopub.status.idle":"2021-11-01T09:57:36.550499Z","shell.execute_reply.started":"2021-11-01T09:57:32.611031Z","shell.execute_reply":"2021-11-01T09:57:36.54965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = train_ds[35]\nsample_seg = sample[\"annotations\"][0][\"segmentation\"]\nsample_seg","metadata":{"execution":{"iopub.status.busy":"2021-11-01T09:57:36.551557Z","iopub.execute_input":"2021-11-01T09:57:36.551784Z","iopub.status.idle":"2021-11-01T09:57:36.558318Z","shell.execute_reply.started":"2021-11-01T09:57:36.551758Z","shell.execute_reply":"2021-11-01T09:57:36.557556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_mask = mask_util.decode(sample_seg)\nsample_polygon = polygonFromMask(sample_mask)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T09:57:36.559498Z","iopub.execute_input":"2021-11-01T09:57:36.560083Z","iopub.status.idle":"2021-11-01T09:57:36.589423Z","shell.execute_reply.started":"2021-11-01T09:57:36.560051Z","shell.execute_reply":"2021-11-01T09:57:36.588649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"restored_rle = polygon_to_rle(sample_polygon)\nrestored_rle","metadata":{"execution":{"iopub.status.busy":"2021-11-01T09:57:36.59294Z","iopub.execute_input":"2021-11-01T09:57:36.593382Z","iopub.status.idle":"2021-11-01T09:57:36.600147Z","shell.execute_reply.started":"2021-11-01T09:57:36.593337Z","shell.execute_reply":"2021-11-01T09:57:36.599538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The information loss due to convertion\nWe will compare the mask before and after conversion in polygon format","metadata":{}},{"cell_type":"code","source":"poly = polygonFromMask(mask_util.decode(restored_rle))\nnp.array(poly),np.array(sample_polygon)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T09:57:36.601242Z","iopub.execute_input":"2021-11-01T09:57:36.601756Z","iopub.status.idle":"2021-11-01T09:57:36.617087Z","shell.execute_reply.started":"2021-11-01T09:57:36.601718Z","shell.execute_reply":"2021-11-01T09:57:36.616324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I tested several different masks, and the results are roughly the same. The converted mask will lose a few pixels.\nNoticed that in `polygon_to_rle`:\n```python\nmask = polygons_to_bitmask([np.asarray(polygon) + 0.25], shape[0], shape[1])\n```\n*0.25* is tried out through experimentation. When I set it to 0, I find that the x coordinate will differ by 1, and when it is set to 0.5, the y coordinate will differ by 1. So I took their average to alleviate this problem.","metadata":{}},{"cell_type":"markdown","source":"## Visualization","metadata":{}},{"cell_type":"markdown","source":"You can see that the restored RLE(**b'YW[52P`08N1N3N2M5L3N000002M<E\\\\Ud5'**) is different from the one(**b'YW[53o?8N1N3N2M5L3N000000O3N;DTec5'**) in the original data. We can visualize it and observe it more intuitively","metadata":{}},{"cell_type":"code","source":"poly_mask = mask_util.decode(restored_rle)\n_, ax = plt.subplots(1, 2, figsize=(40, 16))\nax[0].imshow(poly_mask, cmap=\"gray\")\nax[1].imshow(sample_mask, cmap=\"gray\")\nax[0].set_title(\"restored\")\nax[1].set_title(\"ori\")","metadata":{"execution":{"iopub.status.busy":"2021-11-01T10:01:21.593293Z","iopub.execute_input":"2021-11-01T10:01:21.593687Z","iopub.status.idle":"2021-11-01T10:01:22.530218Z","shell.execute_reply.started":"2021-11-01T10:01:21.593657Z","shell.execute_reply":"2021-11-01T10:01:22.529432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Although there are differences in data, they seem to be similar. So information loss will not affect training.","metadata":{}},{"cell_type":"markdown","source":"## Save annotation in polygon format\nI saved the converted result as a new file and placed it in the **cellis** dataset","metadata":{}},{"cell_type":"code","source":"import json\n\ndef polygons_format_json(json_file, save_dir):\n    with open(json_file) as f:\n        imgs_anns = json.load(f)\n\n    for idx, v in enumerate(imgs_anns[\"annotations\"]):\n        rle = v[\"segmentation\"]\n        compressed_rle = mask_util.frPyObjects(rle, rle.get('size')[0], rle.get('size')[1])\n        mymask = mask_util.decode(compressed_rle)\n        polygons = polygonFromMask(mymask)\n        v[\"segmentation\"] = [polygons]\n    \n    json_str = json.dumps(imgs_anns, indent=4, separators=(',', ':'))\n    with open(save_dir, 'w') as json_file:\n        json_file.write(json_str)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T09:57:36.75263Z","iopub.status.idle":"2021-11-01T09:57:36.752941Z","shell.execute_reply.started":"2021-11-01T09:57:36.75278Z","shell.execute_reply":"2021-11-01T09:57:36.752796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir(\"/kaggle/working/annotations\")\nfor d in [\"train\", \"val\"]:\n    json_file = f\"/kaggle/input/sartorius-cell-instance-segmentation-coco/annotations_{d}.json\"\n    save_dir = f\"/kaggle/working/annotations_{d}_polygon.json\"\n    polygons_format_json(json_file, save_dir)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T09:57:36.754173Z","iopub.status.idle":"2021-11-01T09:57:36.754523Z","shell.execute_reply.started":"2021-11-01T09:57:36.754365Z","shell.execute_reply":"2021-11-01T09:57:36.754382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Display a sample file to check the convertion","metadata":{"execution":{"iopub.status.busy":"2021-11-01T10:02:40.927642Z","iopub.execute_input":"2021-11-01T10:02:40.927928Z","iopub.status.idle":"2021-11-01T10:02:40.931493Z","shell.execute_reply.started":"2021-11-01T10:02:40.927895Z","shell.execute_reply":"2021-11-01T10:02:40.930889Z"}}},{"cell_type":"code","source":"train_json_file = \"../input/cellis/annotations/annotations_train_polygon.json\"\nval_json_file = \"../input/cellis/annotations/annotations_val_polygon.json\"\nregister_coco_instances(\"sar_train_polygon\", {}, train_json_file, dataDir)\nregister_coco_instances(\"sar_val_polygon\", {}, val_json_file, dataDir)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T10:03:47.111961Z","iopub.execute_input":"2021-11-01T10:03:47.112552Z","iopub.status.idle":"2021-11-01T10:03:47.116954Z","shell.execute_reply.started":"2021-11-01T10:03:47.112513Z","shell.execute_reply":"2021-11-01T10:03:47.116123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ann_visualization(data, metadata, index):\n    data = data[index]\n    img = cv2.imread(data[\"file_name\"])\n    visualizer = Visualizer(img[:, :, ::-1], metadata=metadata)\n    out = visualizer.draw_dataset_dict(data)\n    return out","metadata":{"execution":{"iopub.status.busy":"2021-11-01T10:13:55.279829Z","iopub.execute_input":"2021-11-01T10:13:55.28014Z","iopub.status.idle":"2021-11-01T10:13:55.285703Z","shell.execute_reply.started":"2021-11-01T10:13:55.280111Z","shell.execute_reply":"2021-11-01T10:13:55.28495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"before = DatasetCatalog.get('sar_train')\nafter = DatasetCatalog.get('sar_train_polygon')\nmb, ma = MetadataCatalog.get('sar_train'), MetadataCatalog.get('sar_train_polygon')","metadata":{"execution":{"iopub.status.busy":"2021-11-01T10:14:53.759037Z","iopub.execute_input":"2021-11-01T10:14:53.759425Z","iopub.status.idle":"2021-11-01T10:14:59.888832Z","shell.execute_reply.started":"2021-11-01T10:14:53.759379Z","shell.execute_reply":"2021-11-01T10:14:59.887952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index = 32\noutb = ann_visualization(before, mb, index)\nouta = ann_visualization(after, ma, index)\n_, ax = plt.subplots(1, 2, figsize=(40, 30))\nax[0].imshow(outb.get_image()[:, :, ::-1])\nax[1].imshow(outa.get_image()[:, :, ::-1])","metadata":{"execution":{"iopub.status.busy":"2021-11-01T10:19:24.211708Z","iopub.execute_input":"2021-11-01T10:19:24.212485Z","iopub.status.idle":"2021-11-01T10:19:26.698221Z","shell.execute_reply.started":"2021-11-01T10:19:24.212443Z","shell.execute_reply":"2021-11-01T10:19:26.697286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It seems that the convertion is successful!","metadata":{}}]}