{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Please don't forget to upvote! 😃**","metadata":{}},{"cell_type":"markdown","source":"#### **You will need to create Polygon masks in order to use this code and train the model witout error!**\n#### **For this you can use this notebook to create COCO split data:**\n- https://www.kaggle.com/mistag/sartorius-create-coco-annotations/notebook","metadata":{}},{"cell_type":"code","source":"!python -m pip install 'git+https://github.com/facebookresearch/detectron2.git' ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-21T14:29:22.041611Z","iopub.execute_input":"2021-12-21T14:29:22.042202Z","iopub.status.idle":"2021-12-21T14:29:38.833776Z","shell.execute_reply.started":"2021-12-21T14:29:22.042165Z","shell.execute_reply":"2021-12-21T14:29:38.832789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone \"https://github.com/MarkPotanin/copy_paste_aug_detectron2\"\n!cp -r ./copy_paste_aug_detectron2/* ./","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:29:38.835809Z","iopub.execute_input":"2021-12-21T14:29:38.836128Z","iopub.status.idle":"2021-12-21T14:29:40.390163Z","shell.execute_reply.started":"2021-12-21T14:29:38.836090Z","shell.execute_reply":"2021-12-21T14:29:40.389047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Imports**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport pandas as pd \nfrom tqdm import tqdm\nfrom tqdm import tqdm_notebook as tqdm\nfrom datetime import datetime\nimport time\nimport matplotlib.pyplot as plt\nfrom pycocotools.coco import COCO\nimport os, json, cv2, random\nimport skimage.io as io\nimport copy\nfrom pathlib import Path\nfrom typing import Optional\n\n\nfrom tqdm import tqdm\nimport itertools\n\nimport torch\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\n\nfrom glob import glob\nimport numba\nfrom numba import jit\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n\nfrom detectron2.structures import BoxMode\nfrom detectron2 import model_zoo\nfrom detectron2.config import get_cfg\nfrom detectron2.data import DatasetCatalog, MetadataCatalog\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer, launch\nfrom detectron2.evaluation import COCOEvaluator\nfrom detectron2.structures import BoxMode\nfrom detectron2.utils.visualizer import ColorMode\nfrom detectron2.utils.logger import setup_logger\nfrom detectron2.utils.visualizer import Visualizer\n\nfrom detectron2.data import DatasetCatalog, MetadataCatalog, build_detection_test_loader, build_detection_train_loader\nfrom detectron2.data import detection_utils as utils\n\n\nfrom detectron2.data import DatasetCatalog, MetadataCatalog, build_detection_test_loader, build_detection_train_loader\nfrom detectron2.data import detection_utils as utils\nimport detectron2.data.transforms as T\nfrom detectron2.evaluation import COCOEvaluator, inference_on_dataset\n\nsetup_logger()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:29:40.392161Z","iopub.execute_input":"2021-12-21T14:29:40.392498Z","iopub.status.idle":"2021-12-21T14:29:40.408095Z","shell.execute_reply.started":"2021-12-21T14:29:40.392464Z","shell.execute_reply":"2021-12-21T14:29:40.407361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport numpy as np\nfrom copy_paste import CopyPaste\nfrom coco import CocoDetectionCP\nfrom visualize import display_instances\nimport albumentations as A\nimport random\nfrom matplotlib import pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:29:40.410846Z","iopub.execute_input":"2021-12-21T14:29:40.411387Z","iopub.status.idle":"2021-12-21T14:29:40.417024Z","shell.execute_reply.started":"2021-12-21T14:29:40.411340Z","shell.execute_reply":"2021-12-21T14:29:40.416052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Data_Resister_training=\"sartorius_Cell_train\";\nData_Resister_valid=\"sartorius_Cell_valid\";\nfrom detectron2.data.datasets import register_coco_instances\ndataDir=Path('../input/sartorius-cell-instance-segmentation/')\n\nDatasetCatalog.clear()\nMetadataCatalog.clear()\n\nregister_coco_instances(Data_Resister_training,{}, '../input/5foldcleaned/coco_cell_train_fold5_polygon.json', dataDir)\nregister_coco_instances(Data_Resister_valid,{}, '../input/5foldcleaned/coco_cell_valid_fold5_polygon.json', dataDir) \n\n\nmetadata = MetadataCatalog.get(Data_Resister_training)\ndataset_train = DatasetCatalog.get(Data_Resister_training)\ndataset_valid = DatasetCatalog.get(Data_Resister_valid)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:29:40.418554Z","iopub.execute_input":"2021-12-21T14:29:40.419351Z","iopub.status.idle":"2021-12-21T14:29:43.751338Z","shell.execute_reply.started":"2021-12-21T14:29:40.419314Z","shell.execute_reply":"2021-12-21T14:29:43.750413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Original Data**","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(18,11))\nd=dataset_valid[5] \nimg = cv2.imread(d[\"file_name\"])\n\nv = Visualizer(img[:, :, ::-1],\n                metadata=metadata, \n                scale=1,\n                instance_mode=ColorMode.IMAGE_BW\n    )\nout = v.draw_dataset_dict(d)\nax.grid(False)\nax.axis('off')\nax.imshow(out.get_image()[:, :, ::-1])","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:29:43.752745Z","iopub.execute_input":"2021-12-21T14:29:43.753002Z","iopub.status.idle":"2021-12-21T14:29:44.463420Z","shell.execute_reply.started":"2021-12-21T14:29:43.752966Z","shell.execute_reply":"2021-12-21T14:29:44.462456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nfrom torchvision.datasets import CocoDetection\nfrom copy_paste import copy_paste_class\nimport torch.utils.data as data\nfrom PIL import Image\nimport os\nimport os.path\n\nmin_keypoints_per_image = 10\n\ndef _count_visible_keypoints(anno):\n    return sum(sum(1 for v in ann[\"keypoints\"][2::3] if v > 0) for ann in anno)\n\ndef _has_only_empty_bbox(anno):\n    return all(any(o <= 1 for o in obj[\"bbox\"][2:]) for obj in anno)\n\ndef has_valid_annotation(anno):\n    if len(anno) == 0:\n        return False\n    if _has_only_empty_bbox(anno):\n        return False\n    if \"keypoints\" not in anno[0]:\n        return True\n    if _count_visible_keypoints(anno) >= min_keypoints_per_image:\n        return True\n\n    return False\n\n@copy_paste_class\nclass CocoDetectionCP(CocoDetection):\n    def __init__(\n        self,\n        root,\n        annFile,\n        transforms\n    ):\n        super(CocoDetectionCP, self).__init__(\n            root, annFile, None, None, transforms\n        )\n\n        ids = []\n        for img_id in self.ids:\n            ann_ids = self.coco.getAnnIds(imgIds=[img_id], iscrowd=None)\n            anno = self.coco.loadAnns(ann_ids)\n            if has_valid_annotation(anno):\n                ids.append(img_id)\n        self.ids = ids\n\n    def load_example(self, index):\n        img_id = self.ids[index]\n        ann_ids = self.coco.getAnnIds(imgIds=[img_id])\n        target = self.coco.loadAnns(ann_ids)\n\n        path = self.coco.loadImgs([img_id])[0]['file_name']\n        image = cv2.imread(os.path.join(self.root, path))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        masks = []\n        bboxes = []\n        for ix, obj in enumerate(target):\n            masks.append(self.coco.annToMask(obj))\n            bboxes.append(obj['bbox'] + [obj['category_id']] + [ix])\n\n        output = {\n            'image': image,\n            'masks': masks,\n            'bboxes': bboxes\n        }\n        \n        return self.transforms(**output)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:29:44.465097Z","iopub.execute_input":"2021-12-21T14:29:44.466150Z","iopub.status.idle":"2021-12-21T14:29:44.485540Z","shell.execute_reply.started":"2021-12-21T14:29:44.466103Z","shell.execute_reply":"2021-12-21T14:29:44.484813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Create Dataloader with Augemented Images**","metadata":{}},{"cell_type":"code","source":"transform = A.Compose([\n        CopyPaste(blend=True, sigma=1, pct_objects_paste=0.65, p=1.0)\n    ], bbox_params=A.BboxParams(format=\"coco\", min_visibility=0.05)\n)\n\ndata = CocoDetectionCP(\n    \"../input/sartorius-cell-instance-segmentation\", \n    '../input/5foldcleaned/coco_cell_train_fold5_polygon.json', \n    transform\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:29:44.486881Z","iopub.execute_input":"2021-12-21T14:29:44.487223Z","iopub.status.idle":"2021-12-21T14:29:45.761777Z","shell.execute_reply.started":"2021-12-21T14:29:44.487186Z","shell.execute_reply":"2021-12-21T14:29:45.761057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:29:45.762934Z","iopub.execute_input":"2021-12-21T14:29:45.763213Z","iopub.status.idle":"2021-12-21T14:29:45.769051Z","shell.execute_reply.started":"2021-12-21T14:29:45.763174Z","shell.execute_reply":"2021-12-21T14:29:45.768323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Visualize Example**\n\n### **Same Image - More Data :)**","metadata":{}},{"cell_type":"code","source":"for i in range(5):\n    img_data = data[1]\n\n    f, ax = plt.subplots(1, 2, figsize=(16, 16))\n    image = img_data['image']\n    masks = img_data['masks']\n    bboxes = img_data['bboxes']\n\n    empty = np.array([])\n    display_instances(image, empty, empty, empty, empty, show_mask=False, show_bbox=False, ax=ax[0])\n\n    if len(bboxes) > 0:\n        boxes = np.stack([b[:4] for b in bboxes], axis=0)\n        box_classes = np.array([b[-2] for b in bboxes])\n        mask_indices = np.array([b[-1] for b in bboxes])\n        show_masks = np.stack(masks, axis=-1)[..., mask_indices]\n        class_names = {k: data.coco.cats[k]['name'] for k in data.coco.cats.keys()}\n        display_instances(image, boxes, show_masks, box_classes, class_names, show_bbox=True, ax=ax[1])\n    else:\n        display_instances(image, empty, empty, empty, empty, show_mask=False, show_bbox=False, ax=ax[1])","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:29:45.771707Z","iopub.execute_input":"2021-12-21T14:29:45.772152Z","iopub.status.idle":"2021-12-21T14:30:08.199787Z","shell.execute_reply.started":"2021-12-21T14:29:45.772116Z","shell.execute_reply":"2021-12-21T14:30:08.198916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Define Custom Mapper**","metadata":{}},{"cell_type":"code","source":"DATA = {i:q for q,i in enumerate(data.ids)}\nIDS = list(DATA.keys())\ndataset_dicts_train = [i for i in dataset_train if i['image_id'] in IDS]","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:30:08.201086Z","iopub.execute_input":"2021-12-21T14:30:08.201897Z","iopub.status.idle":"2021-12-21T14:30:08.280893Z","shell.execute_reply.started":"2021-12-21T14:30:08.201851Z","shell.execute_reply":"2021-12-21T14:30:08.280205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pycocotools import mask\nfrom skimage import measure\n\nclass MyMapper:\n    def __init__(self, cfg, is_train: bool = True):\n        self.is_train = is_train\n\n    def __call__(self, dataset_dict):\n        dataset_dict = copy.deepcopy(dataset_dict) \n        img_id = dataset_dict['image_id']\n        \n        aug_sample = data[DATA[img_id]]\n        image = aug_sample['image']\n        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n        dataset_dict[\"image\"] = torch.as_tensor(image.transpose(2, 0, 1).astype(\"float32\"))\n        \n        bboxes = aug_sample['bboxes']\n        box_classes = np.array([b[-2] for b in bboxes])\n        boxes = np.stack([b[:4] for b in bboxes], axis=0)\n        mask_indices = np.array([b[-1] for b in bboxes])\n        masks = aug_sample['masks']\n        annos = []\n        \n        for enum, index in enumerate(mask_indices):\n            curr_mask = masks[index]\n            _gt_binary_mask = np.asfortranarray(curr_mask)\n            encoded_ground_truth = mask.encode(_gt_binary_mask)\n            _area = mask.area(encoded_ground_truth)\n            _gt_bounding_box = mask.toBbox(encoded_ground_truth)\n            contours = measure.find_contours(curr_mask, 0.5)\n            \n            annotation = {\n                \"segmentation\": [],\n                \"iscrowd\": 0,\n                \"bbox\": _gt_bounding_box.tolist(), \n                \"category_id\": metadata.thing_dataset_id_to_contiguous_id[box_classes[enum]],\n                \"bbox_mode\": dataset_train[0]['annotations'][0]['bbox_mode'],\n            }\n            \n            for contour in contours:\n                contour = np.flip(contour, axis=1)\n                segmentation = contour.ravel().tolist()\n                annotation[\"segmentation\"].append(segmentation)\n            annos.append(annotation)\n        \n        image_shape = image.shape[:2]\n        \n        instances = utils.annotations_to_instances(annos, image_shape)\n        dataset_dict[\"instances\"] = utils.filter_empty_instances(instances)\n        \n        return dataset_dict","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:30:08.282202Z","iopub.execute_input":"2021-12-21T14:30:08.282584Z","iopub.status.idle":"2021-12-21T14:30:08.296619Z","shell.execute_reply.started":"2021-12-21T14:30:08.282541Z","shell.execute_reply":"2021-12-21T14:30:08.295908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Taken from https://www.kaggle.com/theoviel/competition-metric-map-iou\nfrom detectron2.evaluation.evaluator import DatasetEvaluator\nimport pycocotools.mask as mask_util\nfrom detectron2.data import DatasetMapper\nfrom detectron2.modeling import build_model, GeneralizedRCNNWithTTA\n\ndef precision_at(threshold, iou):\n    matches = iou > threshold\n    true_positives = np.sum(matches, axis=1) == 1  # Correct objects\n    false_positives = np.sum(matches, axis=0) == 0  # Missed objects\n    false_negatives = np.sum(matches, axis=1) == 0  # Extra objects\n    return np.sum(true_positives), np.sum(false_positives), np.sum(false_negatives)\n\ndef score(pred, targ):\n    pred_masks = pred['instances'].pred_masks.cpu().numpy()\n    enc_preds = [mask_util.encode(np.asarray(p, order='F')) for p in pred_masks]\n    enc_targs = list(map(lambda x:x['segmentation'], targ))\n    ious = mask_util.iou(enc_preds, enc_targs, [0]*len(enc_targs))\n    prec = []\n    for t in np.arange(0.5, 1.0, 0.05):\n        tp, fp, fn = precision_at(t, ious)\n        p = tp / (tp + fp + fn)\n        prec.append(p)\n        \n    return np.mean(prec)\n\n\nclass MAPIOUEvaluator(DatasetEvaluator):\n    def __init__(self, dataset_name):\n        dataset_dicts = DatasetCatalog.get(dataset_name)\n        self.annotations_cache = {item['image_id']:item['annotations'] for item in dataset_dicts}\n            \n    def reset(self):\n        self.scores = []\n\n    def process(self, inputs, outputs):\n        for inp, out in zip(inputs, outputs):\n            if len(out['instances']) == 0:\n                self.scores.append(0)    \n            else:\n                targ = self.annotations_cache[inp['image_id']]\n                self.scores.append(score(out, targ))\n\n    def evaluate(self):\n        return {\"MaP IoU\": np.mean(self.scores)}","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:30:08.298111Z","iopub.execute_input":"2021-12-21T14:30:08.298920Z","iopub.status.idle":"2021-12-21T14:30:08.314778Z","shell.execute_reply.started":"2021-12-21T14:30:08.298883Z","shell.execute_reply":"2021-12-21T14:30:08.314058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyTrainer(DefaultTrainer):\n    @classmethod\n    def build_train_loader(cls, cfg):\n        return build_detection_train_loader(cfg, mapper=MyMapper(cfg),)\n    \n#     @classmethod\n#     def build_evaluator(cls, cfg, dataset_name, output_folder=None):\n#         return COCOEvaluator(dataset_name, cfg, False, output_folder)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:30:08.316211Z","iopub.execute_input":"2021-12-21T14:30:08.316545Z","iopub.status.idle":"2021-12-21T14:30:08.326099Z","shell.execute_reply.started":"2021-12-21T14:30:08.316509Z","shell.execute_reply":"2021-12-21T14:30:08.325345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Training**","metadata":{}},{"cell_type":"code","source":"cfg = get_cfg()\nconfig_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\n\ncfg.merge_from_file(model_zoo.get_config_file(config_name))\ncfg.DATASETS.TRAIN = (Data_Resister_training,)\ncfg.DATASETS.TEST = (Data_Resister_valid,)\n\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(config_name)\n# cfg.MODEL.DEVICE = \"cpu\"\n\ncfg.DATALOADER.NUM_WORKERS = 2\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = 3 \ncfg.SOLVER.IMS_PER_BATCH = 2 \ncfg.INPUT.MASK_FORMAT='polygon'\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5\n\n\ncfg.SOLVER.LR_SCHEDULER_NAME = \"WarmupCosineLR\"\ncfg.SOLVER.BASE_LR = 0.005 \ncfg.SOLVER.MOMENTUM = 0.9\n\n\n    \ncfg.SOLVER.WARMUP_ITERS = 400 \ncfg.SOLVER.MAX_ITER = 40000 \ncfg.TEST.EVAL_PERIOD = 300\ncfg.SOLVER.CHECKPOINT_PERIOD = 300\n\n\nos.makedirs(cfg.OUTPUT_DIR, exist_ok=True) \ntrainer = MyTrainer(cfg) ","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:30:08.327981Z","iopub.execute_input":"2021-12-21T14:30:08.328663Z","iopub.status.idle":"2021-12-21T14:30:11.836379Z","shell.execute_reply.started":"2021-12-21T14:30:08.328623Z","shell.execute_reply":"2021-12-21T14:30:11.835602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# trainer.resume_or_load(resume=True)\n# trainer.train()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:30:11.837689Z","iopub.execute_input":"2021-12-21T14:30:11.838001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}