{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!python -m pip install 'git+https://github.com/facebookresearch/detectron2.git'","metadata":{"execution":{"iopub.status.busy":"2021-10-17T01:46:25.778465Z","iopub.execute_input":"2021-10-17T01:46:25.779554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nfrom pprint import pprint\nimport gc\nimport os\nimport random\n\nimport copy\nfrom glob import glob\nimport cv2\nfrom PIL import Image\nimport random\nfrom collections import deque, defaultdict\nfrom multiprocessing import Pool, Process\nfrom functools import partial\nfrom pathlib import Path\nimport itertools\n\nimport torch\n\nimport pycocotools\nimport detectron2\nfrom detectron2.config import get_cfg\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer\nfrom detectron2.utils.visualizer import Visualizer, ColorMode\nfrom detectron2.structures import BoxMode\nfrom detectron2.data import datasets, DatasetCatalog, MetadataCatalog, build_detection_train_loader, build_detection_test_loader\nfrom detectron2.data import transforms as T\nfrom detectron2.data import detection_utils as utils\nfrom detectron2.evaluation import COCOEvaluator, verify_results\nfrom detectron2.modeling import GeneralizedRCNNWithTTA\nfrom detectron2.data.transforms import TransformGen\nfrom detectron2.utils.logger import setup_logger\nsetup_logger()\n\nfrom fvcore.transforms.transform import TransformList, Transform, NoOpTransform\nfrom contextlib import contextmanager\n\nimport torch.nn as nn\nfrom collections import Counter","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = Path('../input/sartorius-cell-instance-segmentation')\ntrain_img_dir = Path(data_dir / 'train')\ntest_img_dir = Path(data_dir / 'test')\n\nsub_path = Path(data_dir / 'sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(data_dir / 'train.csv')\nsub_df = pd.read_csv(sub_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Some EDA","metadata":{}},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How many cell type?","metadata":{}},{"cell_type":"code","source":"# List of unique cell types:\nprint(list(set(df['cell_type'].tolist())))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We have 3 differents types of cells, We will use them on MetadataCatalog.get(dataset_label).thing_classes\n1. **astro**\n2. **cort**\n3. **shsy5y**","metadata":{}},{"cell_type":"markdown","source":"## Number of instances","metadata":{}},{"cell_type":"code","source":"# Number of instances\ndf.shape[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Number of unique instances","metadata":{}},{"cell_type":"code","source":"print('There are '+str(len(list(set(df['id'].tolist()))))+' unique image')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Groupby image_id to get an image per row\nunique_images = df.groupby('id')[['annotation', 'width', 'height', 'cell_type']].agg(lambda x: list(x)).reset_index()\nunique_images.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Number of segmentations per image","metadata":{}},{"cell_type":"code","source":"cnt = {}\n\nfor idx, row in unique_images.iterrows(): \n    length = len(unique_images.annotation.iloc[idx])\n    image_id = unique_images.id.iloc[idx]\n    cnt.update({id: length})\n\ndf_count_segments = pd.Series(data=cnt)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Average number of segmentations per image: {df_count_segments.values.mean()}')\nprint(f'Max number of segmentations: {df_count_segments.values.max()}')\nprint(f'Min number of segmentations: {df_count_segments.values.min()}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's take a look to some images ","metadata":{}},{"cell_type":"code","source":"# ref: https://www.kaggle.com/inversion/run-length-decoding-quick-start\n# ref: updated by https://www.kaggle.com/ihelon/cell-segmentation-run-length-decoding\ndef rle_decode(mask_rle, shape, color=1):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height, width, channels) of array to return \n    color: color for the mask\n    Returns numpy array (mask)\n\n    '''\n    s = mask_rle.split()\n    \n    starts = list(map(lambda x: int(x) - 1, s[0::2]))\n    lengths = list(map(int, s[1::2]))\n    ends = [x + y for x, y in zip(starts, lengths)]\n    \n    img = np.zeros((shape[0] * shape[1], shape[2]), dtype=np.float32)\n            \n    for start, end in zip(starts, ends):\n        img[start : end] = color\n    \n    return img.reshape(shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_masks(image_id, colors=True):\n    labels = df[df[\"id\"] == image_id][\"annotation\"].tolist()\n\n    if colors:\n        mask = np.zeros((520, 704, 3))\n        for label in labels:\n            mask += rle_decode(label, shape=(520, 704, 3), color=np.random.rand(3))\n    else:\n        mask = np.zeros((520, 704, 1))\n        for label in labels:\n            mask += rle_decode(label, shape=(520, 704, 1))\n    mask = mask.clip(0, 1)\n\n    image = cv2.imread(f\"../input/sartorius-cell-instance-segmentation/train/{image_id}.png\")\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n    plt.figure(figsize=(16, 32))\n    plt.subplot(3, 1, 1)\n    plt.imshow(image)\n    plt.axis(\"off\")\n    plt.subplot(3, 1, 2)\n    plt.imshow(image)\n    plt.imshow(mask, alpha=0.5)\n    plt.axis(\"off\")\n    plt.subplot(3, 1, 3)\n    plt.imshow(mask)\n    plt.axis(\"off\")\n    \n    plt.show();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_masks(\"ffdb3cc02eef\", colors=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_masks(\"ffdb3cc02eef\", colors=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_masks(\"73df2962444f\", colors=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_masks(\"13325f865bb0\", colors=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Detectron2 on a custom dataset","metadata":{}},{"cell_type":"code","source":"cfg = get_cfg()\n\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model\n\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")\npredictor = DefaultPredictor(cfg)\n\nim = cv2.imread('../input/sartorius-cell-instance-segmentation/train/0030fd0e6378.png')\noutputs = predictor(im[..., ::-1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Detectron2 dataset dict","metadata":{}},{"cell_type":"code","source":"def get_cell_dict(df, img_folder):\n    \n    grps = df['id'].unique().tolist()\n    df_group = df.groupby('id')\n    dataset_dicts = []\n    \n    for idx, image_name in enumerate(tqdm(grps)):\n        \n        # Get all instances of an image\n        group = df_group.get_group(image_name)\n        \n        record = defaultdict()\n        \n        # Full image path \n        file_path = os.path.join(img_folder, image_name + '.jpg')\n        \n        record['height'] = int(520)\n        record['width'] = int(704)\n        record['file_name'] = file_path\n        record['image_id'] = idx\n        \n        objs = []\n        \n        # Iterate over the group's rows as namedtuples\n        for ant in group.itertuples():\n\n            \n            s = ant.annotation[1:-1]\n            #print(s)\n            s = s.split()\n    \n            starts = list(map(lambda x: int(x) - 1, s[0::2]))\n            lengths = list(map(int, s[1::2]))\n            \n            \n            \n            xmin, ymin, xmax, ymax = min(starts), min(lengths), max(starts), max(lengths)\n            \n            poly = []\n            for x,y in zip(starts, lengths):\n                poly.append((x,y))\n            \n            \n            poly = list(itertools.chain.from_iterable(poly))\n            \n            obj = {\n                'bbox': [xmin, ymin, xmax, ymax], # change to XYXY format. Original was in XYWH\n                'bbox_mode': BoxMode.XYXY_ABS,\n                'segmentation': [poly],\n                'category_id': ant.cell_type[0], # only 1 category for this dataset\n                'iscrowd': 0\n                  }\n            \n            objs.append(obj)\n        record['annotations'] = objs\n        dataset_dicts.append(record)\n\n    return dataset_dicts\n\n##saaa7","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CutOut(Transform):\n    \n    def __init__(self, box_size=50, prob_cutmix=0.8):\n        super().__init__()\n        \n        self.box_size = box_size\n        self.prob_cutmix = prob_cutmix\n        \n    def apply_image(self, img):\n        \n        if random.random() > self.prob_cutmix:\n            \n            h, w = img.shape[:2]\n            num_rand = np.random.randint(10, 20)\n            for num_cut in range(num_rand):\n                x_rand, y_rand = random.randint(0, w-self.box_size), random.randint(0, h-self.box_size)\n                img[x_rand:x_rand+self.box_size, y_rand:y_rand+self.box_size, :] = 0\n        \n        return np.asarray(img)\n    \n    def apply_coords(self, coords):\n        return coords.astype(np.float32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DatasetMapper:\n    \"\"\"\n    A callable which takes a dataset dict in Detectron2 Dataset format,\n    and map it into a format used by the model.\n\n    This is a custom version of the DatasetMapper. The only different with Detectron2's \n    DatasetMapper is that we extract attributes from our dataset_dict. \n    \"\"\"\n\n    def __init__(self, cfg, is_train=True):\n        if cfg.INPUT.CROP.ENABLED and is_train:\n            self.crop_gen = T.RandomCrop(cfg.INPUT.CROP.TYPE, cfg.INPUT.CROP.SIZE)\n            logging.getLogger(__name__).info(\"CropGen used in training: \" + str(self.crop_gen))\n        else:\n            self.crop_gen = None\n        \n        self.tfm_gens = [T.RandomBrightness(0.8, 1.8),\n                         T.RandomContrast(0.6, 1.3),\n                         T.RandomSaturation(0.8, 1.4),\n                         T.RandomRotation(angle=[90, 90]),\n                         T.RandomLighting(0.7),\n                         T.RandomFlip(prob=0.4, horizontal=False, vertical=True),\n                         T.RandomCrop('relative_range', (0.4, 0.6)),\n                         CutOut()\n                        ]\n\n        # self.tfm_gens = utils.build_transform_gen(cfg, is_train)\n\n        # fmt: off\n        self.img_format     = cfg.INPUT.FORMAT\n        self.mask_on        = cfg.MODEL.MASK_ON\n        self.mask_format    = cfg.INPUT.MASK_FORMAT\n        self.keypoint_on    = cfg.MODEL.KEYPOINT_ON\n        self.load_proposals = cfg.MODEL.LOAD_PROPOSALS\n        # fmt: on\n        if self.keypoint_on and is_train:\n            # Flip only makes sense in training\n            self.keypoint_hflip_indices = utils.create_keypoint_hflip_indices(cfg.DATASETS.TRAIN)\n        else:\n            self.keypoint_hflip_indices = None\n\n        if self.load_proposals:\n            self.min_box_side_len = cfg.MODEL.PROPOSAL_GENERATOR.MIN_SIZE\n            self.proposal_topk = (\n                cfg.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TRAIN\n                if is_train\n                else cfg.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TEST\n            )\n        self.is_train = is_train\n\n    def __call__(self, dataset_dict):\n        \"\"\"\n        Args:\n            dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format.\n\n        Returns:\n            dict: a format that builtin models in detectron2 accept\n        \"\"\"\n        dataset_dict = copy.deepcopy(dataset_dict)  # it will be modified by code below\n        # USER: Write your own image loading if it's not from a file\n        image = utils.read_image(dataset_dict[\"file_name\"], format=self.img_format)\n        utils.check_image_size(dataset_dict, image)\n\n        if \"annotations\" not in dataset_dict:\n            image, transforms = T.apply_transform_gens(\n                ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, image\n            )\n        else:\n            # Crop around an instance if there are instances in the image.\n            # USER: Remove if you don't use cropping\n            if self.crop_gen:\n                crop_tfm = utils.gen_crop_transform_with_instance(\n                    self.crop_gen.get_crop_size(image.shape[:2]),\n                    image.shape[:2],\n                    np.random.choice(dataset_dict[\"annotations\"]),\n                )\n                image = crop_tfm.apply_image(image)\n            image, transforms = T.apply_transform_gens(self.tfm_gens, image)\n            if self.crop_gen:\n                transforms = crop_tfm + transforms\n\n        image_shape = image.shape[:2]  # h, w\n\n        # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory,\n        # but not efficient on large generic data structures due to the use of pickle & mp.Queue.\n        # Therefore it's important to use torch.Tensor.\n        dataset_dict[\"image\"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1)))\n\n        # USER: Remove if you don't use pre-computed proposals.\n        if self.load_proposals:\n            utils.transform_proposals(\n                dataset_dict, image_shape, transforms, self.min_box_side_len, self.proposal_topk\n            )\n\n        if not self.is_train:\n            # USER: Modify this if you want to keep them for some reason.\n            dataset_dict.pop(\"annotations\", None)\n            dataset_dict.pop(\"sem_seg_file_name\", None)\n            return dataset_dict\n\n        if \"annotations\" in dataset_dict:\n            # USER: Modify this if you want to keep them for some reason.\n            for anno in dataset_dict[\"annotations\"]:\n                if not self.mask_on:\n                    anno.pop(\"segmentation\", None)\n                if not self.keypoint_on:\n                    anno.pop(\"keypoints\", None)\n\n            # USER: Implement additional transformations if you have other types of data\n            annos = [\n                utils.transform_instance_annotations(\n                    obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices\n                )\n                for obj in dataset_dict.pop(\"annotations\")\n                if obj.get(\"iscrowd\", 0) == 0\n            ]\n            instances = utils.annotations_to_instances(\n                annos, image_shape, mask_format=self.mask_format\n            )\n            # Create a tight bounding box from masks, useful when image is cropped\n            if self.crop_gen and instances.has(\"gt_masks\"):\n                instances.gt_boxes = instances.gt_masks.get_bounding_boxes()           \n                          \n            dataset_dict[\"instances\"] = utils.filter_empty_instances(instances)\n            \n             # USER: Remove if you don't do semantic/panoptic segmentation.\n        if \"sem_seg_file_name\" in dataset_dict:\n            with PathManager.open(dataset_dict.pop(\"sem_seg_file_name\"), \"rb\") as f:\n                sem_seg_gt = Image.open(f)\n                sem_seg_gt = np.asarray(sem_seg_gt, dtype=\"uint8\")\n            sem_seg_gt = transforms.apply_segmentation(sem_seg_gt)\n            sem_seg_gt = torch.as_tensor(sem_seg_gt.astype(\"long\"))\n            dataset_dict[\"sem_seg\"] = sem_seg_gt\n\n        return dataset_dict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CellTrainer(DefaultTrainer):\n    \n    @classmethod\n    def build_train_loader(cls, cfg):\n        return build_detection_train_loader(cfg, mapper=DatasetMapper(cfg))\n    \n    @classmethod\n    def build_test_loader(cls, cfg, dataset_name):\n        return build_detection_test_loader(cfg, dataset_name, mapper=DatasetMapper(cfg))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_test_datasets(df, n_sample_size=-1, train_test_split=0.8):\n    \n    # Option to set a smaller df size for testing\n    if n_sample_size == -1:\n        n_sample_size = df.shape[0]\n    elif n_sample_size != -1:\n        n_sample_size = df[:n_sample_size].shape[0]\n\n    # Split df into train / test dataframes\n    n_train = round(n_sample_size * train_test_split)\n    n_test = n_sample_size - n_train\n\n    df_train = df[:n_train].copy()\n    df_test = df[-n_test:].copy()\n    \n    return df_train, df_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def register_dataset(df, dataset_label='cell_train', image_dir=train_img_dir):\n    \n    # Register dataset - if dataset is already registered, give it a new name    \n    try:\n        DatasetCatalog.register(dataset_label, lambda d=df: get_cell_dict(df, image_dir))\n        MetadataCatalog.get(dataset_label).thing_classes = ['astro', 'cort', 'shsy5y']\n    except:\n        # Add random int to dataset name to not run into 'Already registered' error\n        n = random.randint(1, 1000)\n        dataset_label = dataset_label + str(n)\n        DatasetCatalog.register(dataset_label, lambda d=df: get_cell_dict(df, image_dir))\n        MetadataCatalog.get(dataset_label).thing_classes = ['astro', 'cort', 'shsy5y']\n        \n    return MetadataCatalog.get(dataset_label), dataset_label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register train dataset\nmetadata, train_dataset = register_dataset(df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register test dataset\nmetadata, test_dataset = register_dataset(sub_df, dataset_label='cell_test', image_dir=test_img_dir)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_dict = get_cell_dict(df, train_img_dir)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_USE = 'faster_rcnn'\nif MODEL_USE == 'faster_rcnn':\n    MODEL_PATH = 'COCO-Detection/faster_rcnn_R_101_FPN_3x.yaml'\n    WEIGHT_PATH = '../input/r101fpn400/model_final_f96b26.pkl'\nelif MODEL_USE == 'retinanet':\n    MODEL_PATH = 'COCO-Detection/retinanet_R_101_FPN_3x.yaml'\n    WEIGHT_PATH = '/kaggle/input/model-final/model_retina.pth' \nelif MODEL_USE == 'mask_rcnn':\n    MODEL_PATH = 'COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x.yaml'\n    WEIGHT_PATH = 'COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x.yaml'\nelif MODEL_USE == 'cascade_mask_rcnn':\n    MODEL_PATH = 'Misc/cascade_mask_rcnn_R_50_FPN_3x.yaml'\n    WEIGHT_PATH = '/kaggle/input/model-cascade-10000/model_final_cascade_10000.pth'\n\ndef cfg_setup():\n    \n    cfg = get_cfg()\n    cfg.merge_from_file(model_zoo.get_config_file(MODEL_PATH))\n    cfg.MODEL.WEIGHTS = WEIGHT_PATH # model_zoo.get_checkpoint_url(WEIGHT_PATH)  \n    cfg.MODEL.RETINANET.NUM_CLASSES = 3\n    cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 512\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 3\n\n    cfg.DATASETS.TRAIN = (train_dataset,)\n    cfg.DATASETS.TEST = ()\n    cfg.DATALOADER.NUM_WORKERS = 4\n\n    cfg.SOLVER.IMS_PER_BATCH = 4\n    cfg.SOLVER.LR_SCHEDULER_NAME = 'WarmupCosineLR'\n    cfg.SOLVER.BASE_LS = 0.0002\n    cfg.SOLVER.MAX_ITER = 500\n    os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)\n        \n    return cfg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg = cfg_setup()\n#trainer = CellTrainer(cfg)   ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# I am still bilding this notebook, an uptove will motivate me too much, Thanks n_n!\nreference: [Link](https://www.kaggle.com/julienbeaulieu/detectron2-wheat-detection-eda-training-eval)","metadata":{}}]}