{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Initialization","metadata":{}},{"cell_type":"markdown","source":"## Packages","metadata":{}},{"cell_type":"code","source":"%%time\n# Torch 1.7\n!pip install '/kaggle/input/pytorch-170-cuda-toolkit-110221/torch-1.7.0+cu110-cp37-cp37m-linux_x86_64.whl' --no-deps\n\n# MMCV & MMDET Requirements\n!pip install ../input/mmdetection-v217/mmdetection/addict-2.4.0-py3-none-any.whl --no-deps\n!pip install ../input/mmdetection-v217/mmdetection/yapf-0.31.0-py2.py3-none-any.whl --no-deps\n!pip install ../input/mmdetection-v217/mmdetection/terminal-0.4.0-py3-none-any.whl --no-deps\n!pip install ../input/mmdetection-v217/mmdetection/terminaltables-3.1.0-py3-none-any.whl --no-deps\n!pip install ../input/mmdetection-v217/mmdetection/pycocotools-2.0.2/pycocotools-2.0.2 --no-deps\n!pip install ../input/mmdetection-v217/mmdetection/mmpycocotools-12.0.3/mmpycocotools-12.0.3 --no-deps\n\n# MMCV\n!pip install ../input/detection-packages/mmcv_full-1.3.16-cp37-cp37m-manylinux1_x86_64.whl --no-deps\n# MMDET\n!pip install ../input/detection-packages/mmdetection-2.17.0/mmdetection-2.17.0 --no-deps\n# !pip install ../input/detection-packages/mmdetection-master/mmdetection-master --no-deps","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:43:28.270947Z","iopub.execute_input":"2021-12-30T16:43:28.271418Z","iopub.status.idle":"2021-12-30T16:48:14.705488Z","shell.execute_reply.started":"2021-12-30T16:43:28.271308Z","shell.execute_reply":"2021-12-30T16:48:14.704321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cd /kaggle/working/","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:14.707628Z","iopub.execute_input":"2021-12-30T16:48:14.707894Z","iopub.status.idle":"2021-12-30T16:48:15.455502Z","shell.execute_reply.started":"2021-12-30T16:48:14.707839Z","shell.execute_reply":"2021-12-30T16:48:15.454468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport json\nimport glob\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom tqdm.notebook import tqdm\n\nwarnings.simplefilter(\"ignore\", UserWarning)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:15.458769Z","iopub.execute_input":"2021-12-30T16:48:15.459319Z","iopub.status.idle":"2021-12-30T16:48:16.462144Z","shell.execute_reply.started":"2021-12-30T16:48:15.459278Z","shell.execute_reply":"2021-12-30T16:48:16.461046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Params","metadata":{}},{"cell_type":"code","source":"DATA_PATH = \"/kaggle/input/sartorius-cell-instance-segmentation/\"\nCELL_TYPES = [\"shsy5y\", \"astro\", \"cort\"]\nORIG_SIZE = (520, 704)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:16.464669Z","iopub.execute_input":"2021-12-30T16:48:16.465044Z","iopub.status.idle":"2021-12-30T16:48:16.47292Z","shell.execute_reply.started":"2021-12-30T16:48:16.465004Z","shell.execute_reply":"2021-12-30T16:48:16.469124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions","metadata":{}},{"cell_type":"markdown","source":"## Utils","metadata":{}},{"cell_type":"markdown","source":"### Torch","metadata":{}},{"cell_type":"code","source":"import os\nimport torch\nimport random\nimport numpy as np\n\n\ndef seed_everything(seed):\n    \"\"\"\n    Seeds basic parameters for reproductibility of results.\n\n    Args:\n        seed (int): Number of the seed.\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n\ndef load_model_weights(model, filename, verbose=1, cp_folder=\"\"):\n    \"\"\"\n    Loads the weights of a PyTorch model. The exception handles cpu/gpu incompatibilities.\n\n    Args:\n        model (torch model): Model to load the weights to.\n        filename (str): Name of the checkpoint.\n        verbose (int, optional): Whether to display infos. Defaults to 1.\n        cp_folder (str, optional): Folder to load from. Defaults to \"\".\n\n    Returns:\n        torch model: Model with loaded weights.\n    \"\"\"\n\n    if verbose:\n        print(f\"\\n -> Loading weights from {os.path.join(cp_folder,filename)}\\n\")\n    try:\n        model.load_state_dict(os.path.join(cp_folder, filename), strict=True)\n    except BaseException:\n        model.load_state_dict(\n            torch.load(os.path.join(cp_folder, filename), map_location=\"cpu\"),\n            strict=True,\n        )\n    return model","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:16.475002Z","iopub.execute_input":"2021-12-30T16:48:16.475431Z","iopub.status.idle":"2021-12-30T16:48:17.189739Z","shell.execute_reply.started":"2021-12-30T16:48:16.475364Z","shell.execute_reply":"2021-12-30T16:48:17.188893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plots","metadata":{}},{"cell_type":"code","source":"import cv2\nimport skimage\nimport numpy as np\nimport plotly.express as px\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\n\nfrom scipy import ndimage\nfrom mmdet.core import BitmapMasks\nfrom matplotlib.patches import Rectangle\n\n# from utils.metrics import compute_iou\n\n\nGREEN = (56 / 255, 200 / 255, 100 / 255)\nBLUE = (32 / 255, 50 / 255, 155 / 255)\nRED = (238 / 255, 97 / 255, 55 / 255)\n\n\ndef get_random_color():\n    color = tuple(np.random.random(size=3))\n    while np.max(color) - np.min(color) < 0.2:\n        color = tuple(np.random.random(size=3))\n    return color\n\n\ndef plot_sample(img, mask=None, boxes=[], width=1, plotly=False):\n    \"\"\"\n    Plots the contours of a given mask.\n\n    Args:\n        img (numpy array [H x W]): Image.\n        mask (numpy array [H x W x C]): Masks.\n        width (int, optional): Contour width. Defaults to 1.\n\n    Returns:\n        img (numpy array [H x W]): Image with contours.\n    \"\"\"\n\n    if img.max() > 1:\n        img = (img / 255).astype(float)\n\n    if len(img.shape) == 2:\n        img = np.stack([img, img, img], -1)\n\n    img_ = img.copy()\n\n    colors = []\n\n    if isinstance(mask, BitmapMasks):\n        mask = mask.masks.astype(int)\n        for i in range(len(mask)):\n            mask[i] *= (i + 1)\n\n    if mask is not None:\n        if len(mask.shape) == 3:\n            if mask.max() == 1:\n                for i in range(len(mask)):\n                    mask[i] *= (i + 1)\n            mask = mask.max(0)\n\n        for i in range(1, int(np.max(mask)) + 1):\n            m = ((mask == i) * 255).astype(np.uint8)\n            color = get_random_color()\n            colors.append(color)\n\n            contours, _ = cv2.findContours(m, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)\n            cv2.polylines(img_, contours, True, color, width)\n\n    if not plotly:\n        plt.imshow(img_)\n\n        # Add boxes\n        for i, box in enumerate(boxes):\n            color = colors[i] if len(colors) else get_random_color()\n            rect = Rectangle(\n                (box[0], box[1]), box[2] - box[0], box[3] - box[1],\n                linewidth=1, edgecolor=color, facecolor='none', alpha=0.5\n            )\n            plt.gca().add_patch(rect)\n\n    if plotly:\n        return px.imshow(img_)\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:17.191269Z","iopub.execute_input":"2021-12-30T16:48:17.19161Z","iopub.status.idle":"2021-12-30T16:48:39.522451Z","shell.execute_reply.started":"2021-12-30T16:48:17.191555Z","shell.execute_reply":"2021-12-30T16:48:39.521361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Config","metadata":{}},{"cell_type":"code","source":"class Config:\n    \"\"\"\n    Placeholder to load a config from a saved json\n    \"\"\"\n    def __init__(self, dic):\n        for k, v in dic.items():\n            setattr(self, k, v)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:39.52476Z","iopub.execute_input":"2021-12-30T16:48:39.525164Z","iopub.status.idle":"2021-12-30T16:48:39.533795Z","shell.execute_reply.started":"2021-12-30T16:48:39.525092Z","shell.execute_reply":"2021-12-30T16:48:39.532625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### RLE","metadata":{}},{"cell_type":"code","source":"def rle_encoding(x):\n    dots = np.where(x.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b > prev + 1):\n            run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return ' '.join(map(str, run_lengths))\n\n\ndef rle_decode(mask_rle, shape):\n    \"\"\"\n    Decodes a rle.\n\n    Args:\n        mask_rle (str): Run length encoding.\n        shape (tuple [2]): Mask size (height, width).\n\n    Returns:\n        np array [shape]: Mask.\n    \"\"\"\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo: hi] = 1\n    return img.reshape(shape)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:39.535694Z","iopub.execute_input":"2021-12-30T16:48:39.536465Z","iopub.status.idle":"2021-12-30T16:48:39.549356Z","shell.execute_reply.started":"2021-12-30T16:48:39.536408Z","shell.execute_reply":"2021-12-30T16:48:39.548166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"markdown","source":"### Transforms","metadata":{}},{"cell_type":"code","source":"import mmcv\nfrom mmcv.utils import build_from_cfg\nfrom mmdet.datasets.builder import PIPELINES\nfrom mmdet.datasets.pipelines import Compose\n\n\ndef define_pipelines(config_file, multi_image=False):\n    pipe_cfg = mmcv.Config.fromfile(config_file).data\n\n    if not multi_image:\n        pipelines = {\n            k: Compose(\n                [build_from_cfg(aug, PIPELINES, None) for aug in pipe_cfg[k].pipeline]\n            ) for k in pipe_cfg\n        }\n    else:\n        pipelines = {\n            k: pipe_cfg[k].pipeline for k in pipe_cfg\n        }\n    return pipelines","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:39.551235Z","iopub.execute_input":"2021-12-30T16:48:39.551699Z","iopub.status.idle":"2021-12-30T16:48:43.200407Z","shell.execute_reply.started":"2021-12-30T16:48:39.551657Z","shell.execute_reply":"2021-12-30T16:48:43.199389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset","metadata":{}},{"cell_type":"code","source":"import cv2\nimport pycocotools\nimport numpy as np\nfrom torch.utils.data import Dataset\nfrom mmdet.core import BitmapMasks\n\n\nRESULTS_PH = {\n    'scale_factor': np.ones(4, dtype=np.float32),  # if no resizing in augs\n    \"pad_shape\": (0, 0),\n    \"img_norm_cfg\": None,\n    \"flip_direction\": None,\n    \"flip\": None,\n    'img_fields': [\"img\"],\n    'bbox_fields': [\"gt_bboxes\"],\n    'mask_fields': [\"gt_masks\"]\n}\n\n\nclass SartoriusInferenceDataset(Dataset):\n    \"\"\"\n    Segmentation dataset for training / validation.\n    \"\"\"\n    def __init__(self, df, transforms, precompute_masks=True):\n        \"\"\"\n        Constructor.\n\n        Args:\n            df (pandas dataframe): Metadata.\n            transforms (albumentation transforms, optional): Transforms to apply. Defaults to None.\n            train (bool, optional): Indicates if the dataset is used for training. Defaults to True.\n        \"\"\"\n\n        self.df = df\n        self.transforms = transforms\n\n        self.img_paths = df[\"img_path\"].values\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, idx):\n        image = cv2.imread(self.img_paths[idx])\n\n        results = {\n            \"img\": image,\n            \"img_shape\": image.shape[:2],\n            \"ori_shape\": image.shape[:2],\n            \"filename\": self.img_paths[idx],\n            'ori_filename': self.img_paths[idx],\n        }\n        results.update(RESULTS_PH)\n        del results['bbox_fields'], results['mask_fields']\n\n        results_transfo = self.transforms(results.copy())\n\n        # if 'scale_factor' not in results_transfo.keys():\n        #     results_transfo['scale_factor'] = np.ones(4)\n\n        return results_transfo\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:43.203325Z","iopub.execute_input":"2021-12-30T16:48:43.204078Z","iopub.status.idle":"2021-12-30T16:48:43.216723Z","shell.execute_reply.started":"2021-12-30T16:48:43.204035Z","shell.execute_reply":"2021-12-30T16:48:43.215681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loaders","metadata":{}},{"cell_type":"code","source":"from functools import partial\nfrom mmcv.parallel import collate\nfrom torch.utils.data import DataLoader\n\n\ndef define_loaders(\n    train_dataset=None, val_dataset=None,  batch_size=32, val_bs=32, num_workers=0\n):\n    \"\"\"\n    Builds data loaders. TODO\n\n    Args:\n        train_dataset (CollageingDataset): Dataset to train with.\n        val_dataset (CollageingDataset): Dataset to validate with.\n        samples_per_patient (int, optional): Number of images to use per patient. Defaults to 0.\n        batch_size (int, optional): Training batch size. Defaults to 32.\n        val_bs (int, optional): Validation batch size. Defaults to 32.\n\n    Returns:\n       DataLoader: Train loader.\n       DataLoader: Val loader.\n    \"\"\"\n    train_loader, val_loader = None, None\n\n    if val_dataset is not None:\n        val_loader = DataLoader(\n            val_dataset,\n            batch_size=val_bs,\n            shuffle=False,\n            collate_fn=partial(collate, samples_per_gpu=batch_size),\n            num_workers=num_workers,\n            pin_memory=True,\n        )\n\n    return train_loader, val_loader\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:43.22404Z","iopub.execute_input":"2021-12-30T16:48:43.224683Z","iopub.status.idle":"2021-12-30T16:48:43.238836Z","shell.execute_reply.started":"2021-12-30T16:48:43.224628Z","shell.execute_reply":"2021-12-30T16:48:43.237662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport timm\n\nimport numpy as np\nimport torch.nn as nn\nfrom mmdet.models.builder import BACKBONES\n\nMEAN = np.array([0.66437738, 0.50478148, 0.70114894])\nSTD = np.array([0.15825711, 0.24371008, 0.13832686])\n\n\n@BACKBONES.register_module()\nclass EfficientNet(nn.Module):\n    def __init__(self, name, blocks_idx, pretrained=True):\n        \"\"\"\n        Constructor.\n\n        Args:\n            name (name): Model name as specified in timm.\n            blocks_idx (list of ints): Blocks to output features at.\n            pretrained (bool, optional): Whether to load pretrained weights. Defaults to True.\n        \"\"\"\n        super().__init__()\n\n        self.effnet = getattr(timm.models, name)(\n            pretrained=pretrained,\n            drop_path_rate=0.2,\n        )\n\n        self.block_idx = blocks_idx\n        self.nb_fts = [self.effnet.blocks[b][-1].conv_pwl.out_channels for b in self.block_idx]\n        self.nb_ft = self.nb_fts[-1]\n\n        if \"efficientnetv2\" in name:\n            self.mean = np.array([0.5, 0.5, 0.5])\n            self.std = np.array([0.5, 0.5, 0.5])\n        else:\n            self.mean = MEAN\n            self.std = STD\n\n        self.name = name\n\n    def forward(self, x):  # should return a tuple\n        \"\"\"\n        Extract features for an EfficientNet model.\n        Args:\n            x (torch tensor [BS x 3 x H x W]): Input image.\n        Returns:\n            list of torch tensors: features.\n        \"\"\"\n        x = self.effnet.conv_stem(x)\n        x = self.effnet.bn1(x)\n        x = self.effnet.act1(x)\n\n        features = []\n        for i, b in enumerate(self.effnet.blocks):\n            x = b(x)\n            if i in self.block_idx:\n                features.append(x)\n            # print(i, x.size(), i in self.block_idx)\n\n        return features\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:43.241357Z","iopub.execute_input":"2021-12-30T16:48:43.242374Z","iopub.status.idle":"2021-12-30T16:48:49.579038Z","shell.execute_reply.started":"2021-12-30T16:48:43.24233Z","shell.execute_reply":"2021-12-30T16:48:49.577823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nimport mmcv\nimport torch\nimport logging\n\nfrom mmcv.parallel import MMDataParallel\nfrom mmdet.models.builder import build_detector\n\n\ndef define_model(config_file, encoder=\"resnet50\", pretrained_livecell=False, verbose=1):\n    # Configs\n    cfg = mmcv.Config.fromfile(config_file)\n\n    config_backbone_file = config_file.rsplit('/', 1)[0] + \"/config_backbones.py\"\n    cfg_backbones = mmcv.Config.fromfile(config_backbone_file)\n    \n    if 'pretrained' in cfg_backbones.backbones[encoder].keys():\n        cfg_backbones.backbones[encoder]['pretrained'] = False\n\n    cfg.model.backbone = cfg_backbones.backbones[encoder]\n\n    if encoder in cfg_backbones.out_channels.keys():  # update neck channels\n        cfg.model.neck.in_channels = cfg_backbones.out_channels[encoder]\n\n    # Build model\n    model = build_detector(cfg.model)\n    model.test_cfg = cfg[\"model\"][\"test_cfg\"]\n    model.train_cfg = cfg[\"model\"][\"train_cfg\"]\n\n    # Reduce stride\n    if \"resnet\" in encoder or \"resnext\" in encoder:\n        model.backbone.conv1.stride = (1, 1)\n    elif \"efficientnet\" in encoder:\n        model.backbone.effnet.conv_stem.stride = (1, 1)\n\n    model = MMDataParallel(model)\n\n#     # Weights\n#     try:\n#         weights = (\n#             cfg.pretrained_weights_livecell[encoder]\n#             if pretrained_livecell\n#             else cfg.pretrained_weights[encoder]\n#         )\n#     except KeyError:\n#         weights = None\n\n#     model = load_pretrained_weights(\n#         model,\n#         weights,\n#         verbose=verbose,\n#         adapt_swin=\"swin\" in encoder and not pretrained_livecell\n#     )\n\n    return model","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:49.582262Z","iopub.execute_input":"2021-12-30T16:48:49.583481Z","iopub.status.idle":"2021-12-30T16:48:49.595651Z","shell.execute_reply.started":"2021-12-30T16:48:49.583428Z","shell.execute_reply":"2021-12-30T16:48:49.594457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble Model","metadata":{}},{"cell_type":"markdown","source":"### Merging","metadata":{}},{"cell_type":"code","source":"import torch\nfrom mmcv.ops import nms\n\nfrom mmdet.core.bbox import bbox_mapping_back\n\n\ndef merge_aug_proposals(aug_proposals, img_metas, cfg):\n    \"\"\"Merge augmented proposals (multiscale, flip, etc.)\n\n    Args:\n        aug_proposals (list[Tensor]): proposals from different testing\n            schemes, shape (n, 5). Note that they are not rescaled to the\n            original image size.\n\n        img_metas (list[dict]): list of image info dict where each dict has:\n            'img_shape', 'scale_factor', 'flip', and may also contain\n            'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'.\n            For details on the values of these keys see\n            `mmdet/datasets/pipelines/formatting.py:Collect`.\n\n        cfg (dict): rpn test config.\n\n    Returns:\n        Tensor: shape (n, 4), proposals corresponding to original image scale.\n    \"\"\"\n    # Recover augmented proposals\n    recovered_proposals = []\n    for proposals, img_info in zip(aug_proposals, img_metas):\n        img_shape = img_info[\"img_shape\"]\n        scale_factor = img_info[\"scale_factor\"]\n        flip = img_info[\"flip\"]\n        flip_direction = img_info[\"flip_direction\"]\n        _proposals = proposals.clone()\n        _proposals[:, :4] = bbox_mapping_back(\n            _proposals[:, :4], img_shape, scale_factor, flip, flip_direction\n        )\n        recovered_proposals.append(_proposals)\n\n    # Merge proposals with NMS\n    aug_proposals = torch.cat(recovered_proposals, dim=0)\n    merged_proposals, _ = nms(\n        aug_proposals[:, :4].contiguous(),\n        aug_proposals[:, 4].contiguous(),\n        cfg.nms.iou_threshold,\n    )\n\n    # Reorder\n    scores = merged_proposals[:, 4]\n\n    scores, order = scores.sort(0, descending=True)\n\n    order = order[scores > cfg.score_thr]\n    order = order[:cfg.max_per_img]\n\n    merged_proposals = merged_proposals[order, :]\n\n    return merged_proposals\n\n\ndef merge_aug_bboxes(aug_bboxes, aug_scores, img_metas):\n    \"\"\"\n    Merge augmented detection bboxes and scores.\n    This simply takes the mean.\n\n    Args:\n        aug_bboxes (list[Tensor]): shape (n, 4*#class)\n        aug_scores (list[Tensor] or None): shape (n, #class)\n        img_shapes (list[Tensor]): shape (3, ).\n\n    Returns:\n        tuple: (bboxes, scores)\n    \"\"\"\n    # Recover augmented proposals\n    recovered_bboxes = []\n    for bboxes, img_info in zip(aug_bboxes, img_metas):\n        img_shape = img_info[0][\"img_shape\"]\n        scale_factor = img_info[0][\"scale_factor\"]\n        flip = img_info[0][\"flip\"]\n        flip_direction = img_info[0][\"flip_direction\"]\n        bboxes = bbox_mapping_back(\n            bboxes, img_shape, scale_factor, flip, flip_direction\n        )\n        recovered_bboxes.append(bboxes)\n\n    # Merge boxes by averaging predictions\n    bboxes = torch.stack(recovered_bboxes).mean(dim=0)\n\n    if aug_scores is None:\n        return bboxes\n    else:\n        scores = torch.stack(aug_scores).mean(dim=0)\n        return bboxes, scores\n\n\ndef single_class_boxes_nms(merged_bboxes, merged_scores, iou_threshold=0.5):\n    # Use most confident class per candidate\n    det_scores, det_labels = torch.max(merged_scores, 1)\n\n    # Get class & corresponding iou threshold\n    cell_type = torch.mode(det_labels, 0).values.item()\n    thresh = iou_threshold if isinstance(iou_threshold, (float, int)) else iou_threshold[cell_type]\n\n    # Filter with NMS\n    det_bboxes, inds = nms(\n        merged_bboxes.contiguous(), det_scores.contiguous(), thresh\n    )\n\n    return det_bboxes, det_labels[inds]\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:49.599188Z","iopub.execute_input":"2021-12-30T16:48:49.599733Z","iopub.status.idle":"2021-12-30T16:48:49.637195Z","shell.execute_reply.started":"2021-12-30T16:48:49.599681Z","shell.execute_reply":"2021-12-30T16:48:49.635908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Custom mask fct","metadata":{}},{"cell_type":"code","source":"import copy\nimport torch\nimport numpy as np\nfrom warnings import warn\n\nfrom mmcv.ops import batched_nms\nfrom mmdet.models.roi_heads.mask_heads.fcn_mask_head import (\n    BYTES_PER_FLOAT,\n    GPU_MEM_LIMIT,\n    _do_paste_mask,\n)\n\n\ndef get_seg_masks(\n    mask_head,\n    mask_pred,\n    det_bboxes,\n    det_labels,\n    rcnn_test_cfg,\n    ori_shape,\n    scale_factor,\n    rescale,\n    return_per_class=True,\n):\n    \"\"\"\n    Modified version of mmdet/models/roi_heads/mask_heads/fcn_mask_head.py\n    to add the return_per_class argument.\n    \"\"\"\n    if isinstance(mask_pred, torch.Tensor):\n        mask_pred = mask_pred.sigmoid()\n    else:\n        # In AugTest, has been activated before\n        mask_pred = det_bboxes.new_tensor(mask_pred)\n\n    device = mask_pred.device\n    cls_segms = [\n        [] for _ in range(mask_head.num_classes)\n    ]  # BG is not included in num_classes\n    bboxes = det_bboxes[:, :4]\n    labels = det_labels\n\n    # In most cases, scale_factor should have been\n    # converted to Tensor when rescale the bbox\n\n    if not isinstance(scale_factor, torch.Tensor):\n        if isinstance(scale_factor, float):\n            scale_factor = np.array([scale_factor] * 4)\n            warn(\n                \"Scale_factor should be a Tensor or ndarray \"\n                \"with shape (4,), float would be deprecated. \"\n            )\n        assert isinstance(scale_factor, np.ndarray)\n        scale_factor = torch.Tensor(scale_factor)\n\n    if rescale:\n        img_h, img_w = ori_shape[:2]\n        bboxes = bboxes / scale_factor.to(bboxes.device)\n    else:\n        w_scale, h_scale = scale_factor[0], scale_factor[1]\n        img_h = np.round(ori_shape[0] * h_scale.item()).astype(np.int32)\n        img_w = np.round(ori_shape[1] * w_scale.item()).astype(np.int32)\n\n    N = len(mask_pred)\n    if device.type == \"cpu\":\n        num_chunks = N\n    else:\n        # GPU benefits from parallelism for larger chunks,\n        num_chunks = int(\n            np.ceil(N * int(img_h) * int(img_w) * BYTES_PER_FLOAT / GPU_MEM_LIMIT)\n        )\n        assert num_chunks <= N, \"Default GPU_MEM_LIMIT is too small; try increasing it\"\n    chunks = torch.chunk(torch.arange(N, device=device), num_chunks)\n\n    threshold = rcnn_test_cfg.mask_thr_binary\n    im_mask = torch.zeros(\n        N,\n        img_h,\n        img_w,\n        device=device,\n        dtype=torch.bool if threshold >= 0 else torch.uint8,\n    )\n\n    if not mask_head.class_agnostic:\n        mask_pred = mask_pred[range(N), labels][:, None]\n\n    for inds in chunks:\n        masks_chunk, spatial_inds = _do_paste_mask(\n            mask_pred[inds], bboxes[inds], img_h, img_w, skip_empty=device.type == \"cpu\"\n        )\n\n        if threshold >= 0:\n            masks_chunk = (masks_chunk >= threshold).to(dtype=torch.bool)\n        else:\n            masks_chunk = (masks_chunk * 255).to(dtype=torch.uint8)\n\n        im_mask[(inds,) + spatial_inds] = masks_chunk\n\n    for i in range(N):\n        cls_segms[labels[i]].append(im_mask[i].detach().cpu().numpy())\n\n    if return_per_class:\n        return cls_segms\n    else:\n        return im_mask\n\n\ndef get_rpn_boxes_single(\n    rpn_head,\n    cls_scores,\n    bbox_preds,\n    mlvl_anchors,\n    img_shape,\n    scale_factor,\n    cfg,\n):\n    \"\"\"\n    Modified from mmdet/models/dense_heads/rpn_head.py\n\n    Transform outputs for a single batch item into bbox predictions.\n\n        Args:\n        cls_scores (list[Tensor]): Box scores of all scale level\n            each item has shape (num_anchors * num_classes, H, W).\n        bbox_preds (list[Tensor]): Box energies / deltas of all\n            scale level, each item has shape (num_anchors * 4, H, W).\n        mlvl_anchors (list[Tensor]): Anchors of all scale level\n            each item has shape (num_total_anchors, 4).\n        img_shape (tuple[int]): Shape of the input image,\n            (height, width, 3).\n        scale_factor (ndarray): Scale factor of the image arrange as\n            (w_scale, h_scale, w_scale, h_scale).\n        cfg (mmcv.Config): Test / postprocessing configuration,\n            if None, test_cfg would be used.\n    Returns:\n        Tensor: Labeled boxes in shape (n, 5), where the first 4 columns\n            are bounding box positions (tl_x, tl_y, br_x, br_y) and the\n            5-th column is a score between 0 and 1.\n    \"\"\"\n    cfg = copy.deepcopy(cfg)\n    # bboxes from different level should be independent during NMS,\n    # level_ids are used as labels for batched NMS to separate them\n    level_ids = []\n    mlvl_scores = []\n    mlvl_bbox_preds = []\n    mlvl_valid_anchors = []\n    for idx in range(len(cls_scores)):\n        rpn_cls_score = cls_scores[idx]\n        rpn_bbox_pred = bbox_preds[idx]\n        assert rpn_cls_score.size()[-2:] == rpn_bbox_pred.size()[-2:]\n        rpn_cls_score = rpn_cls_score.permute(1, 2, 0)\n        if rpn_head.use_sigmoid_cls:\n            rpn_cls_score = rpn_cls_score.reshape(-1)\n            scores = rpn_cls_score.sigmoid()\n        else:\n            rpn_cls_score = rpn_cls_score.reshape(-1, 2)\n            # We set FG labels to [0, num_class-1] and BG label to\n            # num_class in RPN head since mmdet v2.5, which is unified to\n            # be consistent with other head since mmdet v2.0. In mmdet v2.0\n            # to v2.4 we keep BG label as 0 and FG label as 1 in rpn head.\n            scores = rpn_cls_score.softmax(dim=1)[:, 0]\n        rpn_bbox_pred = rpn_bbox_pred.permute(1, 2, 0).reshape(-1, 4)\n        anchors = mlvl_anchors[idx]\n        if cfg.nms_pre > 0 and scores.shape[0] > cfg.nms_pre:\n            # sort is faster than topk\n            # _, topk_inds = scores.topk(cfg.nms_pre)\n            ranked_scores, rank_inds = scores.sort(descending=True)\n            topk_inds = rank_inds[: cfg.nms_pre]\n            scores = ranked_scores[: cfg.nms_pre]\n            rpn_bbox_pred = rpn_bbox_pred[topk_inds, :]\n            anchors = anchors[topk_inds, :]\n        mlvl_scores.append(scores)\n        mlvl_bbox_preds.append(rpn_bbox_pred)\n        mlvl_valid_anchors.append(anchors)\n        level_ids.append(scores.new_full((scores.size(0),), idx, dtype=torch.long))\n\n    scores = torch.cat(mlvl_scores)\n    anchors = torch.cat(mlvl_valid_anchors)\n    rpn_bbox_pred = torch.cat(mlvl_bbox_preds)\n    proposals = rpn_head.bbox_coder.decode(anchors, rpn_bbox_pred, max_shape=img_shape)\n    ids = torch.cat(level_ids)\n\n    if cfg.min_bbox_size >= 0:\n        w = proposals[:, 2] - proposals[:, 0]\n        h = proposals[:, 3] - proposals[:, 1]\n        valid_mask = (w > cfg.min_bbox_size) & (h > cfg.min_bbox_size)\n        if not valid_mask.all():\n            proposals = proposals[valid_mask]\n            scores = scores[valid_mask]\n            ids = ids[valid_mask]\n    if proposals.numel() > 0:\n        dets, keep = batched_nms(proposals, scores, ids, cfg.nms)\n    else:\n        return proposals.new_zeros(0, 5)\n\n    # print(dets.size())\n    dets = dets[dets[:, 4] > cfg.score_thr]\n    # print(dets.size())\n    dets = dets[:cfg.max_per_img]\n    # print(dets.size())\n\n    return dets\n\n\ndef get_rpn_boxes(rpn_head, cls_scores, bbox_preds, img_metas, cfg):\n    \"\"\"\n    TODO\n    Modified from mmdet/models/dense_heads/rpn_head.py\n\n    Args:\n        rpn_head ([type]): [description]\n        cls_scores ([type]): [description]\n        bbox_preds ([type]): [description]\n        img_metas ([type]): [description]\n        cfg ([type], optional): [description]. Defaults to None.\n        rescale (bool, optional): [description]. Defaults to False.\n        with_nms (bool, optional): [description]. Defaults to True.\n\n    Returns:\n        [type]: [description]\n    \"\"\"\n    assert len(cls_scores) == len(bbox_preds)\n    num_levels = len(cls_scores)\n    device = cls_scores[0].device\n    featmap_sizes = [cls_scores[i].shape[-2:] for i in range(num_levels)]\n    mlvl_anchors = rpn_head.anchor_generator.grid_anchors(featmap_sizes, device=device)\n\n    result_list = []\n    for img_id in range(len(img_metas)):\n        cls_score_list = [cls_scores[i][img_id].detach() for i in range(num_levels)]\n        bbox_pred_list = [bbox_preds[i][img_id].detach() for i in range(num_levels)]\n        img_shape = img_metas[img_id][\"img_shape\"]\n        scale_factor = img_metas[img_id][\"scale_factor\"]\n        proposals = get_rpn_boxes_single(\n            rpn_head,\n            cls_score_list,\n            bbox_pred_list,\n            mlvl_anchors,\n            img_shape,\n            scale_factor,\n            cfg,\n        )\n        result_list.append(proposals)\n\n    return result_list\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:49.639035Z","iopub.execute_input":"2021-12-30T16:48:49.63958Z","iopub.status.idle":"2021-12-30T16:48:49.685119Z","shell.execute_reply.started":"2021-12-30T16:48:49.639533Z","shell.execute_reply":"2021-12-30T16:48:49.683685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Wrappers","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.functional as F\n\n\ndef get_wrappers(names):\n    wrappers = []\n    for name in names:\n        if \"rcnn\" in name:\n            wrappers.append(RCNNEnsemble)\n        elif \"cascade\" in name:\n            wrappers.append(CascadeEnsemble)\n        elif \"htc\" in name:\n            wrappers.append(HTCEnsemble)\n        else:\n            raise NotImplementedError\n\n    return wrappers\n\n\nclass RCNNEnsemble:\n    @staticmethod\n    def get_boxes(model, x, rois, img_shape, scale_factor, img_meta, num_classes):\n        bbox_results = model.roi_head._bbox_forward(x, rois)\n        bboxes, scores = model.roi_head.bbox_head.get_bboxes(\n            rois,\n            bbox_results[\"cls_score\"],\n            bbox_results[\"bbox_pred\"],\n            img_shape,\n            scale_factor,\n            rescale=False,\n            cfg=None,\n        )\n\n        # Keep only desired classes\n        scores = scores[:, :num_classes]\n\n        # Keep box corresponding to most confident class\n        _, det_labels = torch.max(scores, 1)\n\n        bboxes = bboxes.view(bboxes.size(0), -1, 4)\n        bboxes = torch.stack([bboxes[i, c] for i, c in enumerate(det_labels)])\n\n        return bboxes, scores\n\n    @staticmethod\n    def get_masks(model, x, mask_rois, num_classes):\n        masks = model.roi_head._mask_forward(x, mask_rois)[\"mask_pred\"]\n        masks = masks.sigmoid().cpu().numpy()[:, :num_classes]\n\n        return masks\n\n\nclass CascadeEnsemble:\n    @staticmethod\n    def get_boxes(model, x, rois, img_shape, scale_factor, img_meta, num_classes):\n        # https://github.com/open-mmlab/mmdetection/blob/bde7b4b7eea9dd6ee91a486c6996b2d68662366d/mmdet/models/roi_heads/test_mixins.py#L139\n\n        ms_scores = []\n        for i in range(model.roi_head.num_stages):\n            bbox_results = model.roi_head._bbox_forward(i, x, rois)\n            ms_scores.append(bbox_results[\"cls_score\"])\n\n            if i < model.roi_head.num_stages - 1:\n                cls_score = bbox_results[\"cls_score\"]\n                if model.roi_head.bbox_head[i].custom_activation:\n                    cls_score = model.roi_head.bbox_head[i].loss_cls.get_activation(\n                        cls_score\n                    )\n                bbox_label = cls_score[:, :-1].argmax(dim=1)\n                rois = model.roi_head.bbox_head[i].regress_by_class(\n                    rois, bbox_label, bbox_results[\"bbox_pred\"], img_meta[0]\n                )\n\n        cls_score = sum(ms_scores) / float(len(ms_scores))\n        bboxes, scores = model.roi_head.bbox_head[-1].get_bboxes(\n            rois,\n            cls_score,\n            bbox_results[\"bbox_pred\"],\n            img_shape,\n            scale_factor,\n            rescale=False,\n            cfg=None,\n        )\n\n        scores = scores[:, :num_classes]\n\n        return bboxes, scores\n\n    @staticmethod\n    def get_masks(model, x, mask_rois, num_classes):\n        masks = []\n        for i in range(model.roi_head.num_stages):\n            mask = model.roi_head._mask_forward(i, x, mask_rois)['mask_pred']\n            mask = mask.sigmoid()[:, :num_classes]\n            masks.append(mask)\n        masks = torch.stack(masks)\n        masks = masks.mean(0).cpu().numpy()\n\n        return masks\n\n\nclass HTCEnsemble:\n    @staticmethod\n    def get_boxes(model, x, rois, img_shape, scale_factor, img_meta, num_classes):\n        # https://github.com/open-mmlab/mmdetection/blob/a7a16afbf2a4bdb4d023094da73d325cb864838b/mmdet/models/roi_heads/htc_roi_head.py#L505\n        semantic = model.roi_head.semantic_head(x)[1]\n\n        ms_scores = []\n        for i in range(model.roi_head.num_stages):\n            bbox_head = model.roi_head.bbox_head[i]\n            bbox_results = model.roi_head._bbox_forward(\n                i, x, rois, semantic_feat=semantic\n            )\n            ms_scores.append(bbox_results[\"cls_score\"])\n\n            if i < model.roi_head.num_stages - 1:\n                bbox_label = bbox_results[\"cls_score\"].argmax(dim=1)\n                rois = bbox_head.regress_by_class(\n                    rois, bbox_label, bbox_results[\"bbox_pred\"], img_meta[0]\n                )\n\n        cls_score = sum(ms_scores) / float(len(ms_scores))\n        bboxes, scores = model.roi_head.bbox_head[-1].get_bboxes(\n            rois,\n            cls_score,\n            bbox_results[\"bbox_pred\"],\n            img_shape,\n            scale_factor,\n            rescale=False,\n            cfg=None,\n        )\n\n        scores = scores[:, :num_classes]\n\n        return bboxes, scores\n\n    @staticmethod\n    def get_masks(model, x, mask_rois, num_classes):\n        # https://github.com/open-mmlab/mmdetection/blob/a7a16afbf2a4bdb4d023094da73d325cb864838b/mmdet/models/roi_heads/htc_roi_head.py#L592\n\n        mask_feats = model.roi_head.mask_roi_extractor[-1](\n            x[: len(model.roi_head.mask_roi_extractor[-1].featmap_strides)], mask_rois\n        )\n\n        # Semantic feats\n        semantic = model.roi_head.semantic_head(x)[1]\n        mask_semantic_feat = model.roi_head.semantic_roi_extractor([semantic], mask_rois)\n        if mask_semantic_feat.shape[-2:] != mask_feats.shape[-2:]:\n            mask_semantic_feat = F.adaptive_avg_pool2d(\n                mask_semantic_feat, mask_feats.shape[-2:]\n            )\n        mask_feats += mask_semantic_feat\n\n        last_feat = None\n        masks = []\n        for i in range(model.roi_head.num_stages):\n            mask_head = model.roi_head.mask_head[i]\n            if model.roi_head.mask_info_flow:\n                mask_pred, last_feat = mask_head(mask_feats, last_feat)\n            else:\n                mask_pred = mask_head(mask_feats)\n            masks.append(mask_pred.sigmoid()[:, :num_classes])\n\n        masks = torch.stack(masks)\n        masks = masks.mean(0).cpu().numpy()\n\n        return masks\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:49.686881Z","iopub.execute_input":"2021-12-30T16:48:49.68767Z","iopub.status.idle":"2021-12-30T16:48:49.723375Z","shell.execute_reply.started":"2021-12-30T16:48:49.687624Z","shell.execute_reply":"2021-12-30T16:48:49.722245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ensemble Model","metadata":{}},{"cell_type":"code","source":"import mmcv\nimport torch\nimport numpy as np\n\nfrom torch import nn\nfrom mmdet.core import bbox_mapping\nfrom mmdet.core import bbox2roi, merge_aug_masks\nfrom mmdet.models.detectors import BaseDetector\n\n\nclass EnsembleModel(BaseDetector):\n    \"\"\"\n    Wrapper to ensemble models.\n    \"\"\"\n    def __init__(\n        self,\n        models,\n        config,\n        names=[],\n    ):\n        \"\"\"\n        Constructor.\n\n        Args:\n            models (list of mmdet MMDataParallel): Models to ensemble.\n            config (dict): Ensemble config.\n            names (list, optional): Model names. Defaults to [].\n        \"\"\"\n        super().__init__()\n        self.models = nn.ModuleList([model.module for model in models])\n        self.config = config\n        self.names = names\n\n        self.wrappers = get_wrappers(self.names)\n        self.get_configs()\n\n    def get_configs(self):\n        \"\"\"\n        Creates the rpn and rcnn configs from the config dict.\n        \"\"\"\n        self.rpn_cfgs, self.rcnn_cfgs = [], []\n\n        for i in range(3):\n            rpn_cfg = mmcv.Config(\n                dict(\n                    score_thr=self.config['rpn_score_threshold'][i],\n                    nms_pre=self.config['rpn_nms_pre'][i],\n                    max_per_img=self.config['rpn_max_per_img'][i],\n                    nms=dict(type=\"nms\", iou_threshold=self.config['rpn_iou_threshold'][i]),\n                    min_bbox_size=0,\n                )\n            )\n            rcnn_cfg = mmcv.Config(\n                dict(\n                    score_thr=self.config['rcnn_score_threshold'][i],\n                    nms=dict(type=\"nms\", iou_threshold=self.config['rcnn_iou_threshold'][i]),\n                    mask_thr_binary=-1,\n                )\n            )\n            self.rpn_cfgs.append(rpn_cfg)\n            self.rcnn_cfgs.append(rcnn_cfg)\n\n    def extract_feat(self, img, img_metas, **kwargs):\n        \"\"\"\n        Extract features function. Not used but required by MMDet.\n\n        Args:\n            imgs (list of torch tensors [n x C x H x W]): Input image.\n            img_metas (list of dicts [n]): List of MMDet image metadata.\n        \"\"\"\n        pass\n\n    def simple_test(self, img, img_metas, **kwargs):\n        \"\"\"\n        Single image test function. Not used but required by MMDet.\n\n        Args:\n            imgs (list of torch tensors [1 x C x H x W]): Input image.\n            img_metas (list of dicts [1]): List of MMDet image metadata.\n        \"\"\"\n        pass\n\n    def forward(self, img, img_metas, **kwargs):\n        \"\"\"\n        Forward function.\n\n        Args:\n            imgs (list of torch tensors [n_tta x C x H x W]): Input image.\n            img_metas (list of dicts [n_tta]): List of MMDet image metadata.\n\n        Returns:\n            [type]: [description]\n        \"\"\"\n        return self.aug_test(img, img_metas, **kwargs)\n\n    def get_proposals(self, imgs, img_metas):\n        \"\"\"\n        Gets proposals, doesn't use TTA.\n\n        Args:\n            imgs (list of torch tensors [n_tta x C x H x W]): Input image.\n            img_metas (list of dicts [n_tta]): List of MMDet image metadata.\n\n        Returns:\n            list of torch tensors [1 x 5]: Proposals.\n            int: Cell type.\n        \"\"\"\n        aug_bboxes, aug_scores = [], []\n        for i, model in enumerate(self.models):\n            for x, img_meta in zip(model.extract_feats(imgs), img_metas):\n                cls_score, bbox_pred = model.rpn_head(x)\n\n                aug_bboxes.append(bbox_pred)\n                aug_scores.append(cls_score)\n\n                break  # no tta\n\n        merged_bboxes, merged_scores = [], []\n        level_counts = []\n\n        for lvl in range(len(aug_bboxes[0])):\n            merged_bboxes_lvl = torch.stack([bboxes[lvl] for bboxes in aug_bboxes]).mean(dim=0)\n            merged_scores_lvl = torch.stack([scores[lvl] for scores in aug_scores]).mean(dim=0)\n\n            merged_bboxes.append(merged_bboxes_lvl)\n            merged_scores.append(merged_scores_lvl)\n\n            rpn_scores_lvl, rpn_labels_lvl = torch.max(\n                merged_scores_lvl.sigmoid().flatten(start_dim=2)[0], 0\n            )\n            level_counts.append(rpn_labels_lvl[rpn_scores_lvl > 0.7].size(0))\n\n        if np.sum(level_counts[-2:]) > 10:  # astro\n            cell_type = 1\n        elif np.sum(level_counts) < 4500 and level_counts[1] < 750:  # cort\n            cell_type = 2\n        else:  # shsy5y\n            cell_type = 0\n\n        proposal_list = get_rpn_boxes(\n            self.models[0].rpn_head,\n            merged_scores,\n            merged_bboxes,\n            img_metas[0],\n            self.rpn_cfgs[cell_type]\n        )\n\n        return proposal_list, cell_type\n\n    def get_proposals_tta(self, imgs, img_metas):\n        \"\"\"\n        TODO\n        This doesn't work yet.\n\n        Args:\n            imgs ([type]): [description]\n            img_metas ([type]): [description]\n\n        Returns:\n            [type]: [description]\n        \"\"\"\n        raise NotImplementedError\n\n    def get_bboxes(self, imgs, img_metas, proposal_list, rcnn_cfg):\n        \"\"\"\n        Gets rcnn boxes. Adapted from :\n        https://github.com/open-mmlab/mmdetection/blob/bde7b4b7eea9dd6ee91a486c6996b2d68662366d/mmdet/models/roi_heads/test_mixins.py#L139\n        All TTAs are used.\n\n        Args:\n            imgs (list of torch tensors [n_tta x C x H x W]): Input images.\n            img_metas (list of dicts [n_tta]): List of MMDet image metadata.\n            proposal_list ([1 x N]): Proposals.\n\n        Returns:\n            torch tensor [m x 6]: Kept boxes, confidences & labels.\n            list of torch tensors: Augmented boxes before merging.\n        \"\"\"\n        aug_bboxes, aug_scores, aug_img_metas = [], [], []\n\n        for wrapper, model in zip(self.wrappers, self.models):\n            for x, img_meta in zip(model.extract_feats(imgs), img_metas):\n                img_shape = img_meta[0][\"img_shape\"]\n                scale_factor = img_meta[0][\"scale_factor\"]\n                flip = img_meta[0][\"flip\"]\n                flip_direction = img_meta[0][\"flip_direction\"]\n\n                proposals = bbox_mapping(\n                    proposal_list[0][:, :4],\n                    img_shape,\n                    scale_factor,\n                    flip,\n                    flip_direction,\n                )\n                rois = bbox2roi([proposals])\n\n                bboxes, scores = wrapper.get_boxes(\n                    model, x, rois, img_shape, scale_factor, img_meta, self.config['num_classes']\n                )\n\n                aug_bboxes.append(bboxes)\n                aug_scores.append(scores)\n                aug_img_metas.append(img_meta)\n\n        merged_bboxes, merged_scores = merge_aug_bboxes(\n            aug_bboxes, aug_scores, aug_img_metas\n        )\n\n        if self.config['bbox_nms']:\n            det_bboxes, det_labels = single_class_boxes_nms(\n                merged_bboxes,\n                merged_scores,\n                iou_threshold=rcnn_cfg.nms.iou_threshold,\n            )\n            det_bboxes = torch.cat([det_bboxes, det_labels.unsqueeze(-1)], -1)\n\n        else:\n            det_scores, det_labels = torch.max(merged_scores, 1)\n            det_bboxes = torch.cat(\n                [merged_bboxes, det_scores.unsqueeze(1), det_labels.unsqueeze(1)], 1\n            )\n\n            _, order = det_scores.sort(0, descending=True)\n            det_bboxes = det_bboxes[order]\n\n        det_bboxes = det_bboxes[det_bboxes[:, 4] > rcnn_cfg.score_thr]\n\n        return det_bboxes, torch.cat([merged_bboxes, merged_scores], 1)\n\n    def get_masks(self, imgs, img_metas, det_bboxes, det_labels):\n        \"\"\"\n        Gets rcnn boxes. Adapted from :\n        https://github.com/open-mmlab/mmdetection/blob/bde7b4b7eea9dd6ee91a486c6996b2d68662366d/mmdet/models/roi_heads/test_mixins.py#L282\n\n        Only hflip TTA is used.\n\n        Args:\n            imgs (list of torch tensors [n_tta x C x H x W]): Input images.\n            img_metas (list of dicts [n_tta]): List of MMDet image metadata.\n            det_bboxes (torch tensor [m x 5): Boxes & confidences.\n            det_labels (torch tensor [m]): Labels.\n\n        Returns:\n            torch tensor [m x H x W]: Masks.\n            list of torch tensors: Augmented masks before merging.\n        \"\"\"\n        aug_masks, aug_img_metas = [], []\n\n        for wrapper, model in zip(self.wrappers, self.models):\n            for x, img_meta in zip(model.extract_feats(imgs), img_metas):\n                img_shape = img_meta[0][\"img_shape\"]\n                scale_factor = img_meta[0][\"scale_factor\"]\n                flip = img_meta[0][\"flip\"]\n                flip_direction = img_meta[0][\"flip_direction\"]\n                \n                if flip_direction not in self.config['ttas_masks']:\n                    continue\n\n                _bboxes = bbox_mapping(\n                    det_bboxes[:, :4], img_shape, scale_factor, flip, flip_direction\n                )\n                mask_rois = bbox2roi([_bboxes])\n                \n                # Seems to help\n                if self.config['delta']:\n                    if flip_direction in ['vertical', 'diagonal']:\n                        mask_rois[:, 2] = torch.clamp(mask_rois[:, 2] - self.config['delta'], 0, img_shape[0])\n                        mask_rois[:, 4] = torch.clamp(mask_rois[:, 4] - self.config['delta'], 0, img_shape[0])\n                    if flip_direction in ['horizontal', 'diagonal']:\n                        mask_rois[:, 1] = torch.clamp(mask_rois[:, 1] - self.config['delta'], 0, img_shape[1])\n                        mask_rois[:, 3] = torch.clamp(mask_rois[:, 3] - self.config['delta'], 0, img_shape[1])\n\n                masks = wrapper.get_masks(model, x, mask_rois, self.config['num_classes'])\n\n                aug_masks.append(masks)\n                aug_img_metas.append(img_meta)\n\n        merged_masks = merge_aug_masks(aug_masks, aug_img_metas, None)\n\n        mask_head = (\n            self.models[0].roi_head.mask_head if \"rcnn\" in self.names[0]\n            else self.models[0].roi_head.mask_head[-1]\n        )\n\n        masks = get_seg_masks(\n            mask_head,\n            merged_masks,\n            det_bboxes,\n            det_labels,\n            self.rcnn_cfgs[0],\n            img_metas[0][0][\"ori_shape\"],\n            scale_factor=det_bboxes.new_ones(4),\n            rescale=False,\n            return_per_class=False,\n        )\n\n        return masks, aug_masks\n\n    def aug_test(self, imgs, img_metas, return_everything=False, **kwargs):\n        \"\"\"\n        Augmented test function. Adapted from :\n        https://github.com/open-mmlab/mmdetection/blob/bde7b4b7eea9dd6ee91a486c6996b2d68662366d/mmdet/models/roi_heads/standard_roi_head.py#L268\n\n        Args:\n            imgs (list of torch tensors [n_tta x C x H x W]): Input images.\n            img_metas (list of dicts [n_tta]): List of MMDet image metadata.\n            return_everything (bool, optional): Whether to return more stuff. Defaults to False.\n\n        Returns:\n            torch tensor [m x 6]: Kept boxes, confidences & labels.\n            torch tensor [m x H x W]: Masks.\n            list of torch tensors [1 x 5]: Proposals.\n            list of torch tensors: Augmented boxes before merging.\n            list of torch tensors: Augmented masks before merging.\n        \"\"\"\n        proposal_list, cell_type = self.get_proposals(imgs, img_metas)\n\n        bboxes, aug_bboxes = self.get_bboxes(\n            imgs, img_metas, proposal_list, self.rcnn_cfgs[cell_type]\n        )\n\n        assert self.models[0].with_mask\n\n        if bboxes.shape[0] == 0:\n            return bboxes, None\n\n        masks, aug_masks = self.get_masks(\n            imgs, img_metas, bboxes[:, :5], bboxes[:, 5].long()\n        )\n\n        if return_everything:\n            all_stuff = (proposal_list, aug_bboxes, bboxes, aug_masks, masks)\n            return (bboxes, masks), all_stuff\n\n        return (bboxes, masks)\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:49.727511Z","iopub.execute_input":"2021-12-30T16:48:49.727977Z","iopub.status.idle":"2021-12-30T16:48:49.780409Z","shell.execute_reply.started":"2021-12-30T16:48:49.727945Z","shell.execute_reply":"2021-12-30T16:48:49.779118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Post-processing","metadata":{}},{"cell_type":"markdown","source":"### Overlapping","metadata":{}},{"cell_type":"code","source":"import pycocotools\n\n\ndef remove_overlap_naive(masks, ious=None):\n    if ious is None:\n        rles = [pycocotools.mask.encode(np.asarray(m, order='F')) for m in masks]\n        ious = pycocotools.mask.iou(rles, rles, [0] * len(rles))\n\n    for i in range(len(ious)):\n        ious[i, i] = 0\n\n    to_process = np.where(ious.sum(0) > 0)[0]\n\n    if not len(to_process):\n        return masks\n\n    masks = torch.from_numpy(masks).cuda()\n    overlapping_masks = masks[to_process]\n\n    for idx, i in enumerate(to_process):\n        if idx == 0:\n            continue\n        others = overlapping_masks[:idx].max(0)[0]\n        masks[i] *= ~others\n\n    return masks.cpu().numpy()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:49.782019Z","iopub.execute_input":"2021-12-30T16:48:49.782646Z","iopub.status.idle":"2021-12-30T16:48:49.796944Z","shell.execute_reply.started":"2021-12-30T16:48:49.782603Z","shell.execute_reply":"2021-12-30T16:48:49.795931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### NMS","metadata":{}},{"cell_type":"code","source":"def mask_nms(masks, boxes, threshold=0.5):\n    \"\"\"\n    NMS with masks.\n    Removes more masks than the tweaking fct.\n\n    Args:\n        masks ([type]): [description]\n        boxes ([type]): [description]\n        threshold (float, optional): [description]. Defaults to 0.5.\n\n    Returns:\n        [type]: [description]\n    \"\"\"\n    # assert list(np.argsort(boxes[:, 4])[::-1]) == list(range(len(boxes)))\n\n    order = np.argsort(boxes[:, 4])[::-1]\n    masks = masks[order]\n    boxes = boxes[order]\n\n    rle_pred = [pycocotools.mask.encode(np.asarray(m, order='F')) for m in masks]\n    ious = pycocotools.mask.iou(rle_pred, rle_pred, [0] * len(rle_pred))\n\n    picks = []\n    idxs = list(range(len(ious)))\n    # removed = []\n\n    while len(idxs) > 0:\n        idx = idxs[0]\n        overlapping = np.where(ious[idx] > threshold)[0]\n\n        # removed += [v for v in overlapping if v > idx]\n\n        if len(overlapping):\n            picks.append(idx)\n            idxs = [i for i in idxs if i not in overlapping]\n        else:\n            idxs = idxs[1:]\n\n    masks = masks[picks]\n    boxes = boxes[picks]\n    return masks, boxes, picks","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:49.79864Z","iopub.execute_input":"2021-12-30T16:48:49.79932Z","iopub.status.idle":"2021-12-30T16:48:49.813778Z","shell.execute_reply.started":"2021-12-30T16:48:49.799273Z","shell.execute_reply":"2021-12-30T16:48:49.812583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Remove Small Masks","metadata":{}},{"cell_type":"code","source":"def remove_small_masks(masks, boxes, min_size=0):\n    if min_size == 0:\n        return masks, boxes\n\n    sizes = masks.sum(-1).sum(-1)\n    to_keep = sizes > min_size\n\n    if to_keep.min() == 1:\n        return masks, boxes\n\n    smallest = sizes.min()\n    to_keep = sizes > smallest\n\n    return masks[to_keep], boxes[to_keep]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:49.81562Z","iopub.execute_input":"2021-12-30T16:48:49.816174Z","iopub.status.idle":"2021-12-30T16:48:49.826435Z","shell.execute_reply.started":"2021-12-30T16:48:49.816128Z","shell.execute_reply":"2021-12-30T16:48:49.825027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Corrupt","metadata":{}},{"cell_type":"code","source":"import cv2\n\ndef degrade_mask(mask):\n    cont, hier = cv2.findContours(mask.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n\n    img_cont = np.zeros((mask.shape[0], mask.shape[1], 3), dtype=np.uint8)\n    img_cont = cv2.drawContours(img_cont, cont, -1, (255, 255, 255), 1)\n    img_cont = img_cont[:, :, 0]\n\n    conv_mask = np.zeros((mask.shape[0], mask.shape[1], 3), dtype=np.uint8)\n\n    for c in cont:\n        conv_mask = cv2.fillConvexPoly(conv_mask, points=c, color=(1, 1, 1))\n    conv_mask = conv_mask[:, :, 0].astype(mask.dtype)\n\n    return conv_mask, img_cont","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:49.828682Z","iopub.execute_input":"2021-12-30T16:48:49.829132Z","iopub.status.idle":"2021-12-30T16:48:49.840461Z","shell.execute_reply.started":"2021-12-30T16:48:49.829087Z","shell.execute_reply":"2021-12-30T16:48:49.839419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Overall fct","metadata":{}},{"cell_type":"code","source":"def process_masks(boxes, masks, thresholds_mask, thresholds_nms, thresholds_conf, min_sizes, remove_overlap=True, corrupt=False):\n    # Cell type\n    cell = np.argmax(np.bincount(boxes[:, 5].astype(int)))\n\n    # Thresholds\n    thresh_mask = (\n        thresholds_mask if isinstance(thresholds_mask, (float, int))\n        else thresholds_mask[cell]\n    )\n    thresh_nms = (\n        thresholds_nms if isinstance(thresholds_nms, (float, int))\n        else thresholds_nms[cell]\n    )\n    thresh_conf = (\n        thresholds_conf if isinstance(thresholds_conf, (float, int))\n        else thresholds_conf[cell]\n    )\n    min_size = (\n        min_sizes if isinstance(min_sizes, (float, int))\n        else min_sizes[cell]\n    )\n\n    # Binarize\n    masks = masks > (thresh_mask * 255)\n\n    # Sort by decreasing conf\n    order = np.argsort(boxes[:, 4])[::-1]\n    masks = masks[order]\n    boxes = boxes[order]\n\n    # Remove low confidence\n    last = (\n        np.argmax(boxes[:, 4] < thresh_conf) if np.min(boxes[:, 4]) < thresh_conf\n        else len(boxes)\n    )\n    masks = masks[:last]\n    boxes = boxes[:last]\n\n    # NMS\n    if thresh_nms > 0:\n        masks, boxes, _ = mask_nms(masks, boxes, thresh_nms)\n        \n    # Remove small masks\n    if min_size:\n        masks, boxes = remove_small_masks(masks, boxes, min_size=min_size)\n       \n    # Corrupt\n    if corrupt and cell == 1:  # astro\n        masks = np.array([degrade_mask(mask)[0] for mask in masks])\n\n    # Remove overlap\n    if remove_overlap:\n        masks = remove_overlap_naive(masks)\n\n    return masks, boxes, cell","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:49.842408Z","iopub.execute_input":"2021-12-30T16:48:49.842954Z","iopub.status.idle":"2021-12-30T16:48:49.859306Z","shell.execute_reply.started":"2021-12-30T16:48:49.842909Z","shell.execute_reply":"2021-12-30T16:48:49.85821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict","metadata":{}},{"cell_type":"code","source":"import torch\n\n\ndef predict_and_process(dataset, model, thresholds_mask, thresholds_nms, thresholds_conf, min_sizes, device=\"cuda\"):\n    \"\"\"\n    Performs inference on an image.\n    TODO\n\n    Args:\n        dataset (InferenceDataset): Inference dataset.\n        model (torch model): Segmentation model.\n        batch_size (int, optional): Batch size. Defaults to 32.\n        tta (bool, optional): Whether to apply tta. Defaults to False.\n\n    Returns:\n        torch tensor [H x W]: Prediction on the image.\n    \"\"\"\n    loader = define_loaders(None, dataset, val_bs=1, num_workers=0)[1]\n\n    rles, cell_types = [], []\n\n    model.eval()\n    with torch.no_grad():\n        for batch in loader:\n            boxes, masks = model(**batch, return_loss=False, rescale=True)\n            boxes = boxes.cpu().numpy()\n            masks = masks.cpu().numpy()\n            \n            masks, boxes, cell_type = process_masks(\n                boxes, masks, thresholds_mask, thresholds_nms, thresholds_conf, min_sizes, remove_overlap=True, corrupt=True\n            )\n\n            rles.append([rle_encoding(mask) for mask in masks])\n            cell_types.append(cell_type)\n\n    return rles, cell_types\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:49.861255Z","iopub.execute_input":"2021-12-30T16:48:49.862182Z","iopub.status.idle":"2021-12-30T16:48:49.874823Z","shell.execute_reply.started":"2021-12-30T16:48:49.862135Z","shell.execute_reply":"2021-12-30T16:48:49.873792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"def inference(df, configs, weights, ensemble_config, thresholds_mask, thresholds_nms, thresholds_conf, min_sizes):\n\n    pipelines = define_pipelines(configs[0].data_config)\n\n    models, names = [], []\n    for config, fold_weights in zip(configs, weights):\n        for weight in fold_weights:\n            model = define_model(\n                config.model_config, encoder=config.encoder, verbose=0\n            )\n            model = load_model_weights(model, weight)\n            models.append(model)\n            names.append(weight.split('/')[-1])\n\n    dataset = SartoriusInferenceDataset(df, transforms=pipelines['test_tta'] if ensemble_config[\"use_tta\"] else pipelines['test'])\n    \n    model = MMDataParallel(\n        EnsembleModel(\n            models,\n            ensemble_config,\n            names=names,\n        )\n    )\n\n    rles, cell_types = predict_and_process(\n        dataset, model, thresholds_mask, thresholds_nms, thresholds_conf, min_sizes, device=config.device\n    )\n\n    return rles, cell_types\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:49.879089Z","iopub.execute_input":"2021-12-30T16:48:49.879527Z","iopub.status.idle":"2021-12-30T16:48:49.890671Z","shell.execute_reply.started":"2021-12-30T16:48:49.879481Z","shell.execute_reply":"2021-12-30T16:48:49.889376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge rles","metadata":{}},{"cell_type":"code","source":"def merge_rles(df_cort, df_astro, df_shsy5y):\n    df = df_cort.copy().merge(df_astro[['id', 'rle_astro']], on=\"id\", how=\"left\")\n    df = df.merge(df_shsy5y[['id', 'rle_shsy5y']], on=\"id\", how=\"left\")\n    \n    rles = []\n    for i in range(len(df)):\n        if df[\"cell_type\"][i] == 0:\n            rle = df['rle_shsy5y'][i]\n        elif df[\"cell_type\"][i] == 1:\n            rle = df['rle_astro'][i]\n        else:\n            rle = df['rle_cort'][i]\n            \n        assert isinstance(rle, list)\n        rles.append(rle)\n    \n    return df, rles","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-30T16:48:49.892592Z","iopub.execute_input":"2021-12-30T16:48:49.893577Z","iopub.status.idle":"2021-12-30T16:48:49.906251Z","shell.execute_reply.started":"2021-12-30T16:48:49.893546Z","shell.execute_reply":"2021-12-30T16:48:49.905179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main","metadata":{}},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(DATA_PATH + \"sample_submission.csv\")\ndf['img_path'] = DATA_PATH + \"test/\" + df['id'] + \".png\"","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:48:49.909281Z","iopub.execute_input":"2021-12-30T16:48:49.910091Z","iopub.status.idle":"2021-12-30T16:48:49.948838Z","shell.execute_reply.started":"2021-12-30T16:48:49.910046Z","shell.execute_reply":"2021-12-30T16:48:49.94791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SUCCESS = [False, False, False]","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:48:49.950571Z","iopub.execute_input":"2021-12-30T16:48:49.950942Z","iopub.status.idle":"2021-12-30T16:48:49.956207Z","shell.execute_reply.started":"2021-12-30T16:48:49.95087Z","shell.execute_reply":"2021-12-30T16:48:49.954934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ENSEMBLE_CONFIG = {\n    \"use_tta\": True,\n    \"use_tta_masks\": True,\n    \"num_classes\": 3,\n\n    \"rpn_nms_pre\": [5000, 2000, 1000],\n    \"rpn_iou_threshold\": [0.7, 0.75, 0.6],\n    \"rpn_score_threshold\": [0.9, 0.9, 0.95],\n    \"rpn_max_per_img\": [None, None, None],\n\n    \"bbox_nms\": True,\n    \"rcnn_iou_threshold\": [0.7, 0.9, 0.6],\n    \"rcnn_score_threshold\": [0.2, 0.25, 0.5],\n    \n    \"ttas_masks\": [None, \"horizontal\", \"vertical\", \"diagonal\"],\n    \"delta\": 0.5,\n}\n\n# # Best LB :\n# MIN_SIZES = [0, 0, 0]\n# Best CV :\nMIN_SIZES = [0, 150, 75]","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:48:49.958102Z","iopub.execute_input":"2021-12-30T16:48:49.958737Z","iopub.status.idle":"2021-12-30T16:48:49.96895Z","shell.execute_reply.started":"2021-12-30T16:48:49.958679Z","shell.execute_reply":"2021-12-30T16:48:49.967682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cort & Classification","metadata":{}},{"cell_type":"code","source":"# Best lb :\nLOG_PATH = \"../input/sartorius-cps-ens11/\"\n\nEXP_FOLDERS = [\n    LOG_PATH + \"2021-12-11_2/\",  # 1. Cascade b5 - 0.3121\n    LOG_PATH + \"2021-12-11_4/\",  # 2. Cascade rx101 - 0.3141\n    LOG_PATH + \"2021-12-12_0/\",  # 3. Cascade r50 - 0.3125\n    LOG_PATH + \"seb_mrcnn_resnext101_lossdecay/\", # 11. mrcnn r101 0.3131\n    LOG_PATH + \"seb_mrcnn_r50_lossdecay/\", # 12. mrcnn r50 0.3125\n    LOG_PATH + \"2021-12-15_0/\",  # 14. Cascade b6 - 0.3121\n]\n\nTHRESHOLDS_MASK = 0.45\nTHRESHOLDS_NMS = [0.1, 0.1, 0.15]\nTHRESHOLDS_CONF = [0.3, 0.4, 0.65]\nENSEMBLE_CONFIG['rcnn_score_threshold'] = THRESHOLDS_CONF","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:48:49.971132Z","iopub.execute_input":"2021-12-30T16:48:49.971788Z","iopub.status.idle":"2021-12-30T16:48:49.984531Z","shell.execute_reply.started":"2021-12-30T16:48:49.971743Z","shell.execute_reply":"2021-12-30T16:48:49.983474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Best CV :\n# LOG_PATH = \"../input/sartorius-cps-ens11/\"\n# LOG_PATH_2 = \"../input/sartorius-cps-last/\"\n\n# EXP_FOLDERS = [\n#     LOG_PATH + \"2021-12-11_2/\",  # 1. Cascade b5 - 0.3121\n#     LOG_PATH + \"2021-12-11_4/\",  # 2. Cascade rx101 - 0.3141\n#     LOG_PATH + \"2021-12-15_0/\",  # 14. Cascade b6 - 0.3121\n#     LOG_PATH_2 + \"seb_mrcnn_b5/\", # 19. mrcnn b5 - 0.3086\n#     LOG_PATH_2 + \"2021-12-22_6/\",  #  21. htc b4 - 0.3083\n#     LOG_PATH_2 + \"seb_mrcnn_rx101_decay_bn_flip_aug/\",  # 24. mrcnn rx101 - 0.3141\n# ]\n\n# THRESHOLDS_MASK = 0.45\n# THRESHOLDS_NMS = [0.1, 0.1, 0.15]\n# THRESHOLDS_CONF = [0.3, 0.4, 0.65]  # [0.3, 0.4, 0.65]","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:48:49.987177Z","iopub.execute_input":"2021-12-30T16:48:49.987978Z","iopub.status.idle":"2021-12-30T16:48:49.996926Z","shell.execute_reply.started":"2021-12-30T16:48:49.9879Z","shell.execute_reply":"2021-12-30T16:48:49.995632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"configs, weights = [], []\n\nfor exp_folder in EXP_FOLDERS:\n    config = Config(json.load(open(exp_folder + \"config.json\", 'r')))\n    config.model_config = exp_folder + config.model_config.split('/')[-1]\n    config.data_config = exp_folder + config.data_config.split('/')[-1]\n    configs.append(config)\n\n    weights.append(sorted(glob.glob(exp_folder + \"*.pt\")))","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:48:49.998883Z","iopub.execute_input":"2021-12-30T16:48:49.999297Z","iopub.status.idle":"2021-12-30T16:48:50.11078Z","shell.execute_reply.started":"2021-12-30T16:48:49.999255Z","shell.execute_reply":"2021-12-30T16:48:50.109816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nrles_cort, cell_types = inference(df, configs, weights, ENSEMBLE_CONFIG, THRESHOLDS_MASK, THRESHOLDS_NMS, THRESHOLDS_CONF, MIN_SIZES)\n\ndf['rle_cort'] = rles_cort\ndf['cell_type'] = cell_types\nSUCCESS[0] = True","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:48:50.117795Z","iopub.execute_input":"2021-12-30T16:48:50.118054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Astro","metadata":{}},{"cell_type":"code","source":"LOG_PATH = \"../input/sartorius-cps-ens11/\"\n\nEXP_FOLDERS = [\n    LOG_PATH + \"2021-12-11_2/\",  # 1. Cascade b5 - 0.3121\n    LOG_PATH + \"2021-12-11_4/\",  # 2. Cascade rx101 - 0.3141\n    LOG_PATH + \"2021-12-12_0/\",  # 3. Cascade r50 - 0.3125\n    LOG_PATH + \"seb_mrcnn_resnext101_lossdecay/\", # 11. mrcnn r101 0.3131\n    LOG_PATH + \"2021-12-15_1/\",  # 15. htc r50 - 0.3121\n    LOG_PATH + \"seb_mrcnn_r101_64x4/\",  # 22. mrcnn rx101_64x4 - 0.3127\n]\n\nTHRESHOLDS_MASK = 0.45\nTHRESHOLDS_NMS = [0.05, 0.05, 0.05]\nTHRESHOLDS_CONF = [0.45, 0.45, 0.45]\nENSEMBLE_CONFIG['rcnn_score_threshold'] = THRESHOLDS_CONF\n\nENSEMBLE_CONFIG[\"ttas_masks\"] = [None, \"horizontal\", \"vertical\", \"diagonal\"]\nENSEMBLE_CONFIG[\"delta\"] = 0.5","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"configs, weights = [], []\n\nfor exp_folder in EXP_FOLDERS:\n    config = Config(json.load(open(exp_folder + \"config.json\", 'r')))\n    config.model_config = exp_folder + config.model_config.split('/')[-1]\n    config.data_config = exp_folder + config.data_config.split('/')[-1]\n    configs.append(config)\n\n    weights.append(sorted(glob.glob(exp_folder + \"*.pt\")))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nassert SUCCESS[0]\n\ndf_astro = df[df['cell_type'] == 1].reset_index()\n\nrles_astro, _ = inference(df_astro, configs, weights, ENSEMBLE_CONFIG, THRESHOLDS_MASK, THRESHOLDS_NMS, THRESHOLDS_CONF, MIN_SIZES)\n\ndf_astro['rle_astro'] = rles_astro\nSUCCESS[1] = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Shsy5y","metadata":{}},{"cell_type":"code","source":"LOG_PATH = \"../input/sartorius-cps-ens11/\"\nLOG_PATH_2 = \"../input/sartorius-cps-ens10/\"\n\nEXP_FOLDERS = [\n    LOG_PATH + \"2021-12-12_0/\",  # 3. Cascade r50 - 0.3125\n    LOG_PATH_2 + \"seb_mrcnn_resnet50_new_splits/\", # 8. maskrcnn r50 - 0.3118\n    LOG_PATH + \"2021-12-15_1/\",  # 15. htc r50 - 0.3121\n    LOG_PATH_2 + \"2021-12-20_1/\",  #  16. Cascade rx101_64x4 - 0.3130\n    LOG_PATH_2 + \"2021-12-22_2/\",  #  20. cascade b6 192 crops - 0.3118\n    LOG_PATH + \"seb_mrcnn_r101_64x4/\",  # 22. mrcnn rx101_64x4 - 0.3127\n]\n\nTHRESHOLDS_MASK = 0.45\nTHRESHOLDS_NMS = [0.1, 0.1, 0.1]\nTHRESHOLDS_CONF = [0.35, 0.35, 0.35]\nENSEMBLE_CONFIG['rcnn_score_threshold'] = THRESHOLDS_CONF\n\n\n# Best CV :\nENSEMBLE_CONFIG[\"ttas_masks\"] = [None, \"horizontal\"]  #, \"vertical\", \"diagonal\"]\nENSEMBLE_CONFIG[\"delta\"] = 0.\n# # Best LB ? :\n# ENSEMBLE_CONFIG[\"ttas_masks\"] = [None, \"horizontal\", \"vertical\", \"diagonal\"]\n# ENSEMBLE_CONFIG[\"delta\"] = 0.5","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"configs, weights = [], []\n\nfor exp_folder in EXP_FOLDERS:\n    config = Config(json.load(open(exp_folder + \"config.json\", 'r')))\n    config.model_config = exp_folder + config.model_config.split('/')[-1]\n    config.data_config = exp_folder + config.data_config.split('/')[-1]\n    configs.append(config)\n\n    weights.append(sorted(glob.glob(exp_folder + \"*.pt\")))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nassert SUCCESS[1]\n\ndf_shsy5y = df[df['cell_type'] == 0].reset_index()\n\nrles_shsy5y, _ = inference(df_shsy5y, configs, weights, ENSEMBLE_CONFIG, THRESHOLDS_MASK, THRESHOLDS_NMS, THRESHOLDS_CONF, MIN_SIZES)\n\ndf_shsy5y['rle_shsy5y'] = rles_shsy5y\nSUCCESS[2] = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sub + viz","metadata":{}},{"cell_type":"code","source":"assert all(SUCCESS)\n\ndf, rles = merge_rles(df, df_astro, df_shsy5y)\n\nsubmission = []\npipelines = define_pipelines(configs[0].data_config)\ndataset = SartoriusInferenceDataset(df, transforms=pipelines['test_viz'], precompute_masks=False)\n\nfor idx, (rle, img_id) in enumerate(zip(rles, df['id'].values)):\n    if idx < 3:\n        img = dataset[idx]['img'][0].numpy().transpose(1, 2, 0)\n        img = (img - img.min()) / (img.max() - img.min())\n        img = img[:ORIG_SIZE[0], :ORIG_SIZE[1]]\n\n        masks = np.array([rle_decode(enc, ORIG_SIZE) for enc in rle])\n\n        assert masks.sum(0).max() <= 1\n        \n        plt.figure(figsize=(15, 15))\n        plot_sample(img, masks.astype(int))\n        plt.axis(False)\n        plt.show()\n    \n    for enc in rle:\n        submission.append((img_id, enc))\n        \n    if not len(rle):  # Empty\n        submission.append((image_id, \"\"))\n\ndf_sub = pd.DataFrame(submission, columns=['id', 'predicted'])\ndf_sub.to_csv(\"submission.csv\", index=False)\ndf_sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Done !","metadata":{}}]}