{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/detectron-05/whls/pycocotools-2.0.2/dist/pycocotools-2.0.2.tar --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/fvcore-0.1.5.post20211019/fvcore-0.1.5.post20211019 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/antlr4-python3-runtime-4.8/antlr4-python3-runtime-4.8 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/detectron2-0.5/detectron2 --no-index --find-links ../input/detectron-05/whls ","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-30T17:43:50.830890Z","iopub.execute_input":"2021-12-30T17:43:50.831850Z","iopub.status.idle":"2021-12-30T17:48:18.549585Z","shell.execute_reply.started":"2021-12-30T17:43:50.831656Z","shell.execute_reply":"2021-12-30T17:48:18.548454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp ../input/resnest200/mask_rcnn_ResNeSt200.yaml /opt/conda/lib/python3.7/site-packages/detectron2/model_zoo/configs/COCO-InstanceSegmentation/\n!cp ../input/resnest200/resnet.py /opt/conda/lib/python3.7/site-packages/detectron2/modeling/backbone/\n!cp ../input/resnest200/splat.py /opt/conda/lib/python3.7/site-packages/detectron2/modeling/backbone/","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:48:18.552423Z","iopub.execute_input":"2021-12-30T17:48:18.553246Z","iopub.status.idle":"2021-12-30T17:48:21.048726Z","shell.execute_reply.started":"2021-12-30T17:48:18.553200Z","shell.execute_reply":"2021-12-30T17:48:21.047406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:48:21.053149Z","iopub.execute_input":"2021-12-30T17:48:21.053440Z","iopub.status.idle":"2021-12-30T17:48:21.821696Z","shell.execute_reply.started":"2021-12-30T17:48:21.053366Z","shell.execute_reply":"2021-12-30T17:48:21.820622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import detectron2\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg, CfgNode\nfrom PIL import Image\nimport cv2\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom fastcore.all import *\ndetectron2.__version__\nimport torch\n\nimport pycocotools.mask as mask_util\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:48:21.824871Z","iopub.execute_input":"2021-12-30T17:48:21.826125Z","iopub.status.idle":"2021-12-30T17:48:23.219876Z","shell.execute_reply.started":"2021-12-30T17:48:21.826092Z","shell.execute_reply":"2021-12-30T17:48:23.218863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataDir=Path('../input/sartorius-cell-instance-segmentation')","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:48:23.223445Z","iopub.execute_input":"2021-12-30T17:48:23.224262Z","iopub.status.idle":"2021-12-30T17:48:23.230012Z","shell.execute_reply.started":"2021-12-30T17:48:23.224216Z","shell.execute_reply":"2021-12-30T17:48:23.228745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make detectron2 return per pixel mask score\ndef paste_masks_in_image(masks, boxes, image_shape, threshold=0.5):\n    \"\"\"\n    Copy pasted from detectron2.layers.mask_ops.paste_masks_in_image and deleted thresholding of the mask\n    \"\"\"\n#     print(torch.unique(masks))\n    assert masks.shape[-1] == masks.shape[-2], \"Only square mask predictions are supported\"\n    N = len(masks)\n    if N == 0:\n        return masks.new_empty((0,) + image_shape, dtype=torch.uint8)\n    if not isinstance(boxes, torch.Tensor):\n        boxes = boxes.tensor\n    device = boxes.device\n    assert len(boxes) == N, boxes.shape\n\n    img_h, img_w = image_shape\n\n    # The actual implementation split the input into chunks,\n    # and paste them chunk by chunk.\n    if device.type == \"cpu\":\n        # CPU is most efficient when they are pasted one by one with skip_empty=True\n        # so that it performs minimal number of operations.\n        num_chunks = N\n    else:\n        # GPU benefits from parallelism for larger chunks, but may have memory issue\n        num_chunks = int(np.ceil(N * img_h * img_w * BYTES_PER_FLOAT / GPU_MEM_LIMIT))\n        assert (\n            num_chunks <= N\n        ), \"Default GPU_MEM_LIMIT in mask_ops.py is too small; try increasing it\"\n    chunks = torch.chunk(torch.arange(N, device=device), num_chunks)\n\n    img_masks = torch.zeros(\n        N, img_h, img_w, device=device, dtype=torch.float32\n    )\n#     print(torch.unique(masks))\n    for inds in chunks:\n        masks_chunk, spatial_inds = _do_paste_mask(\n            masks[inds, None, :, :], boxes[inds], img_h, img_w, skip_empty=device.type == \"cpu\"\n        )\n        img_masks[(inds,) + spatial_inds] = masks_chunk\n#     print(torch.unique(img_masks))\n    return img_masks\n\nfrom typing import Any, Iterator, List, Union\nimport numpy as np\n\ndef BitMasks__init__(self, tensor: Union[torch.Tensor, np.ndarray]):\n    \"\"\"\n    Args:\n        tensor: bool Tensor of N,H,W, representing N instances in the image.\n    \"\"\"\n    device = tensor.device if isinstance(tensor, torch.Tensor) else torch.device(\"cpu\")\n    tensor = torch.as_tensor(tensor, dtype=torch.float32, device=device)\n    assert tensor.dim() == 3, tensor.size()\n    self.image_size = tensor.shape[1:]\n    self.tensor = tensor\n    \ndetectron2.structures.masks.BitMasks.__init__.__code__ = BitMasks__init__.__code__\n\ndetectron2.layers.mask_ops.paste_masks_in_image.__code__ = paste_masks_in_image.__code__","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:48:23.231996Z","iopub.execute_input":"2021-12-30T17:48:23.233236Z","iopub.status.idle":"2021-12-30T17:48:23.252774Z","shell.execute_reply.started":"2021-12-30T17:48:23.233187Z","shell.execute_reply":"2021-12-30T17:48:23.251545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_predictor_v21():\n    cfg = get_cfg()\n\n    cfg.MODEL.RESNETS.RADIX = 1\n    cfg.MODEL.RESNETS.DEEP_STEM = False\n    cfg.MODEL.RESNETS.AVD = False\n    # Apply avg_down to the downsampling layer for residual path \n    cfg.MODEL.RESNETS.AVG_DOWN = False\n    cfg.MODEL.RESNETS.BOTTLENECK_WIDTH = 64\n\n    cfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_ResNeSt200.yaml\"))\n    cfg.INPUT.MASK_FORMAT='bitmask'\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 3 \n    cfg.MODEL.WEIGHTS = '../input/sartoriusnew-mask-rcnn-v21/model_0017999.pth'\n    # cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5\n    cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = 0.99 # keep almost every instances\n\n    cfg.TEST.DETECTIONS_PER_IMAGE = 10000\n\n    cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 1024   \n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 3  \n\n    cfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE = 1024\n    cfg.MODEL.RPN.POSITIVE_FRACTION = 0.7\n\n    # img size\n    cfg.INPUT.MIN_SIZE_TRAIN = (1024, )\n    cfg.INPUT.MAX_SIZE_TRAIN = (2000, )\n    cfg.INPUT.MIN_SIZE_TEST = 1024\n    cfg.INPUT.MAX_SIZE_TEST = 2000\n\n    cfg.TEST.AUG.MIN_SIZES = (1024, )\n    cfg.TEST.AUG.FLIP = False\n    \n    predictor = DefaultPredictor(cfg)\n    \n    return cfg, predictor\n\ndef get_predictor_v16():\n    cfg = get_cfg()\n\n    cfg.MODEL.RESNETS.RADIX = 1\n    cfg.MODEL.RESNETS.DEEP_STEM = False\n    cfg.MODEL.RESNETS.AVD = False\n    # Apply avg_down to the downsampling layer for residual path \n    cfg.MODEL.RESNETS.AVG_DOWN = False\n    cfg.MODEL.RESNETS.BOTTLENECK_WIDTH = 64\n\n    cfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_ResNeSt200.yaml\"))\n    cfg.INPUT.MASK_FORMAT='bitmask'\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 3 \n    cfg.MODEL.WEIGHTS = '../input/sartoriusnew-mask-rcnn-v16/model_0028999.pth'\n    # cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5\n    cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = 0.99 # keep almost every instances\n\n    cfg.TEST.DETECTIONS_PER_IMAGE = 10000\n\n    cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 1024   \n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 3  \n\n    cfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE = 1024\n    cfg.MODEL.RPN.POSITIVE_FRACTION = 0.7\n\n    # img size\n    cfg.INPUT.MIN_SIZE_TRAIN = (1024, )\n    cfg.INPUT.MAX_SIZE_TRAIN = (2000, )\n    cfg.INPUT.MIN_SIZE_TEST = 1024\n    cfg.INPUT.MAX_SIZE_TEST = 2000\n\n    cfg.TEST.AUG.MIN_SIZES = (1024, )\n    cfg.TEST.AUG.FLIP = False\n    \n    predictor = DefaultPredictor(cfg)\n    \n    return cfg, predictor","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:48:23.254953Z","iopub.execute_input":"2021-12-30T17:48:23.255321Z","iopub.status.idle":"2021-12-30T17:48:23.275530Z","shell.execute_reply.started":"2021-12-30T17:48:23.255278Z","shell.execute_reply":"2021-12-30T17:48:23.274352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# From https://www.kaggle.com/stainsby/fast-tested-rle\ndef rle_decode(mask_rle, shape=(520, 704)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape)  # Needed to align to RLE direction\n\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef check_overlap(msk):\n    msk = msk.astype(np.bool).astype(np.uint8)\n    return np.any(np.sum(msk, axis=-1)>1)\n\ndef filter_masks_by_area(mask_min_area, outputs):\n    pred_class = torch.mode(outputs['instances'].pred_classes)[0]\n    area_min = mask_min_area[pred_class]\n    take = (outputs['instances'].pred_masks >= 0.5).sum(axis=(1,2)) >= area_min\n    outputs['instances'] = outputs['instances'][take]\n    return outputs\n\ndef get_masks(pred, shape=(520,704)):\n    pred_class = torch.mode(pred['instances'].pred_classes)[0]\n    \n#     if(pred_class != 1):\n#         return ['']\n    \n    # filter masks by area min\n    pred = filter_masks_by_area([70, 120, 50], pred)\n    \n    pred_masks = pred['instances'].pred_masks\n    pred_masks = pred_masks.cpu().numpy()\n    res = []\n    used = np.zeros(shape, dtype=int) \n    for mask in pred_masks:\n        mask = mask * (1-used)\n        used += mask\n        res.append(rle_encode(mask))\n    return res","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:48:23.279707Z","iopub.execute_input":"2021-12-30T17:48:23.280061Z","iopub.status.idle":"2021-12-30T17:48:23.299555Z","shell.execute_reply.started":"2021-12-30T17:48:23.280028Z","shell.execute_reply":"2021-12-30T17:48:23.298410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.neighbors import KDTree\n\ndef calculate_max_IOU_with_gt(targ, pred):\n    pred_masks = pred['instances'].pred_masks >= 0.5\n    enc_preds = [mask_util.encode(np.asarray(p, order='F')) for p in pred_masks]\n    enc_targs = list(map(lambda x:x['segmentation'], targ))\n    ious = mask_util.iou(enc_preds, enc_targs, [0]*len(enc_targs))\n    \n    return ious.max(axis=1)\n\ndef print_log(log):\n    for k in log.keys():\n        print(k, log[k])\n        \ndef get_overlapping_features(pred):\n    pred_masks = pred['instances'].pred_masks >= 0.5\n    enc_preds = [mask_util.encode(np.asarray(p, order='F')) for p in pred_masks]\n    ious = mask_util.iou(enc_preds, enc_preds, [0]*len(enc_preds))\n    return ious.max(axis=1), ious.min(axis=1), ious.mean(axis=1),\\\n             ious.std(axis=1), (ious > 0).sum(axis=1)\n\ndef get_contour_features(pred):\n    masks = (pred['instances'].pred_masks.numpy() >= 0.5).astype('uint8')\n    \n    data_dict = {\n        'centroid_x':[],\n        'centroid_y':[],\n        'num_contours': [],\n        'equi_diameter':[],\n        'hull_area':[],\n        'solidity':[],\n        'is_convex':[],\n        'perimeter':[],\n        'rotation_ang':[],\n        'major_axis_length':[],\n        'minor_axis_length':[]\n    }\n    \n    for mask in masks:\n        contours, _ = cv2.findContours(mask, 1, 2)\n        areas = [cv2.contourArea(cnt) for cnt in contours]\n        max_ind = np.argmax(areas)\n        \n        area = areas[max_ind]\n        cnt = contours[max_ind]\n        \n        M = cv2.moments(cnt)\n        cx = int(M['m10']/M['m00'])\n        cy = int(M['m01']/M['m00'])\n        \n        hull = cv2.convexHull(cnt)\n        hull_area = cv2.contourArea(hull)\n        solidity = float(area)/hull_area if hull_area > 0 else -1\n        \n        equi_diameter = np.sqrt(4*area/np.pi)\n        is_convex = int(cv2.isContourConvex(cnt))\n        perimeter = cv2.arcLength(cnt,True)\n        \n        try:\n            ellipse = cv2.fitEllipse(cnt)\n            _,(major_axis_length, minor_axis_length), rotation_ang = ellipse\n        except:\n            (major_axis_length, minor_axis_length), rotation_ang = (-1,-1),-1\n        \n        data_dict['centroid_x'].append(cx)\n        data_dict['centroid_y'].append(cy)\n        data_dict['num_contours'].append(len(contours))\n        data_dict['equi_diameter'].append(equi_diameter)\n        data_dict['solidity'].append(solidity)\n        data_dict['hull_area'].append(hull_area)\n        data_dict['is_convex'].append(is_convex)\n        data_dict['perimeter'].append(perimeter)\n        data_dict['rotation_ang'].append(rotation_ang)\n        data_dict['major_axis_length'].append(major_axis_length)\n        data_dict['minor_axis_length'].append(minor_axis_length)\n        \n    return pd.DataFrame(data_dict)\n\ndef get_pixel_scores_features(outputs):\n    pred_masks = outputs['instances'].pred_masks\n    pred_masks_non_zeros = [mask[mask > 0] for mask in pred_masks]\n    min_pscores = [mask.min().item() for mask in pred_masks_non_zeros]\n    max_pscores = [mask.max().item() for mask in pred_masks_non_zeros]\n    median_pscores = [mask.median().item() for mask in pred_masks_non_zeros]\n    mean_pscores = [mask.mean().item() for mask in pred_masks_non_zeros]\n                    \n    q1_pscores =  [mask.quantile(0.25).item() for mask in pred_masks_non_zeros]\n    q3_pscores =  [mask.quantile(0.75).item() for mask in pred_masks_non_zeros]\n    std_pscores = [mask.std().item() for mask in pred_masks_non_zeros]\n    \n    ret = {\n        'min_pixel_score':min_pscores,\n        'max_pixel_score':max_pscores,\n        'median_pixel_score':median_pscores,\n        'mean_pixel_score':mean_pscores,\n        'q1_pixel_score':q1_pscores,\n        'q3_pixel_score':q3_pscores,\n        'std_pixel_score':std_pscores\n    }\n    return pd.DataFrame(ret)\n\ndef get_image_pixel_features(im, outputs):\n    pred_masks = outputs['instances'].pred_masks\n    pred_masks_binary = [mask > 0.5 for mask in pred_masks]\n    im_masks = [im[mask,0] for mask in pred_masks_binary]\n    \n    min_pscores = [mask.min().item() for mask in im_masks]\n    max_pscores = [mask.max().item() for mask in im_masks]\n    median_pscores = [np.median(mask).item() for mask in im_masks]\n    mean_pscores = [mask.mean().item() for mask in im_masks]\n                    \n    q1_pscores =  [np.quantile(mask, 0.25).item() for mask in im_masks]\n    q3_pscores =  [np.quantile(mask, 0.75) for mask in im_masks]\n    std_pscores = [mask.std() for mask in im_masks]\n    \n    ret = {\n        'im_min_pixel':min_pscores,\n        'im_max_pixel':max_pscores,\n        'im_median_pixel':median_pscores,\n        'im_mean_pixel':mean_pscores,\n        'im_q1_pixel':q1_pscores,\n        'im_q3_pixel':q3_pscores,\n        'im_std_pixel':std_pscores\n    }\n    return pd.DataFrame(ret)\n\ndef get_kdtree_nb_features(single_features):\n    cols = ['centroid_x', 'centroid_y']\n    X = single_features[cols]\n    tree = KDTree(X)  \n    \n    ret = dict()\n    for r in [25, 50, 75, 100, 150, 200]:\n        ind, dist = tree.query_radius(X, r=r, return_distance=True, sort_results=True)  \n        ind = [i[1:] for i in ind] # exclude neareast neighbor (itself)\n        dist = [d[1:] for d in dist] # exclude neareast neighbor (itself)\n        \n        ret[f'kdtree_nb_r{r}_count'] = [len(ind) for i in ind]\n        \n        ret[f'kdtree_nb_r{r}_median_dist'] = [np.median(d) if len(d)>0 else -1 for d in dist]\n        ret[f'kdtree_nb_r{r}_mean_dist'] = [d.mean() if len(d)>0 else -1 for d in dist]\n        ret[f'kdtree_nb_r{r}_std_dist'] = [np.std(d) if len(d)>0 else -1 for d in dist]\n        \n        ret[f'kdtree_nb_r{r}_median_area'] = [single_features.loc[i, 'mask_area'].median() if len(i)>0 else -1 for i in ind]\n        ret[f'kdtree_nb_r{r}_mean_area'] = [single_features.loc[i, 'mask_area'].mean() if len(i)>0 else -1 for i in ind]\n        ret[f'kdtree_nb_r{r}_std_area'] = [single_features.loc[i, 'mask_area'].std() if len(i)>0 else -1 for i in ind]\n        \n        ret[f'kdtree_nb_r{r}_median_box_score'] = [single_features.loc[i, 'box_score'].median() if len(i)>0 else -1 for i in ind]\n        ret[f'kdtree_nb_r{r}_mean_box_score'] = [single_features.loc[i, 'box_score'].mean() if len(i)>0 else -1 for i in ind]\n        ret[f'kdtree_nb_r{r}_std_box_score'] = [single_features.loc[i, 'box_score'].std() if len(i)>0 else -1 for i in ind]\n        \n    for k in [2,3,5,7]:\n        dist, ind = tree.query(X, k=k, return_distance=True)  \n        ind = [i[1:] for i in ind] # exclude neareast neighbor (itself)\n        dist = [d[1:] for d in dist] # exclude neareast neighbor (itself)\n        \n        ret[f'kdtree_nb_top{k}_median_dist'] = [np.median(d) if len(d)>0 else -1 for d in dist]\n        ret[f'kdtree_nb_top{k}_mean_dist'] = [d.mean() if len(d)>0 else -1  for d in dist]\n        ret[f'kdtree_nb_top{k}_std_dist'] = [np.std(d) if len(d)>0 else -1 for d in dist]\n        \n        ret[f'kdtree_nb_top{k}_median_area'] = [single_features.loc[i, 'mask_area'].median() if len(i)>0 else -1 for i in ind]\n        ret[f'kdtree_nb_top{k}_mean_area'] = [single_features.loc[i, 'mask_area'].mean() if len(i)>0 else -1 for i in ind]\n        ret[f'kdtree_nb_top{k}_std_area'] = [single_features.loc[i, 'mask_area'].std() if len(i)>0 else -1 for i in ind]\n        \n        ret[f'kdtree_nb_top{k}_median_box_score'] = [single_features.loc[i, 'box_score'].median() if len(i)>0 else -1 for i in ind]\n        ret[f'kdtree_nb_top{k}_mean_box_score'] = [single_features.loc[i, 'box_score'].mean() if len(i)>0 else -1 for i in ind]\n        ret[f'kdtree_nb_top{k}_std_box_score'] = [single_features.loc[i, 'box_score'].std() if len(i)>0 else -1 for i in ind]\n        \n    return pd.DataFrame(ret)    \n    \ndef get_features(im, outputs):\n    pred_masks = outputs['instances'].pred_masks\n    \n    mask_areas = (pred_masks >= 0.5).sum(axis=(1,2))\n    pred_boxes = outputs['instances'].pred_boxes.tensor\n    widths = pred_boxes[:,2] - pred_boxes[:,0]\n    heights = pred_boxes[:,3] - pred_boxes[:,1]\n    box_areas = widths * heights\n    \n    box_scores = outputs['instances'].scores\n    instance_count = len(outputs['instances'])\n    \n    aspect_ratios = widths / heights\n    extents = mask_areas / box_areas\n    \n    neighbor_iou_max, neighbor_iou_min, neighbor_iou_mean, \\\n        neighbor_iou_std, neighbor_overlap_count = get_overlapping_features(outputs)\n    \n    contour_features = get_contour_features(outputs)\n    pixel_features = get_pixel_scores_features(outputs)\n    im_pixel_features = get_image_pixel_features(im, outputs)\n    \n    ret = pd.DataFrame({\n        'box_score':box_scores,\n        'mask_area':mask_areas,\n        'box_area':box_areas,\n        'box_x1':pred_boxes[:,0],\n        'box_y1':pred_boxes[:,1],\n        'box_x2':pred_boxes[:,2],\n        'box_y2':pred_boxes[:,3],\n        'width':widths,\n        'height':heights,\n        'instance_count':instance_count,\n        \n        'neighbor_iou_max':neighbor_iou_max,\n        'neighbor_iou_min':neighbor_iou_min,\n        'neighbor_iou_mean':neighbor_iou_mean,\n        'neighbor_iou_std':neighbor_iou_std,\n        'neighbor_overlap_count':neighbor_overlap_count,\n        \n        'aspect_ratio':aspect_ratios,\n        'extent':extents\n    })\n    \n    ret = pd.concat([ret, contour_features, pixel_features, im_pixel_features], axis=1)\n    kdtree_nb_features = get_kdtree_nb_features(ret)\n    ret = pd.concat([ret, kdtree_nb_features], axis=1)\n    return ret","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:48:23.302512Z","iopub.execute_input":"2021-12-30T17:48:23.302877Z","iopub.status.idle":"2021-12-30T17:48:24.462145Z","shell.execute_reply.started":"2021-12-30T17:48:23.302828Z","shell.execute_reply":"2021-12-30T17:48:24.461194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import copy\nfrom detectron2.data.detection_utils import read_image\nfrom detectron2.modeling.test_time_augmentation import GeneralizedRCNNWithTTA, DatasetMapperTTA\nfrom detectron2.modeling.roi_heads.fast_rcnn import fast_rcnn_inference_single_image\nfrom detectron2.modeling.postprocessing import detector_postprocess\nfrom detectron2.structures import Boxes, Instances\nfrom fvcore.transforms import HFlipTransform, NoOpTransform\n\nfrom contextlib import contextmanager\nfrom itertools import count\nimport itertools","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:48:24.463761Z","iopub.execute_input":"2021-12-30T17:48:24.464070Z","iopub.status.idle":"2021-12-30T17:48:24.472373Z","shell.execute_reply.started":"2021-12-30T17:48:24.464027Z","shell.execute_reply":"2021-12-30T17:48:24.471082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_nms(pred, nms_thresh=0.5):\n    pred_masks = pred['instances'].pred_masks.detach().cpu().numpy()\n    scores = pred['instances'].scores.detach().cpu().numpy()\n    # print(pred_masks.dtype)\n    if(pred_masks.dtype == np.float32):\n        pred_masks = (pred_masks >= 0.5 ).astype('bool')\n        # print(pred_masks.dtype)\n\n    # calculate IOU\n    enc_preds = [mask_util.encode(np.asarray(p, order='F')) for p in pred_masks]\n    ious = mask_util.iou(enc_preds, enc_preds, [0]*len(enc_preds))\n\n    orders = np.arange(len(scores))\n    keeps = []\n\n    while len(orders) > 0:\n\n        # append the idx of the instance X with highest score to keeps list\n        keeps.append(orders[0])\n        # remove the idx of the above instance from the remaining list\n        orders = orders[1:]\n\n        # check IOU of the instance X with all the instance in the remaining list\n        look_up = ious[keeps[-1], orders]\n\n        # filter those having IOU > nms_thresh\n        mask = look_up < nms_thresh\n        orders = orders[mask]\n\n        if(len(orders)==0):\n            break\n    \n    new_pred = pred.copy()\n    new_pred['instances'] = new_pred['instances'][keeps]\n    return new_pred","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:51:35.683639Z","iopub.execute_input":"2021-12-30T17:51:35.683986Z","iopub.status.idle":"2021-12-30T17:51:35.695509Z","shell.execute_reply.started":"2021-12-30T17:51:35.683952Z","shell.execute_reply":"2021-12-30T17:51:35.694110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@contextmanager\ndef turn_off_roi_heads(model, attrs):\n        \"\"\"\n        Open a context where some heads in `model.roi_heads` are temporarily turned off.\n        Args:\n            attr (list[str]): the attribute in `model.roi_heads` which can be used\n                to turn off a specific head, e.g., \"mask_on\", \"keypoint_on\".\n        \"\"\"\n        roi_heads = model.roi_heads\n        old = {}\n        for attr in attrs:\n            try:\n                old[attr] = getattr(roi_heads, attr)\n            except AttributeError:\n                # The head may not be implemented in certain ROIHeads\n                pass\n\n        if len(old.keys()) == 0:\n            yield\n        else:\n            for attr in old.keys():\n                setattr(roi_heads, attr, False)\n            yield\n            for attr in old.keys():\n                setattr(roi_heads, attr, old[attr])\n\ndef model_batch_forward(model, inputs, instances=[None]):\n    outputs = model.inference(inputs, \n                   instances if instances[0] is not None else None,\n                   do_postprocess=False)\n    return outputs\n\ndef merge_detections(all_boxes, all_scores, all_classes, shape_hw, num_classes=3, \n                     box_nms_thresh=0.99, max_detection_per_im=10000):\n    # select from the union of all results\n    num_boxes = len(all_boxes)\n    \n    # +1 because fast_rcnn_inference expects background scores as well\n    all_scores_2d = torch.zeros(num_boxes, num_classes + 1, device=all_boxes.device)\n    for idx, cls, score in zip(count(), all_classes, all_scores):\n        all_scores_2d[idx, cls] = score\n\n    merged_instances, _ = fast_rcnn_inference_single_image(\n        all_boxes,\n        all_scores_2d,\n        shape_hw,\n        1e-8,\n        box_nms_thresh,\n        max_detection_per_im,\n    )\n    \n    return merged_instances\n\ndef rescale_detected_boxes(augmented_inputs, merged_instances, tfms):\n    augmented_instances = []\n    for input, tfm in zip(augmented_inputs, tfms):\n        # Transform the target box to the augmented image's coordinate space\n        pred_boxes = merged_instances.pred_boxes.tensor.cpu().numpy()\n        pred_boxes = torch.from_numpy(tfm.apply_box(pred_boxes))\n\n        aug_instances = Instances(\n            image_size=input[\"image\"].shape[1:3],\n            pred_boxes=Boxes(pred_boxes),\n            pred_classes=merged_instances.pred_classes,\n            scores=merged_instances.scores,\n        )\n        augmented_instances.append(aug_instances)\n    return augmented_instances\n\ndef reduce_pred_masks(outputs, tfms):\n    # Should apply inverse transforms on masks.\n    # We assume only resize & flip are used. pred_masks is a scale-invariant\n    # representation, so we handle flip specially\n    for output, tfm in zip(outputs, tfms):\n        if any(isinstance(t, HFlipTransform) for t in tfm.transforms):\n            output.pred_masks = output.pred_masks.flip(dims=[3])\n    all_pred_masks = torch.stack([o.pred_masks for o in outputs], dim=0)\n    avg_pred_masks = torch.mean(all_pred_masks, dim=0)\n    return avg_pred_masks\n\ndef ensemble(inp_dict, list_configs, list_predictors, \n            mask_nms_thresh=0.1, max_detection_per_im=10000,\n            conf_thresh=[0.3,0.5,0.6]):\n    # read image\n    inp = copy.copy(inp_dict)\n    image = read_image(inp.pop(\"file_name\"), list_predictors[0].model.input_format)\n    image = torch.from_numpy(np.ascontiguousarray(image.transpose(2, 0, 1)))  # CHW\n    inp[\"image\"] = image\n    if \"height\" not in inp and \"width\" not in inp:\n        inp[\"height\"] = image.shape[1]\n        inp[\"width\"] = image.shape[2]\n    orig_shape = (inp[\"height\"], inp[\"width\"])\n    \n    # dataset mapper for each model\n    list_tta_mappers = []\n    list_augmented_inputs = []\n    list_tfms = []\n    \n    all_boxes = []\n    all_scores = []\n    all_classes = []\n    \n    for cfg, predictor in zip(list_configs, list_predictors):\n        tta_mapper = DatasetMapperTTA(cfg)\n        list_tta_mappers.append(tta_mapper)\n        augmented_inputs = tta_mapper(inp)\n        \n        tfms = [x.pop(\"transforms\") for x in augmented_inputs]\n        list_augmented_inputs.append(augmented_inputs)\n        list_tfms.append(tfms)\n        \n#         return augmented_inputs\n        with turn_off_roi_heads(predictor.model, ['mask_on', 'keypoint_on']), torch.no_grad():\n            outputs = model_batch_forward(predictor.model, augmented_inputs)\n            \n        for output, tfm in zip(outputs, tfms):\n            # Need to inverse the transforms on boxes, to obtain results on original image\n            pred_boxes = output.pred_boxes.tensor\n            original_pred_boxes = tfm.inverse().apply_box(pred_boxes.cpu().numpy())\n            all_boxes.append(torch.from_numpy(original_pred_boxes).to(pred_boxes.device))\n\n            all_scores.extend(output.scores)\n            all_classes.extend(output.pred_classes)\n            \n    all_boxes = torch.cat(all_boxes, dim=0)\n    \n    merged_instances = merge_detections(all_boxes, all_scores, all_classes, orig_shape)\n\n    # filter by confidence score\n    pred_class = torch.mode(merged_instances.pred_classes)[0]\n    take = merged_instances.scores >= conf_thresh[pred_class]\n    merged_instances = merged_instances[take]\n    \n    list_outputs = []\n    for cfg, predictor, augmented_inputs, tfms in zip(list_configs, list_predictors, list_augmented_inputs, list_tfms):\n        augmented_instances = rescale_detected_boxes(\n            augmented_inputs, merged_instances, tfms\n        )\n                  \n        with torch.no_grad():\n            outputs = model_batch_forward(predictor.model, augmented_inputs, augmented_instances)\n        \n        del augmented_inputs, augmented_instances\n        \n        list_outputs.extend(outputs)\n        \n    list_tfms = itertools.chain.from_iterable(list_tfms)\n    \n    merged_instances.pred_masks = reduce_pred_masks(list_outputs, list_tfms)\n    merged_instances = detector_postprocess(merged_instances, *orig_shape)\n    \n    final_outputs = {'instances':merged_instances}\n    final_outputs = custom_nms(final_outputs, nms_thresh=mask_nms_thresh)\n    # final_outputs = post_process_output(final_outputs, mask_nms_thresh=mask_nms_thresh, conf_thresh=conf_thresh)\n    \n    return final_outputs","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:48:24.490831Z","iopub.execute_input":"2021-12-30T17:48:24.491482Z","iopub.status.idle":"2021-12-30T17:48:24.526639Z","shell.execute_reply.started":"2021-12-30T17:48:24.491435Z","shell.execute_reply":"2021-12-30T17:48:24.525335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg1, predictor1 = get_predictor_v21()\ncfg2, predictor2 = get_predictor_v16()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:48:24.529070Z","iopub.execute_input":"2021-12-30T17:48:24.529730Z","iopub.status.idle":"2021-12-30T17:48:44.998998Z","shell.execute_reply.started":"2021-12-30T17:48:24.529683Z","shell.execute_reply":"2021-12-30T17:48:44.997977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_cols = pd.read_csv('../input/sartorius-stage2-models/features_ver_5_ens_m1022_m1018.csv')['features'].tolist()\n\ny_col = 'FP'\n\nstage2_thresh = 0.3\n\nimport pickle\nwith open('../input/sartorius-stage2-models/model_ver_5_ens_m1022_m1018.pkl', 'rb') as f:\n    stage2_model = pickle.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:56:12.009779Z","iopub.execute_input":"2021-12-30T17:56:12.010107Z","iopub.status.idle":"2021-12-30T17:56:12.025852Z","shell.execute_reply.started":"2021-12-30T17:56:12.010076Z","shell.execute_reply":"2021-12-30T17:56:12.024560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids, masks=[],[]\ntest_names = (dataDir/'test').ls()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:56:12.203461Z","iopub.execute_input":"2021-12-30T17:56:12.203789Z","iopub.status.idle":"2021-12-30T17:56:12.211718Z","shell.execute_reply.started":"2021-12-30T17:56:12.203756Z","shell.execute_reply":"2021-12-30T17:56:12.210607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FINAL_THRESH = [0.5, 0.8, 0.8]","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:56:12.322290Z","iopub.execute_input":"2021-12-30T17:56:12.322984Z","iopub.status.idle":"2021-12-30T17:56:12.327699Z","shell.execute_reply.started":"2021-12-30T17:56:12.322947Z","shell.execute_reply":"2021-12-30T17:56:12.326484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fn in tqdm(test_names, total=len(test_names)):\n    d = {'file_name':str(fn), 'height':520, 'width':704}\n    im = cv2.imread(d['file_name'])\n    pred = ensemble(d, \n                   [cfg1, cfg2], \n                   [predictor1, predictor2],\n                  conf_thresh=FINAL_THRESH)\n    \n    pred['instances'] = pred['instances'].to('cpu')\n    \n    pred_class = torch.mode(pred['instances'].pred_classes)[0]\n    \n    # stage 2 prediction\n    features = get_features(im, pred)\n    \n    features['instance_num'] = np.arange(0, len(features))\n    features['cell_type'] = [pred_class]*len(features)\n\n    features[X_cols] = features[X_cols].fillna(-1)\n    \n    predicted_iou = stage2_model.predict(features[X_cols])\n    features['pred_FP'] = predicted_iou < stage2_thresh\n    TP_list = features[features['pred_FP']==0].instance_num.tolist()\n    pred['instances'] = pred['instances'][TP_list]\n    pred['instances'].pred_masks = (pred['instances'].pred_masks > 0.5)\n    \n    encoded_masks = get_masks(pred)\n    for enc in encoded_masks:\n        ids.append(fn.stem)\n        masks.append(enc)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:56:12.481304Z","iopub.execute_input":"2021-12-30T17:56:12.481957Z","iopub.status.idle":"2021-12-30T17:56:39.353867Z","shell.execute_reply.started":"2021-12-30T17:56:12.481903Z","shell.execute_reply":"2021-12-30T17:56:39.352893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df = pd.DataFrame({'id':ids, 'predicted':masks})\nfinal_df.to_csv('submission.csv', index=False)\nfinal_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:56:39.355952Z","iopub.execute_input":"2021-12-30T17:56:39.356871Z","iopub.status.idle":"2021-12-30T17:56:39.382257Z","shell.execute_reply.started":"2021-12-30T17:56:39.356826Z","shell.execute_reply":"2021-12-30T17:56:39.381241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = final_df.id.unique()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:56:39.384097Z","iopub.execute_input":"2021-12-30T17:56:39.384511Z","iopub.status.idle":"2021-12-30T17:56:39.390704Z","shell.execute_reply.started":"2021-12-30T17:56:39.384454Z","shell.execute_reply":"2021-12-30T17:56:39.389544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize sample\ni = 0\nfname = test_names[i]\nencoded_masks = final_df[final_df.id == ids[i]]['predicted']\nprint(len(encoded_masks))\n\n_, axs = plt.subplots(1,2, figsize=(40,15))\naxs[1].imshow(cv2.imread(str(fname)))\nfor enc in encoded_masks:\n    dec = rle_decode(enc)\n    axs[0].imshow(np.ma.masked_where(dec==0, dec))","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:56:39.393879Z","iopub.execute_input":"2021-12-30T17:56:39.394500Z","iopub.status.idle":"2021-12-30T17:57:20.813907Z","shell.execute_reply.started":"2021-12-30T17:56:39.394454Z","shell.execute_reply":"2021-12-30T17:57:20.812254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize sample\ni = 1\nfname = test_names[i]\nencoded_masks = final_df[final_df.id == ids[i]]['predicted']\nprint(len(encoded_masks))\n\n_, axs = plt.subplots(1,2, figsize=(40,15))\naxs[1].imshow(cv2.imread(str(fname)))\nfor enc in encoded_masks:\n    dec = rle_decode(enc)\n    axs[0].imshow(np.ma.masked_where(dec==0, dec))","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:57:20.815661Z","iopub.execute_input":"2021-12-30T17:57:20.816293Z","iopub.status.idle":"2021-12-30T17:57:37.797903Z","shell.execute_reply.started":"2021-12-30T17:57:20.816234Z","shell.execute_reply":"2021-12-30T17:57:37.796359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize sample\ni = 2\nfname = test_names[i]\nencoded_masks = final_df[final_df.id == ids[i]]['predicted']\nprint(len(encoded_masks))\n\n_, axs = plt.subplots(1,2, figsize=(40,15))\naxs[1].imshow(cv2.imread(str(fname)))\nfor enc in encoded_masks:\n    dec = rle_decode(enc)\n    axs[0].imshow(np.ma.masked_where(dec==0, dec))","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:57:37.799468Z","iopub.execute_input":"2021-12-30T17:57:37.800501Z","iopub.status.idle":"2021-12-30T17:57:52.192389Z","shell.execute_reply.started":"2021-12-30T17:57:37.800456Z","shell.execute_reply":"2021-12-30T17:57:52.190752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2021-12-30T17:57:52.194149Z","iopub.execute_input":"2021-12-30T17:57:52.194814Z","iopub.status.idle":"2021-12-30T17:57:53.185643Z","shell.execute_reply.started":"2021-12-30T17:57:52.194725Z","shell.execute_reply":"2021-12-30T17:57:53.184379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}