{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Ensemble NMS - Detectron2 [Inference]","metadata":{"papermill":{"duration":0.025648,"end_time":"2021-12-11T06:57:37.617023","exception":false,"start_time":"2021-12-11T06:57:37.591375","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Hi kagglers, This is `Ensemble NMW - Detectron2 [Inference]` notebook.\n* [Sartorius Segmentation - Detectron2 [training]](https://www.kaggle.com/ammarnassanalhajali/sartorius-segmentation-detectron2-training) \n* [Sartorius Segmentation - Detectron2 [Inference]](https://www.kaggle.com/ammarnassanalhajali/sartorius-segmentation-detectron2-inference) \n* [K-fold CrossValidation COCO Dataset Generator](https://www.kaggle.com/ammarnassanalhajali/k-fold-crossvalidation-coco-dataset-generator) \n\n\n### Please if this kernel is useful, <font color='red'>please upvote !!</font>","metadata":{"papermill":{"duration":0.013174,"end_time":"2021-12-11T06:57:37.649999","exception":false,"start_time":"2021-12-11T06:57:37.636825","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Other notebooks in this competition \n- [Sartorius Segmentation - Keras U-Net[Training]](https://www.kaggle.com/ammarnassanalhajali/sartorius-segmentation-keras-u-net-training)\n- [Sartorius Segmentation - Keras U-Net[Inference]](https://www.kaggle.com/ammarnassanalhajali/sartorius-segmentation-keras-u-net-inference/edit)","metadata":{"papermill":{"duration":0.012917,"end_time":"2021-12-11T06:57:37.676029","exception":false,"start_time":"2021-12-11T06:57:37.663112","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Intro\nEnsembling multiple weaker performing models can help to get the results that you want.","metadata":{"papermill":{"duration":0.013112,"end_time":"2021-12-11T06:57:37.702414","exception":false,"start_time":"2021-12-11T06:57:37.689302","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Install and import libraries","metadata":{"papermill":{"duration":0.013137,"end_time":"2021-12-11T06:57:37.728835","exception":false,"start_time":"2021-12-11T06:57:37.715698","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install ../input/detectron-05/whls/pycocotools-2.0.2/dist/pycocotools-2.0.2.tar --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/fvcore-0.1.5.post20211019/fvcore-0.1.5.post20211019 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/antlr4-python3-runtime-4.8/antlr4-python3-runtime-4.8 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/detectron2-0.5/detectron2 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/ensemble-boxes-104/ensemble_boxes-1.0.4/ -f ./ --no-index","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":214.247843,"end_time":"2021-12-11T07:01:11.990284","exception":false,"start_time":"2021-12-11T06:57:37.742441","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-22T14:29:13.546841Z","iopub.execute_input":"2021-12-22T14:29:13.547261Z","iopub.status.idle":"2021-12-22T14:32:28.386725Z","shell.execute_reply.started":"2021-12-22T14:29:13.547168Z","shell.execute_reply":"2021-12-22T14:32:28.385761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport json\nimport time\nimport numpy as np\nimport pandas as pd\nimport torch\nimport detectron2\nfrom tqdm.auto import tqdm\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.data.datasets import register_coco_instances\nfrom detectron2.evaluation import inference_on_dataset\nfrom detectron2.evaluation.evaluator import DatasetEvaluator\nfrom detectron2.data import DatasetCatalog, build_detection_test_loader\nimport pycocotools.mask as mask_util\nfrom skimage import morphology\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom fastcore.all import *\nfrom ensemble_boxes import *\nos.environ['CUDA_VISIBLE_DEVICES'] = '0' \nif torch.cuda.is_available():\n    DEVICE = torch.device('cuda')\n    print('GPU is available')\nelse:\n    DEVICE = torch.device('cpu')\n    print('CPU is used')\nprint('detectron ver:', detectron2.__version__)","metadata":{"papermill":{"duration":1.794759,"end_time":"2021-12-11T07:01:13.818453","exception":false,"start_time":"2021-12-11T07:01:12.023694","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-22T14:32:28.388321Z","iopub.execute_input":"2021-12-22T14:32:28.388597Z","iopub.status.idle":"2021-12-22T14:32:29.930864Z","shell.execute_reply.started":"2021-12-22T14:32:28.388568Z","shell.execute_reply":"2021-12-22T14:32:29.930077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## My Models","metadata":{"papermill":{"duration":0.033373,"end_time":"2021-12-11T07:01:13.885547","exception":false,"start_time":"2021-12-11T07:01:13.852174","status":"completed"},"tags":[]}},{"cell_type":"code","source":"best_model=(\n    {'file': 'model_best_32768_thr_35_55_75_CV32593.pth','config_name':'Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv.yaml', 'LB score': 0.338,'ths':[0.35, 0.55, 0.75]},\n    {'file': 'model_cv3285_ths35_55_70.pth','config_name':'Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv.yaml', 'LB score': 0.337,'ths':[0.35, 0.55, 0.7]},\n#     {'file': 'model_cv3278_ths5_65_75.pth','config_name':'Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv.yaml', 'LB score': 0.337,'ths':[0.5, 0.65, 0.75]},\n#     {'file': 'model_best_32961_thr_35_5_8_CV32748.pth','config_name':'Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv.yaml', 'LB score': 0.337,'ths':[0.35, 0.5, 0.8]},\n#     {'file': 'semi_r50_LB334_cv3234_thr26_38_61.pth','config_name':'Misc/cascade_mask_rcnn_R_50_FPN_3x.yaml', 'LB score': 0.334,'ths':[0.26, 0.38, 0.61]}\n            )","metadata":{"papermill":{"duration":0.041421,"end_time":"2021-12-11T07:01:13.960561","exception":false,"start_time":"2021-12-11T07:01:13.91914","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-22T14:32:29.932295Z","iopub.execute_input":"2021-12-22T14:32:29.932753Z","iopub.status.idle":"2021-12-22T14:32:29.937798Z","shell.execute_reply.started":"2021-12-22T14:32:29.932703Z","shell.execute_reply":"2021-12-22T14:32:29.936966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#config_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\nmdl_path = \"../input/det2r152\"\nDATA_PATH = \"../input/sartorius-cell-instance-segmentation\"\nMODELS = []\nBEST_MODELS =[]\nTHSS = []\nID_TEST = 0\nSUBM_PATH = f'{DATA_PATH}/test'\nSINGLE_MODE = False\nNMS = True\n# MIN_PIXELS = [75, 150, 75]\nMIN_PIXELS = [59, 136, 74]\nIOU_TH = .4\n\nfor b_m in best_model:\n    model_name=b_m[\"file\"]\n    model_ths=b_m[\"ths\"]\n    config_name=b_m[\"config_name\"]\n    BEST_MODELS.append(model_name)\n    THSS.append(model_ths)\n    cfg = get_cfg()\n    cfg.merge_from_file(model_zoo.get_config_file(config_name))\n    cfg.INPUT.MASK_FORMAT = 'bitmask'\n    \n    cfg.MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS = [[0.2, 0.5, 1.0, 2.0, 5.0]]  # [[0.5, 1.0, 2.0]]\n    cfg.MODEL.ANCHOR_GENERATOR.SIZES = [[9], [18],[64], [120], [240]]\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 3\n    cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 1024\n    cfg.TEST.DETECTIONS_PER_IMAGE = 1000\n    cfg.INPUT.MIN_SIZE_TEST = 1040\n#     cfg.INPUT.MIN_SIZE_TEST = 1300\n    cfg.INPUT.MAX_SIZE_TEST = 1408\n#     cfg.INPUT.MAX_SIZE_TEST = 1760\n    if 'semi_r50_LB334_cv3234_thr26_38_61' in model_name:\n        cfg.MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS = [[0.25, 0.5, 1.0, 2.0, 4.0]]\n        cfg.MODEL.ANCHOR_GENERATOR.SIZES = [[18], [34], [64], [128], [256]]\n        cfg.INPUT.MIN_SIZE_TEST = 800\n        cfg.INPUT.MAX_SIZE_TEST = 1333\n        cfg.MODEL.WEIGHTS = '../input/det2r152/semi_r50_LB334_cv3234_thr26_38_61.pth'\n    if 'model_cv3285_ths35_55_70' in model_name:\n        cfg.MODEL.RESNETS.DEPTH = 101\n        cfg.INPUT.MAX_SIZE_TEST= 1440\n        cfg.INPUT.MIN_SIZE_TEST= 1040\n#         cfg.INPUT.MAX_SIZE_TEST= 1760\n#         cfg.INPUT.MIN_SIZE_TEST= 1300\n        cfg.MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS = [[0.2, 0.5, 1.0, 2.0, 5.0]]  # [[0.5, 1.0, 2.0]]\n        cfg.MODEL.ANCHOR_GENERATOR.SIZES = [[9], [17], [31], [64], [127]]\n        cfg.MODEL.WEIGHTS = '../input/semidet2x152/semi_101/semi_puck_0439999/model_cv3285_ths35_55_70.pth'\n    if 'model_best_32768_thr_35_55_75_CV32593' in model_name:\n        cfg.MODEL.WEIGHTS = '../input/det2r152/model_best_32768_thr_35_55_75_CV32593.pth'\n    if 'model_best_32961_thr_35_5_8_CV32748' in model_name:\n        cfg.MODEL.WEIGHTS = '../input/det2r152/model_best_32961_thr_35_5_8_CV32748.pth'    \n    if 'model_best_32845_thr_35_5_75_CV32724' in model_name:\n        cfg.MODEL.WEIGHTS = '../input/det2r152/model_best_32845_thr_35_5_75_CV32724.pth'\n    if 'model_cv3278_ths5_65_75' in model_name:\n        cfg.MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS = [[0.2, 0.5, 1.0, 2.0, 5.0]]  # [[0.5, 1.0, 2.0]]\n        cfg.MODEL.ANCHOR_GENERATOR.SIZES = [[9], [17], [31], [64], [127]]\n        cfg.MODEL.WEIGHTS = '../input/semidet2x152/semi_new_anchor_1212/semi_new_1212_cmp_0/model_cv3278_ths5_65_75.pth'\n    if 'model_best_33111_thr_45_5_8_CV32863' in model_name:\n        cfg.MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS = [[0.25, 0.5, 1.0, 2.0, 4.0]]  # [[0.5, 1.0, 2.0]]\n        cfg.MODEL.ANCHOR_GENERATOR.SIZES = [[18], [34], [64], [128], [256]]\n        cfg.MODEL.WEIGHTS = '../input/det2r152/model_best_33111_thr_45_5_8_CV32863.pth'\n    \n#     cfg.MODEL.WEIGHTS = f'{mdl_path}/{model_name}'  \n    MODELS.append(DefaultPredictor(cfg))\nprint(f'all loaded:\\nthresholds: {THSS}\\nmodels: {BEST_MODELS}')","metadata":{"papermill":{"duration":13.975435,"end_time":"2021-12-11T07:01:27.969039","exception":false,"start_time":"2021-12-11T07:01:13.993604","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-22T14:32:29.941722Z","iopub.execute_input":"2021-12-22T14:32:29.94212Z","iopub.status.idle":"2021-12-22T14:32:37.867932Z","shell.execute_reply.started":"2021-12-22T14:32:29.942083Z","shell.execute_reply":"2021-12-22T14:32:37.867133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODELS","metadata":{"papermill":{"duration":0.051272,"end_time":"2021-12-11T07:01:28.058814","exception":false,"start_time":"2021-12-11T07:01:28.007542","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-22T14:32:37.869419Z","iopub.execute_input":"2021-12-22T14:32:37.86983Z","iopub.status.idle":"2021-12-22T14:32:37.883292Z","shell.execute_reply.started":"2021-12-22T14:32:37.869792Z","shell.execute_reply":"2021-12-22T14:32:37.87944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utils","metadata":{"papermill":{"duration":0.038002,"end_time":"2021-12-11T07:01:28.134842","exception":false,"start_time":"2021-12-11T07:01:28.09684","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def rle_decode(mask_rle, shape=(520, 704)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) \n                       for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo : hi] = 1\n    return img.reshape(shape)  # Needed to align to RLE direction\n\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    \n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef pred_masks(file_name, path, model, ths, min_pixels):\n    img = cv2.imread(f'{path}/{file_name}')\n    output = model(img)\n    pred_classes = output['instances'].pred_classes.cpu().numpy().tolist()\n    pred_class = max(set(pred_classes), key=pred_classes.count)\n    take = output['instances'].scores >= ths[pred_class]\n    pred_masks = output['instances'].pred_masks[take]\n    pred_masks = pred_masks.cpu().numpy()\n    result = []\n    used = np.zeros(img.shape[:2], dtype=int) \n    for i, mask in enumerate(pred_masks):\n        mask = mask * (1 - used)\n        if mask.sum() >= min_pixels[pred_class]:\n            used += mask\n            result.append(rle_encode(mask))\n    return result\n\ndef ensemble_preds(file_name, path, models, ths):\n    img = cv2.imread(f'{path}/{file_name}')\n    classes = []\n    scores = []\n    bboxes = []\n    masks = []\n    for i, model in enumerate(models):\n        output = model(img)\n        pred_classes = output['instances'].pred_classes.cpu().numpy().tolist()\n        pred_class = max(set(pred_classes), key=pred_classes.count)\n        take = output['instances'].scores >= ths[i][pred_class]\n        classes.extend(output['instances'].pred_classes[take].cpu().numpy().tolist())\n        scores.extend(output['instances'].scores[take].cpu().numpy().tolist())\n        bboxes.extend(output['instances'].pred_boxes[take].tensor.cpu().numpy().tolist())\n        masks.extend(output['instances'].pred_masks[take].cpu().numpy())\n    assert len(classes) == len(masks) , 'ensemble lenght mismatch'\n    #scores, classes, bboxes, masks = zip(*sorted(zip(scores, classes, bboxes, masks),reverse=True))\n    return classes, scores, bboxes, masks\n\ndef nms_predictions(classes, scores, bboxes, masks, \n                    iou_th=.5, shape=(520, 704)):\n    he, wd = shape[0], shape[1]\n    boxes_list = [[[x[0] / wd, x[1] / he, x[2] / wd, x[3] / he] for x in bboxes]]\n    scores_list = [[x for x in scores]]\n    classes_list = [[x for x in classes]]\n    nms_bboxes, nms_scores, nms_classes = non_maximum_weighted(\n        boxes_list, \n        scores_list, \n        classes_list, \n        weights=None,\n        iou_thr=0.5,skip_box_thr=0.0001  \n    )\n    nms_masks = []\n    for s in nms_scores:\n        nms_masks.append(masks[scores.index(s)])\n    nms_scores, nms_classes, nms_masks = zip(*sorted(zip(nms_scores, nms_classes, nms_masks), reverse=True))\n    return nms_classes, nms_scores, nms_masks\n\ndef ensemble_pred_masks(masks, classes, min_pixels, shape=(520, 704)):\n    result = []\n    #pred_class = max(set(classes), key=classes.count)\n    pred_class = int(max(set(classes), key=classes.count).item())\n    used = np.zeros(shape, dtype=int) \n    for i, mask in enumerate(masks):\n        mask = mask * (1 - used)\n        if mask.sum() >= min_pixels[pred_class]:\n            used += mask\n            result.append(rle_encode(mask))\n    return result\n\ndef ensemble_pred_masks_val(masks, classes, min_pixels, shape=(520, 704)):\n    result = []\n    #pred_class = max(set(classes), key=classes.count)\n    pred_class = int(max(set(classes), key=classes.count).item())\n    used = np.zeros(shape, dtype=int) \n    for i, mask in enumerate(masks):\n        mask = mask * (1 - used)\n        if mask.sum() >= min_pixels[pred_class]:\n            used += mask\n            result.append(mask)\n    return result","metadata":{"papermill":{"duration":0.073286,"end_time":"2021-12-11T07:01:28.247215","exception":false,"start_time":"2021-12-11T07:01:28.173929","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-22T14:32:37.88735Z","iopub.execute_input":"2021-12-22T14:32:37.887866Z","iopub.status.idle":"2021-12-22T14:32:37.957625Z","shell.execute_reply.started":"2021-12-22T14:32:37.887827Z","shell.execute_reply":"2021-12-22T14:32:37.956647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://github.com/MouseLand/cellpose/blob/master/cellpose/utils.py\n# https://scikit-image.org/docs/dev/api/skimage.morphology.html#skimage.morphology.remove_small_holes\ndef fill_small_hole(mask):\n    hsz = np.count_nonzero(mask)*HOLE_SIZE_PERCENT #turn hole size into percentage\n    padmsk = morphology.remove_small_holes(mask,hsz)\n    return padmsk","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:32:37.961752Z","iopub.execute_input":"2021-12-22T14:32:37.96454Z","iopub.status.idle":"2021-12-22T14:32:37.976115Z","shell.execute_reply.started":"2021-12-22T14:32:37.964493Z","shell.execute_reply":"2021-12-22T14:32:37.973691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BOXES_THRESHOLDS_VOTE = [0.1, 0.1, 0.1]\nIOU_THRESHOLDS_VOTE = [0.8,0.8,0.8]\n# MASK_THRESHOLDS_VOTE = [0.4, 0.75, 0.75]\nMASK_THRESHOLDS_VOTE = [0.1, 0.1, 0.1]\nCELL_TYPES = {0: 'shsy5y', 1: 'astro', 2: 'cort'}\nHOLE_SIZE_PERCENT = 0.1","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:57:20.310587Z","iopub.execute_input":"2021-12-22T14:57:20.311485Z","iopub.status.idle":"2021-12-22T14:57:20.317892Z","shell.execute_reply.started":"2021-12-22T14:57:20.311446Z","shell.execute_reply":"2021-12-22T14:57:20.317064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_iou(box, boxes):\n    # get area of a\n    area_a = (box[3]-box[1])*(box[2]-box[0])\n    # get area of b\n    area_b = (boxes[:,3]-boxes[:,1])*(boxes[:,2]-boxes[:,0])\n\n    # get left top x of IoU\n    iou_x1 = np.maximum(boxes[:,0], box[0])\n    # get left top y of IoU\n    iou_y1 = np.maximum(boxes[:,1], box[1])\n    # get right bottom of IoU\n    iou_x2 = np.minimum(boxes[:,2], box[2])\n    # get right bottom of IoU\n    iou_y2 = np.minimum(boxes[:,3], box[3])\n\n    # get width of IoU\n    iou_w = iou_x2 - iou_x1\n    iou_w = np.maximum(iou_w, 0)\n    # get height of IoU\n    iou_h = iou_y2 - iou_y1\n    iou_h = np.maximum(iou_h, 0)\n    \n    # get area of IoU\n    area_iou = iou_w * iou_h\n    # get overlap ratio between IoU and all area\n    iou = area_iou / (area_a + area_b - area_iou+1e-6)\n    return iou\n    \ndef process_preds_det2(pred):\n    pred_class = torch.mode(pred['instances'].pred_classes)[0]\n    take = pred['instances'].scores >= BOXES_THRESHOLDS_VOTE[pred_class]\n    pred_masks = pred['instances'].pred_masks[take]\n    mask_scores = pred['instances'].scores[take].cpu().numpy()\n    boxes = pred['instances'].pred_boxes[take].tensor.cpu().numpy()\n    pred_masks = pred_masks.cpu().numpy()\n    pred_masks = mask_scores[:, np.newaxis, np.newaxis] * pred_masks\n    nbox = len(boxes)\n    out_masks = []\n    for idx in range(nbox):\n        box = boxes[idx]\n        iou = compute_iou(box, boxes)\n        take = iou>IOU_THRESHOLDS_VOTE[pred_class]\n        mask = pred_masks[take].mean(axis=0)>=MASK_THRESHOLDS_VOTE[pred_class]\n        mask = fill_small_hole(mask)\n        out_masks.append(mask)\n    return out_masks\n\ndef process_masks(boxes, pred_masks, mask_scores, classes):\n    mask_scores = np.array(mask_scores)\n    boxes = np.array(boxes)\n    pred_masks = mask_scores[:, np.newaxis, np.newaxis] * pred_masks\n    nbox = len(boxes)\n    out_masks = []\n    for idx in range(nbox):\n        box = boxes[idx]\n        pred_class = classes[idx]\n        iou = compute_iou(box, boxes)\n        take = iou>IOU_THRESHOLDS_VOTE[pred_class]\n        mask = pred_masks[take].mean(axis=0)>=MASK_THRESHOLDS_VOTE[pred_class]\n        mask = fill_small_hole(mask)\n        out_masks.append(mask)\n    return out_masks\ndef get_bounding_box(masks):\n    bboxes = []\n    for mask in masks:\n        yindices = np.where(np.any(mask, axis=0))[0]\n        xindices = np.where(np.any(mask, axis=1))[0]\n        if yindices.shape[0]:\n            y1, y2 = yindices[[0, -1]]\n            x1, x2 = xindices[[0, -1]]\n            y2 += 1\n            x2 += 1\n        else:\n            y1, x1, y2, x2 = 0, 0, 0, 0\n        bboxes.append((y1, x1, y2, x2))\n    return bboxes\n\ndef nms_predictions_votes(classes, scores, bboxes, masks, \n                    iou_th=.5, shape=(520, 704)):\n    # preprocess\n    masks = process_masks(bboxes, masks, scores, classes)\n    bboxes = get_bounding_box(masks)\n    he, wd = shape[0], shape[1]\n    boxes_list = [[[x[0] / wd, x[1] / he, x[2] / wd, x[3] / he] for x in bboxes]]\n    scores_list = [[x for x in scores]]\n    classes_list = [[x for x in classes]]\n    nms_bboxes, nms_scores, nms_classes = non_maximum_weighted(\n        boxes_list, \n        scores_list, \n        classes_list, \n        weights=None,\n        iou_thr=0.3,skip_box_thr=0.0001  \n    )\n    nms_masks = []\n    for s in nms_scores:\n        nms_masks.append(masks[scores.index(s)])\n    nms_scores, nms_classes, nms_masks = zip(*sorted(zip(nms_scores, nms_classes, nms_masks), reverse=True))\n    return nms_classes, nms_scores, nms_masks","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:59:49.661975Z","iopub.execute_input":"2021-12-22T14:59:49.662478Z","iopub.status.idle":"2021-12-22T14:59:49.687337Z","shell.execute_reply.started":"2021-12-22T14:59:49.662441Z","shell.execute_reply":"2021-12-22T14:59:49.686094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nms_predictions = nms_predictions_votes","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:57:28.696255Z","iopub.execute_input":"2021-12-22T14:57:28.696813Z","iopub.status.idle":"2021-12-22T14:57:28.700881Z","shell.execute_reply.started":"2021-12-22T14:57:28.696776Z","shell.execute_reply":"2021-12-22T14:57:28.699915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## NMS CV","metadata":{"papermill":{"duration":0.040295,"end_time":"2021-12-11T07:01:28.329733","exception":false,"start_time":"2021-12-11T07:01:28.289438","status":"completed"},"tags":[]}},{"cell_type":"code","source":"register_coco_instances('sartorius_val',{},'../input/jsonsartorius/3classes/annotations_val.json', \n                        '../input/sartorius-cell-instance-segmentation/')\n\nval_ds = DatasetCatalog.get('sartorius_val')\nVAL_PATH = f'{DATA_PATH}/train'","metadata":{"papermill":{"duration":0.054704,"end_time":"2021-12-11T07:01:28.4275","exception":false,"start_time":"2021-12-11T07:01:28.372796","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-22T14:42:29.073462Z","iopub.execute_input":"2021-12-22T14:42:29.074041Z","iopub.status.idle":"2021-12-22T14:42:29.259317Z","shell.execute_reply.started":"2021-12-22T14:42:29.07401Z","shell.execute_reply":"2021-12-22T14:42:29.258267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def precision_at(threshold, iou):\n    matches = iou > threshold\n    true_positives = np.sum(matches, axis=1) == 1  # Correct objects\n    false_positives = np.sum(matches, axis=0) == 0  # Missed objects\n    false_negatives = np.sum(matches, axis=1) == 0  # Extra objects\n    return np.sum(true_positives), np.sum(false_positives), np.sum(false_negatives)\n\n\n# for val_name in tqdm(val_names):\ndef nms_cv_score():\n    After_nms_CV = []\n    for item in val_ds:\n    #     im =  cv2.imread(item['file_name'].split('/')[4])\n        classes, scores, bboxes, masks = ensemble_preds(\n            file_name=item['file_name'].split('/')[4], \n            path=VAL_PATH, \n            models=MODELS, \n            ths=THSS\n        )\n        if NMS:\n            classes, scores, masks = nms_predictions(\n                classes, \n                scores, \n                bboxes, \n                masks, \n                iou_th=IOU_TH\n            )\n\n        encoded_masks = ensemble_pred_masks_val(\n            masks, \n            classes, \n            min_pixels=MIN_PIXELS\n        )\n\n        enc_preds = [mask_util.encode(np.asarray(p, order='F', dtype='uint8')) for p in encoded_masks]\n        enc_targs = list(map(lambda x:x['segmentation'], item['annotations']))\n        ious = mask_util.iou(enc_preds, enc_targs, [0]*len(enc_targs))\n        prec = []\n        for t in np.arange(0.5, 1.0, 0.05):\n            tp, fp, fn = precision_at(t, ious)\n            p = tp / (tp + fp + fn)\n            prec.append(p)\n        mean_pre = np.mean(prec)\n        After_nms_CV.append(mean_pre)\n    \n    return np.mean(After_nms_CV)","metadata":{"execution":{"iopub.status.busy":"2021-12-22T15:00:03.766214Z","iopub.execute_input":"2021-12-22T15:00:03.766848Z","iopub.status.idle":"2021-12-22T15:00:03.778355Z","shell.execute_reply.started":"2021-12-22T15:00:03.766808Z","shell.execute_reply":"2021-12-22T15:00:03.777254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"After_nms_CV_all = nms_cv_score()\nAfter_nms_CV_all","metadata":{"execution":{"iopub.status.busy":"2021-12-22T15:00:07.199847Z","iopub.execute_input":"2021-12-22T15:00:07.200427Z","iopub.status.idle":"2021-12-22T15:06:57.785991Z","shell.execute_reply.started":"2021-12-22T15:00:07.200372Z","shell.execute_reply":"2021-12-22T15:06:57.78525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_names = os.listdir(SUBM_PATH)\nprint('test images:', len(test_names))","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:42:29.264828Z","iopub.status.idle":"2021-12-22T14:42:29.26529Z","shell.execute_reply.started":"2021-12-22T14:42:29.265054Z","shell.execute_reply":"2021-12-22T14:42:29.265082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_names","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:42:29.283317Z","iopub.execute_input":"2021-12-22T14:42:29.283517Z","iopub.status.idle":"2021-12-22T14:42:29.288769Z","shell.execute_reply.started":"2021-12-22T14:42:29.283494Z","shell.execute_reply":"2021-12-22T14:42:29.288014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{"papermill":{"duration":0.038012,"end_time":"2021-12-11T07:01:55.642891","exception":false,"start_time":"2021-12-11T07:01:55.604879","status":"completed"},"tags":[]}},{"cell_type":"code","source":"subm_ids, subm_masks = [], []\nshow_img =[]\ni=0\nfor test_name in tqdm(test_names):\n    if SINGLE_MODE:\n        encoded_masks = pred_masks(\n            test_name, \n            path=SUBM_PATH, \n            model=MODELS[0],\n            ths=THSS[0],\n            min_pixels=MIN_PIXELS\n        )\n    else:\n        classes, scores, bboxes, masks = ensemble_preds(\n            file_name=test_name, \n            path=SUBM_PATH, \n            models=MODELS, \n            ths=THSS\n        )\n        if NMS:\n            classes, scores, masks = nms_predictions(\n                classes, \n                scores, \n                bboxes, \n                masks, \n                iou_th=IOU_TH\n            )\n        encoded_masks = ensemble_pred_masks(\n            masks, \n            classes, \n            min_pixels=MIN_PIXELS\n        )\n    if i<4:\n        i+=1\n        show_img.append(encoded_masks)\n    for enc_mask in encoded_masks:\n        subm_ids.append(test_name[:test_name.find('.')])\n        subm_masks.append(enc_mask)","metadata":{"papermill":{"duration":2.980903,"end_time":"2021-12-11T07:01:58.661498","exception":false,"start_time":"2021-12-11T07:01:55.680595","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-22T14:57:36.977103Z","iopub.execute_input":"2021-12-22T14:57:36.977371Z","iopub.status.idle":"2021-12-22T14:57:46.504353Z","shell.execute_reply.started":"2021-12-22T14:57:36.977341Z","shell.execute_reply":"2021-12-22T14:57:46.503601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataDir=Path('../input/sartorius-cell-instance-segmentation')\ntest_image_names = (dataDir/'test').ls()\nfor j in range(3):\n    _, axs = plt.subplots(1,2, figsize=(40,30))\n    axs[1].imshow(cv2.imread(str(test_image_names[j])))\n    axs[1].axis(\"off\")\n    mask = np.zeros((520, 704, 1))\n    encoded_masks = show_img[j]\n    for enc in encoded_masks:\n        mask += rle_decode(enc, shape=(520, 704, 1))\n    \n    mask = mask.clip(0, 1)\n    axs[0].imshow(mask)\n    axs[0].axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:57:48.801764Z","iopub.execute_input":"2021-12-22T14:57:48.802505Z","iopub.status.idle":"2021-12-22T14:57:51.464438Z","shell.execute_reply.started":"2021-12-22T14:57:48.802453Z","shell.execute_reply":"2021-12-22T14:57:51.46379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame({\n    'id': subm_ids, \n    'predicted': subm_masks\n}).to_csv('submission.csv', index=False)\npd.read_csv('submission.csv').head()","metadata":{"papermill":{"duration":0.131584,"end_time":"2021-12-11T07:01:58.870796","exception":false,"start_time":"2021-12-11T07:01:58.739212","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-22T14:42:41.793854Z","iopub.execute_input":"2021-12-22T14:42:41.794304Z","iopub.status.idle":"2021-12-22T14:42:41.821025Z","shell.execute_reply.started":"2021-12-22T14:42:41.79426Z","shell.execute_reply":"2021-12-22T14:42:41.820298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# References\n1. https://www.kaggle.com/vgarshin/detectron2-inference-with-ensemble-and-nms","metadata":{"papermill":{"duration":0.069861,"end_time":"2021-12-11T07:01:59.019808","exception":false,"start_time":"2021-12-11T07:01:58.949947","status":"completed"},"tags":[]}}]}