{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Ensemble NMS - Detectron2 [Inference]","metadata":{"papermill":{"duration":0.025648,"end_time":"2021-12-11T06:57:37.617023","exception":false,"start_time":"2021-12-11T06:57:37.591375","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Hi kagglers, This is `Ensemble NMW - Detectron2 [Inference]` notebook.\n* [Sartorius Segmentation - Detectron2 [training]](https://www.kaggle.com/ammarnassanalhajali/sartorius-segmentation-detectron2-training) \n* [Sartorius Segmentation - Detectron2 [Inference]](https://www.kaggle.com/ammarnassanalhajali/sartorius-segmentation-detectron2-inference) \n* [K-fold CrossValidation COCO Dataset Generator](https://www.kaggle.com/ammarnassanalhajali/k-fold-crossvalidation-coco-dataset-generator) \n\n\n### Please if this kernel is useful, <font color='red'>please upvote !!</font>","metadata":{"papermill":{"duration":0.013174,"end_time":"2021-12-11T06:57:37.649999","exception":false,"start_time":"2021-12-11T06:57:37.636825","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Other notebooks in this competition \n- [Sartorius Segmentation - Keras U-Net[Training]](https://www.kaggle.com/ammarnassanalhajali/sartorius-segmentation-keras-u-net-training)\n- [Sartorius Segmentation - Keras U-Net[Inference]](https://www.kaggle.com/ammarnassanalhajali/sartorius-segmentation-keras-u-net-inference/edit)","metadata":{"papermill":{"duration":0.012917,"end_time":"2021-12-11T06:57:37.676029","exception":false,"start_time":"2021-12-11T06:57:37.663112","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Intro\nEnsembling multiple weaker performing models can help to get the results that you want.","metadata":{"papermill":{"duration":0.013112,"end_time":"2021-12-11T06:57:37.702414","exception":false,"start_time":"2021-12-11T06:57:37.689302","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Install and import libraries","metadata":{"papermill":{"duration":0.013137,"end_time":"2021-12-11T06:57:37.728835","exception":false,"start_time":"2021-12-11T06:57:37.715698","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install ../input/detectron-05/whls/pycocotools-2.0.2/dist/pycocotools-2.0.2.tar --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/fvcore-0.1.5.post20211019/fvcore-0.1.5.post20211019 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/antlr4-python3-runtime-4.8/antlr4-python3-runtime-4.8 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/detectron2-0.5/detectron2 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/ensemble-boxes-104/ensemble_boxes-1.0.4/ -f ./ --no-index","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":214.247843,"end_time":"2021-12-11T07:01:11.990284","exception":false,"start_time":"2021-12-11T06:57:37.742441","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-19T17:46:56.75828Z","iopub.execute_input":"2021-12-19T17:46:56.758645Z","iopub.status.idle":"2021-12-19T17:50:30.404233Z","shell.execute_reply.started":"2021-12-19T17:46:56.75856Z","shell.execute_reply":"2021-12-19T17:50:30.403354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport json\nimport time\nimport numpy as np\nimport pandas as pd\nimport torch\nimport detectron2\nfrom tqdm.auto import tqdm\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.data.datasets import register_coco_instances\nfrom detectron2.evaluation import inference_on_dataset\nfrom detectron2.evaluation.evaluator import DatasetEvaluator\nfrom detectron2.data import DatasetCatalog, build_detection_test_loader\nimport pycocotools.mask as mask_util\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom fastcore.all import *\nfrom ensemble_boxes import *\nos.environ['CUDA_VISIBLE_DEVICES'] = '0' \nif torch.cuda.is_available():\n    DEVICE = torch.device('cuda')\n    print('GPU is available')\nelse:\n    DEVICE = torch.device('cpu')\n    print('CPU is used')\nprint('detectron ver:', detectron2.__version__)","metadata":{"papermill":{"duration":1.794759,"end_time":"2021-12-11T07:01:13.818453","exception":false,"start_time":"2021-12-11T07:01:12.023694","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-19T17:50:30.408167Z","iopub.execute_input":"2021-12-19T17:50:30.408414Z","iopub.status.idle":"2021-12-19T17:50:32.260499Z","shell.execute_reply.started":"2021-12-19T17:50:30.408384Z","shell.execute_reply":"2021-12-19T17:50:32.259557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## My Models","metadata":{"papermill":{"duration":0.033373,"end_time":"2021-12-11T07:01:13.885547","exception":false,"start_time":"2021-12-11T07:01:13.852174","status":"completed"},"tags":[]}},{"cell_type":"code","source":"best_model=(\n    {'file': 'R50-306.pth','config_name':'COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml', 'LB score': 0.306,'ths':[.18, .38, .58]},\n    {'file': '50_FPN_3x_F3_R82_300.pth','config_name':'COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml', 'LB score': 0.300,'ths':[.18, .38, .58]},\n    {'file': '32x8d_FPN_3x_F3_R57_295.pth','config_name':'COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x.yaml', 'LB score': 0.295,'ths':[.18, .38, .58]},\n    {'file': '50_FPN_3x_F5_ATTT32_300.pth','config_name':'COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml', 'LB score': 0.300,'ths':[.19, .39, .57]}\n            )","metadata":{"papermill":{"duration":0.041421,"end_time":"2021-12-11T07:01:13.960561","exception":false,"start_time":"2021-12-11T07:01:13.91914","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-19T17:50:32.261813Z","iopub.execute_input":"2021-12-19T17:50:32.26219Z","iopub.status.idle":"2021-12-19T17:50:32.270382Z","shell.execute_reply.started":"2021-12-19T17:50:32.26215Z","shell.execute_reply":"2021-12-19T17:50:32.269714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#config_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\nmdl_path = \"../input/dtectron2-models-5fold\"\nDATA_PATH = \"../input/sartorius-cell-instance-segmentation\"\nMODELS = []\nBEST_MODELS =[]\nTHSS = []\nID_TEST = 0\nSUBM_PATH = f'{DATA_PATH}/test'\nSINGLE_MODE = False\nNMS = True\nMIN_PIXELS = [75, 150, 75]\nIOU_TH = 0.3\nfor b_m in best_model:\n    model_name=b_m[\"file\"]\n    model_ths=b_m[\"ths\"]\n    config_name=b_m[\"config_name\"]\n    BEST_MODELS.append(model_name)\n    THSS.append(model_ths)\n    cfg = get_cfg()\n    cfg.merge_from_file(model_zoo.get_config_file(config_name))\n    cfg.INPUT.MASK_FORMAT = 'bitmask'\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 3 \n    cfg.MODEL.WEIGHTS = f'{mdl_path}/{model_name}'  \n    cfg.TEST.DETECTIONS_PER_IMAGE = 1000\n    MODELS.append(DefaultPredictor(cfg))\nprint(f'all loaded:\\nthresholds: {THSS}\\nmodels: {BEST_MODELS}')","metadata":{"papermill":{"duration":13.975435,"end_time":"2021-12-11T07:01:27.969039","exception":false,"start_time":"2021-12-11T07:01:13.993604","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-19T17:50:32.273129Z","iopub.execute_input":"2021-12-19T17:50:32.273724Z","iopub.status.idle":"2021-12-19T17:50:46.141483Z","shell.execute_reply.started":"2021-12-19T17:50:32.273687Z","shell.execute_reply":"2021-12-19T17:50:46.140649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODELS","metadata":{"papermill":{"duration":0.051272,"end_time":"2021-12-11T07:01:28.058814","exception":false,"start_time":"2021-12-11T07:01:28.007542","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-19T17:50:46.142729Z","iopub.execute_input":"2021-12-19T17:50:46.143654Z","iopub.status.idle":"2021-12-19T17:50:46.1526Z","shell.execute_reply.started":"2021-12-19T17:50:46.143613Z","shell.execute_reply":"2021-12-19T17:50:46.151783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utils","metadata":{"papermill":{"duration":0.038002,"end_time":"2021-12-11T07:01:28.134842","exception":false,"start_time":"2021-12-11T07:01:28.09684","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def rle_decode(mask_rle, shape=(520, 704)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) \n                       for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo : hi] = 1\n    return img.reshape(shape)  # Needed to align to RLE direction\n\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    \n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef pred_masks(file_name, path, model, ths, min_pixels):\n    img = cv2.imread(f'{path}/{file_name}')\n    output = model(img)\n    pred_classes = output['instances'].pred_classes.cpu().numpy().tolist()\n    pred_class = max(set(pred_classes), key=pred_classes.count)\n    take = output['instances'].scores >= ths[pred_class]\n    pred_masks = output['instances'].pred_masks[take]\n    pred_masks = pred_masks.cpu().numpy()\n    result = []\n    used = np.zeros(img.shape[:2], dtype=int) \n    for i, mask in enumerate(pred_masks):\n        mask = mask * (1 - used)\n        if mask.sum() >= min_pixels[pred_class]:\n            used += mask\n            result.append(rle_encode(mask))\n    return result\n\ndef ensemble_preds(file_name, path, models, ths):\n    img = cv2.imread(f'{path}/{file_name}')\n    classes = []\n    scores = []\n    bboxes = []\n    masks = []\n    for i, model in enumerate(models):\n        output = model(img)\n        pred_classes = output['instances'].pred_classes.cpu().numpy().tolist()\n        pred_class = max(set(pred_classes), key=pred_classes.count)\n        take = output['instances'].scores >= ths[i][pred_class]\n        classes.extend(output['instances'].pred_classes[take].cpu().numpy().tolist())\n        scores.extend(output['instances'].scores[take].cpu().numpy().tolist())\n        bboxes.extend(output['instances'].pred_boxes[take].tensor.cpu().numpy().tolist())\n        masks.extend(output['instances'].pred_masks[take].cpu().numpy())\n    assert len(classes) == len(masks) , 'ensemble lenght mismatch'\n    #scores, classes, bboxes, masks = zip(*sorted(zip(scores, classes, bboxes, masks),reverse=True))\n    return classes, scores, bboxes, masks\n\ndef nms_predictions(classes, scores, bboxes, masks, \n                    iou_th=.5, shape=(520, 704)):\n    he, wd = shape[0], shape[1]\n    boxes_list = [[[x[0] / wd, x[1] / he, x[2] / wd, x[3] / he] for x in bboxes]]\n    scores_list = [[x for x in scores]]\n    classes_list = [[x for x in classes]]\n    nms_bboxes, nms_scores, nms_classes = non_maximum_weighted(\n        boxes_list, \n        scores_list, \n        classes_list, \n        weights=None,\n        iou_thr=0.3,skip_box_thr=0.0001  \n    )\n    nms_masks = []\n    for s in nms_scores:\n        nms_masks.append(masks[scores.index(s)])\n    nms_scores, nms_classes, nms_masks = zip(*sorted(zip(nms_scores, nms_classes, nms_masks), reverse=True))\n    return nms_classes, nms_scores, nms_masks\n\ndef ensemble_pred_masks(masks, classes, min_pixels, shape=(520, 704)):\n    result = []\n    #pred_class = max(set(classes), key=classes.count)\n    pred_class = int(max(set(classes), key=classes.count).item())\n    used = np.zeros(shape, dtype=int) \n    for i, mask in enumerate(masks):\n        mask = mask * (1 - used)\n        if mask.sum() >= min_pixels[pred_class]:\n            used += mask\n            result.append(rle_encode(mask))\n    return result","metadata":{"papermill":{"duration":0.073286,"end_time":"2021-12-11T07:01:28.247215","exception":false,"start_time":"2021-12-11T07:01:28.173929","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-19T18:01:39.154428Z","iopub.execute_input":"2021-12-19T18:01:39.154686Z","iopub.status.idle":"2021-12-19T18:01:39.17835Z","shell.execute_reply.started":"2021-12-19T18:01:39.154658Z","shell.execute_reply":"2021-12-19T18:01:39.176798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Demo inference","metadata":{"papermill":{"duration":0.040295,"end_time":"2021-12-11T07:01:28.329733","exception":false,"start_time":"2021-12-11T07:01:28.289438","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_names = os.listdir(SUBM_PATH)\nprint('test images:', len(test_names))","metadata":{"papermill":{"duration":0.054704,"end_time":"2021-12-11T07:01:28.4275","exception":false,"start_time":"2021-12-11T07:01:28.372796","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-19T17:52:25.480096Z","iopub.execute_input":"2021-12-19T17:52:25.480533Z","iopub.status.idle":"2021-12-19T17:52:25.487078Z","shell.execute_reply.started":"2021-12-19T17:52:25.480496Z","shell.execute_reply":"2021-12-19T17:52:25.486143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_masks_single = pred_masks(\n    test_names[ID_TEST], \n    path=SUBM_PATH, \n    model=MODELS[0],\n    ths=THSS[0],\n    min_pixels=MIN_PIXELS\n)","metadata":{"papermill":{"duration":5.991517,"end_time":"2021-12-11T07:01:34.455532","exception":false,"start_time":"2021-12-11T07:01:28.464015","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-19T18:01:43.639736Z","iopub.execute_input":"2021-12-19T18:01:43.640256Z","iopub.status.idle":"2021-12-19T18:01:44.171477Z","shell.execute_reply.started":"2021-12-19T18:01:43.640219Z","shell.execute_reply":"2021-12-19T18:01:44.170756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes, scores, bboxes, masks = ensemble_preds(\n    file_name=test_names[ID_TEST] , \n    path=SUBM_PATH, \n    models=MODELS, \n    ths=THSS\n)\nif NMS:\n    classes, scores, masks = nms_predictions(\n        classes, \n        scores, \n        bboxes,\n        masks, iou_th=IOU_TH\n    )\nencoded_masks = ensemble_pred_masks(masks, classes, min_pixels=MIN_PIXELS)","metadata":{"papermill":{"duration":4.168061,"end_time":"2021-12-11T07:01:38.658349","exception":false,"start_time":"2021-12-11T07:01:34.490288","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-19T18:01:45.446941Z","iopub.execute_input":"2021-12-19T18:01:45.447519Z","iopub.status.idle":"2021-12-19T18:01:46.936495Z","shell.execute_reply.started":"2021-12-19T18:01:45.447479Z","shell.execute_reply":"2021-12-19T18:01:46.935754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, axs = plt.subplots(2, 2, figsize=(14, 8))\naxs[0][0].imshow(cv2.imread(f'{SUBM_PATH}/{test_names[ID_TEST]}'))\naxs[0][0].axis('off')\naxs[0][0].set_title(test_names[ID_TEST])\nfor en_mask in encoded_masks_single:\n    dec_mask = rle_decode(en_mask)\n    axs[0][1].imshow(np.ma.masked_where(dec_mask == 0, dec_mask))\n    axs[0][1].axis('off')\n    axs[0][1].set_title('single model')\naxs[1][0].imshow(cv2.imread(f'{SUBM_PATH}/{test_names[ID_TEST]}'))\naxs[1][0].axis('off')\naxs[1][0].set_title(test_names[ID_TEST])\nfor en_mask in encoded_masks:\n    dec_mask = rle_decode(en_mask)\n    axs[1][1].imshow(np.ma.masked_where(dec_mask == 0, dec_mask))\n    axs[1][1].axis('off')\n    axs[1][1].set_title('ensemble models')\nplt.show()","metadata":{"papermill":{"duration":16.87357,"end_time":"2021-12-11T07:01:55.566995","exception":false,"start_time":"2021-12-11T07:01:38.693425","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-19T18:01:53.200834Z","iopub.execute_input":"2021-12-19T18:01:53.201379Z","iopub.status.idle":"2021-12-19T18:02:10.269269Z","shell.execute_reply.started":"2021-12-19T18:01:53.201341Z","shell.execute_reply":"2021-12-19T18:02:10.268588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{"papermill":{"duration":0.038012,"end_time":"2021-12-11T07:01:55.642891","exception":false,"start_time":"2021-12-11T07:01:55.604879","status":"completed"},"tags":[]}},{"cell_type":"code","source":"subm_ids, subm_masks = [], []\nfor test_name in tqdm(test_names):\n    if SINGLE_MODE:\n        encoded_masks = pred_masks(\n            test_name, \n            path=SUBM_PATH, \n            model=MODELS[0],\n            ths=THSS[0],\n            min_pixels=MIN_PIXELS\n        )\n    else:\n        classes, scores, bboxes, masks = ensemble_preds(\n            file_name=test_name, \n            path=SUBM_PATH, \n            models=MODELS, \n            ths=THSS\n        )\n        if NMS:\n            classes, scores, masks = nms_predictions(\n                classes, \n                scores, \n                bboxes, \n                masks, \n                iou_th=IOU_TH\n            )\n        encoded_masks = ensemble_pred_masks(\n            masks, \n            classes, \n            min_pixels=MIN_PIXELS\n        )\n    for enc_mask in encoded_masks:\n        subm_ids.append(test_name[:test_name.find('.')])\n        subm_masks.append(enc_mask)","metadata":{"papermill":{"duration":2.980903,"end_time":"2021-12-11T07:01:58.661498","exception":false,"start_time":"2021-12-11T07:01:55.680595","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-19T17:58:22.836163Z","iopub.execute_input":"2021-12-19T17:58:22.836571Z","iopub.status.idle":"2021-12-19T17:58:26.129492Z","shell.execute_reply.started":"2021-12-19T17:58:22.836523Z","shell.execute_reply":"2021-12-19T17:58:26.128778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame({\n    'id': subm_ids, \n    'predicted': subm_masks\n}).to_csv('submission.csv', index=False)\npd.read_csv('submission.csv').head()","metadata":{"papermill":{"duration":0.131584,"end_time":"2021-12-11T07:01:58.870796","exception":false,"start_time":"2021-12-11T07:01:58.739212","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-19T17:58:26.131425Z","iopub.execute_input":"2021-12-19T17:58:26.131901Z","iopub.status.idle":"2021-12-19T17:58:26.160952Z","shell.execute_reply.started":"2021-12-19T17:58:26.131847Z","shell.execute_reply":"2021-12-19T17:58:26.160154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# References\n1. https://www.kaggle.com/vgarshin/detectron2-inference-with-ensemble-and-nms","metadata":{"papermill":{"duration":0.069861,"end_time":"2021-12-11T07:01:59.019808","exception":false,"start_time":"2021-12-11T07:01:58.949947","status":"completed"},"tags":[]}}]}