{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Detectron: ensemble inference with NMS","metadata":{"papermill":{"duration":0.013003,"end_time":"2021-10-27T20:21:06.575583","exception":false,"start_time":"2021-10-27T20:21:06.56258","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Acknowledgements","metadata":{}},{"cell_type":"markdown","source":"Based on these excellent notebooks [Positive score with Detectron 2/3 - Training](https://www.kaggle.com/slawekbiel/positive-score-with-detectron-2-3-training) and [Positive score with Detectron 3/3 - Inference](https://www.kaggle.com/slawekbiel/positive-score-with-detectron-3-3-inference). Please upvote them.\n\n[Weighted boxes fusion](https://github.com/ZFTurbo/Weighted-Boxes-Fusion) library is also used.","metadata":{}},{"cell_type":"markdown","source":"## Install and import libraries","metadata":{"papermill":{"duration":0.009696,"end_time":"2021-10-27T20:21:06.597177","exception":false,"start_time":"2021-10-27T20:21:06.587481","status":"completed"},"tags":[]}},{"cell_type":"code","source":"KAGGLE = True","metadata":{"execution":{"iopub.status.busy":"2021-11-13T20:21:43.892139Z","iopub.execute_input":"2021-11-13T20:21:43.892592Z","iopub.status.idle":"2021-11-13T20:21:43.912481Z","shell.execute_reply.started":"2021-11-13T20:21:43.892515Z","shell.execute_reply":"2021-11-13T20:21:43.9115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IPATH = '../input/detectron-05/whls'\nif KAGGLE:\n    !pip install {IPATH}/pycocotools-2.0.2/dist/pycocotools-2.0.2.tar --no-index --find-links ../input/detectron-05/whls \n    !pip install {IPATH}/fvcore-0.1.5.post20211019/fvcore-0.1.5.post20211019 --no-index --find-links ../input/detectron-05/whls \n    !pip install {IPATH}/antlr4-python3-runtime-4.8/antlr4-python3-runtime-4.8 --no-index --find-links ../input/detectron-05/whls \n    !pip install {IPATH}/detectron2-0.5/detectron2 --no-index --find-links ../input/detectron-05/whls \n    !pip install ../input/ensemble-boxes-104/ensemble_boxes-1.0.4/ -f ./ --no-index","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":201.727571,"end_time":"2021-10-27T20:24:28.33438","exception":false,"start_time":"2021-10-27T20:21:06.606809","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-13T20:21:43.914945Z","iopub.execute_input":"2021-11-13T20:21:43.915523Z","iopub.status.idle":"2021-11-13T20:26:16.545883Z","shell.execute_reply.started":"2021-11-13T20:21:43.915479Z","shell.execute_reply":"2021-11-13T20:26:16.544533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport json\nimport time\nimport numpy as np\nimport pandas as pd\nimport torch\nimport detectron2\nfrom tqdm.auto import tqdm\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.data.datasets import register_coco_instances\nfrom detectron2.evaluation import inference_on_dataset\nfrom detectron2.evaluation.evaluator import DatasetEvaluator\nfrom detectron2.data import DatasetCatalog, build_detection_test_loader\nimport pycocotools.mask as mask_util\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom fastcore.all import *\nfrom ensemble_boxes import *\nos.environ['CUDA_VISIBLE_DEVICES'] = '0' if KAGGLE else '1'\nif torch.cuda.is_available():\n    DEVICE = torch.device('cuda')\n    print('GPU is available')\nelse:\n    DEVICE = torch.device('cpu')\n    print('CPU is used')\nprint('detectron ver:', detectron2.__version__)","metadata":{"papermill":{"duration":1.200842,"end_time":"2021-10-27T20:24:29.563049","exception":false,"start_time":"2021-10-27T20:24:28.362207","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-13T20:26:16.554327Z","iopub.execute_input":"2021-11-13T20:26:16.555275Z","iopub.status.idle":"2021-11-13T20:26:18.842088Z","shell.execute_reply.started":"2021-11-13T20:26:16.555228Z","shell.execute_reply":"2021-11-13T20:26:18.841159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config load","metadata":{}},{"cell_type":"code","source":"VERS = ['vdtrn5f0', 'vdtrn5f1', 'vdtrn5f2', 'vdtrn5f3', 'vdtrn5f4']\nDATA_PATH = '../input/sartorius-cell-instance-segmentation' if KAGGLE else './data'\nMODELS = []\nCONFIGS = []\nTHSS = []\nBEST_MODELS =[]\nID_TEST = 0\nSUBM_PATH = f'{DATA_PATH}/test'\nSINGLE_MODE = False\nNMS = True\nMIN_PIXELS = [110, 60, 50]\nIOU_TH = .4\n\nfor ver in VERS:\n    mdl_path = f'../input/sartorius-models-{ver}' if KAGGLE else f'./models_{ver}'\n    print('loading:', mdl_path)\n    with open(f'{mdl_path}/config.json', 'r') as file:\n        config = json.load(file)\n        CONFIGS.append(config)\n    with open(f'{mdl_path}/ths.json', 'r') as file:\n        THSS.append(json.load(file))\n    with open(f'{mdl_path}/best_model.json', 'r') as file:\n        best_model = json.load(file)\n        BEST_MODELS.append(best_model)\n    model_name = best_model['file']\n    print('model to load:', model_name, '| score:', best_model['score'])\n    cfg = get_cfg()\n    cfg.merge_from_file(model_zoo.get_config_file(config['chk_point']))\n    cfg.INPUT.MASK_FORMAT = 'bitmask'\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 3 \n    cfg.MODEL.WEIGHTS = f'{mdl_path}/{model_name}'  \n    cfg.TEST.DETECTIONS_PER_IMAGE = 1000\n    MODELS.append(DefaultPredictor(cfg))\nprint(f'all loaded:\\nconfigs: {CONFIGS}\\nthresholds: {THSS}\\nmodels: {BEST_MODELS}')\n\nstart_time = time.time()","metadata":{"execution":{"iopub.status.busy":"2021-11-13T20:26:18.845269Z","iopub.execute_input":"2021-11-13T20:26:18.846076Z","iopub.status.idle":"2021-11-13T20:26:40.390815Z","shell.execute_reply.started":"2021-11-13T20:26:18.846032Z","shell.execute_reply":"2021-11-13T20:26:40.389614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utils","metadata":{}},{"cell_type":"code","source":"def rle_decode(mask_rle, shape=(520, 704)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) \n                       for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo : hi] = 1\n    return img.reshape(shape)  # Needed to align to RLE direction\n\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    \n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef pred_masks(file_name, path, model, ths, min_pixels):\n    img = cv2.imread(f'{path}/{file_name}')\n    output = model(img)\n    pred_classes = output['instances'].pred_classes.cpu().numpy().tolist()\n    pred_class = max(set(pred_classes), key=pred_classes.count)\n    take = output['instances'].scores >= ths[pred_class]\n    pred_masks = output['instances'].pred_masks[take]\n    pred_masks = pred_masks.cpu().numpy()\n    result = []\n    used = np.zeros(img.shape[:2], dtype=int) \n    for i, mask in enumerate(pred_masks):\n        mask = mask * (1 - used)\n        if mask.sum() >= min_pixels[pred_class]:\n            used += mask\n            result.append(rle_encode(mask))\n    return result\n\ndef ensemble_preds(file_name, path, models, ths):\n    img = cv2.imread(f'{path}/{file_name}')\n    classes = []\n    scores = []\n    bboxes = []\n    masks = []\n    for i, model in enumerate(models):\n        output = model(img)\n        pred_classes = output['instances'].pred_classes.cpu().numpy().tolist()\n        pred_class = max(set(pred_classes), key=pred_classes.count)\n        take = output['instances'].scores >= ths[i][pred_class]\n        classes.extend(output['instances'].pred_classes[take].cpu().numpy().tolist())\n        scores.extend(output['instances'].scores[take].cpu().numpy().tolist())\n        bboxes.extend(output['instances'].pred_boxes[take].tensor.cpu().numpy().tolist())\n        masks.extend(output['instances'].pred_masks[take].cpu().numpy())\n    assert len(classes) == len(masks) , 'ensemble lenght mismatch'\n    scores, classes, bboxes, masks = zip(\n        *sorted(zip(scores, classes, bboxes, masks), \n                reverse=True))\n    return classes, scores, bboxes, masks\n\ndef nms_predictions(classes, scores, bboxes, masks, \n                    iou_th=.5, shape=(520, 704)):\n    he, wd = shape[0], shape[1]\n    boxes_list = [[x[0] / wd, x[1] / he, x[2] / wd, x[3] / he]\n                  for x in bboxes]\n    scores_list = [x for x in scores]\n    labels_list = [x for x in classes]\n    nms_bboxes, nms_scores, nms_classes = nms(\n        boxes=[boxes_list], \n        scores=[scores_list], \n        labels=[labels_list], \n        weights=None,\n        iou_thr=iou_th\n    )\n    nms_masks = []\n    for s in nms_scores:\n        nms_masks.append(masks[scores.index(s)])\n    nms_scores, nms_classes, nms_masks = zip(\n        *sorted(\n            zip(nms_scores, nms_classes, nms_masks), \n            reverse=True))\n    return nms_classes, nms_scores, nms_masks\n\ndef ensemble_pred_masks(masks, classes, min_pixels, shape=(520, 704)):\n    result = []\n    pred_class = max(set(classes), key=classes.count)\n    used = np.zeros(shape, dtype=int) \n    for i, mask in enumerate(masks):\n        mask = mask * (1 - used)\n        if mask.sum() >= min_pixels[pred_class]:\n            used += mask\n            result.append(rle_encode(mask))\n    return result","metadata":{"papermill":{"duration":0.038173,"end_time":"2021-10-27T20:24:29.684335","exception":false,"start_time":"2021-10-27T20:24:29.646162","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-13T20:26:40.399884Z","iopub.execute_input":"2021-11-13T20:26:40.40048Z","iopub.status.idle":"2021-11-13T20:26:40.442361Z","shell.execute_reply.started":"2021-11-13T20:26:40.400425Z","shell.execute_reply":"2021-11-13T20:26:40.441055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Demo inference","metadata":{"papermill":{"duration":0.025925,"end_time":"2021-10-27T20:24:37.162884","exception":false,"start_time":"2021-10-27T20:24:37.136959","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_names = os.listdir(SUBM_PATH)\nprint('test images:', len(test_names))","metadata":{"papermill":{"duration":0.037264,"end_time":"2021-10-27T20:24:29.748077","exception":false,"start_time":"2021-10-27T20:24:29.710813","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-13T20:26:40.446055Z","iopub.execute_input":"2021-11-13T20:26:40.446769Z","iopub.status.idle":"2021-11-13T20:26:40.557623Z","shell.execute_reply.started":"2021-11-13T20:26:40.446722Z","shell.execute_reply":"2021-11-13T20:26:40.556567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_masks_single = pred_masks(\n    test_names[ID_TEST], \n    path=SUBM_PATH, \n    model=MODELS[0],\n    ths=THSS[0],\n    min_pixels=MIN_PIXELS\n)","metadata":{"papermill":{"duration":21.807086,"end_time":"2021-10-27T20:24:58.995678","exception":false,"start_time":"2021-10-27T20:24:37.188592","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-13T20:26:40.561224Z","iopub.execute_input":"2021-11-13T20:26:40.561926Z","iopub.status.idle":"2021-11-13T20:26:47.202186Z","shell.execute_reply.started":"2021-11-13T20:26:40.56179Z","shell.execute_reply":"2021-11-13T20:26:47.20089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes, scores, bboxes, masks = ensemble_preds(\n    file_name=test_names[ID_TEST] , \n    path=SUBM_PATH, \n    models=MODELS, \n    ths=THSS\n)\nif NMS:\n    classes, scores, masks = nms_predictions(\n        classes, \n        scores, \n        bboxes,\n        masks, iou_th=IOU_TH\n    )\nencoded_masks = ensemble_pred_masks(masks, classes, min_pixels=MIN_PIXELS)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T20:26:47.205675Z","iopub.execute_input":"2021-11-13T20:26:47.206644Z","iopub.status.idle":"2021-11-13T20:26:52.893833Z","shell.execute_reply.started":"2021-11-13T20:26:47.206597Z","shell.execute_reply":"2021-11-13T20:26:52.892603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, axs = plt.subplots(2, 2, figsize=(14, 8))\naxs[0][0].imshow(cv2.imread(f'{SUBM_PATH}/{test_names[ID_TEST]}'))\naxs[0][0].axis('off')\naxs[0][0].set_title(test_names[ID_TEST])\nfor en_mask in encoded_masks_single:\n    dec_mask = rle_decode(en_mask)\n    axs[0][1].imshow(np.ma.masked_where(dec_mask == 0, dec_mask))\n    axs[0][1].axis('off')\n    axs[0][1].set_title('single model')\naxs[1][0].imshow(cv2.imread(f'{SUBM_PATH}/{test_names[ID_TEST]}'))\naxs[1][0].axis('off')\naxs[1][0].set_title(test_names[ID_TEST])\nfor en_mask in encoded_masks:\n    dec_mask = rle_decode(en_mask)\n    axs[1][1].imshow(np.ma.masked_where(dec_mask == 0, dec_mask))\n    axs[1][1].axis('off')\n    axs[1][1].set_title('ensemble models')\nplt.show()\n\nelapsed_time = time.time() - start_time\nprint(f'time elapsed: {elapsed_time // 60:.0f} min {elapsed_time % 60:.0f} sec')","metadata":{"execution":{"iopub.status.busy":"2021-11-13T20:26:52.895905Z","iopub.execute_input":"2021-11-13T20:26:52.896434Z","iopub.status.idle":"2021-11-13T20:27:13.188229Z","shell.execute_reply.started":"2021-11-13T20:26:52.89639Z","shell.execute_reply":"2021-11-13T20:27:13.187127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{"papermill":{"duration":0.041734,"end_time":"2021-10-27T20:24:59.084607","exception":false,"start_time":"2021-10-27T20:24:59.042873","status":"completed"},"tags":[]}},{"cell_type":"code","source":"subm_ids, subm_masks = [], []\nfor test_name in tqdm(test_names):\n    if SINGLE_MODE:\n        encoded_masks = pred_masks(\n            test_name, \n            path=SUBM_PATH, \n            model=MODELS[0],\n            ths=THSS[0],\n            min_pixels=MIN_PIXELS\n        )\n    else:\n        classes, scores, bboxes, masks = ensemble_preds(\n            file_name=test_name, \n            path=SUBM_PATH, \n            models=MODELS, \n            ths=THSS\n        )\n        if NMS:\n            classes, scores, masks = nms_predictions(\n                classes, \n                scores, \n                bboxes, \n                masks, \n                iou_th=IOU_TH\n            )\n        encoded_masks = ensemble_pred_masks(\n            masks, \n            classes, \n            min_pixels=MIN_PIXELS\n        )\n    for enc_mask in encoded_masks:\n        subm_ids.append(test_name[:test_name.find('.')])\n        subm_masks.append(enc_mask)","metadata":{"papermill":{"duration":0.673062,"end_time":"2021-10-27T20:24:59.800298","exception":false,"start_time":"2021-10-27T20:24:59.127236","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-13T20:27:13.189689Z","iopub.execute_input":"2021-11-13T20:27:13.190691Z","iopub.status.idle":"2021-11-13T20:27:17.091267Z","shell.execute_reply.started":"2021-11-13T20:27:13.190642Z","shell.execute_reply":"2021-11-13T20:27:17.090267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame({\n    'id': subm_ids, \n    'predicted': subm_masks\n}).to_csv('submission.csv', index=False)\npd.read_csv('submission.csv').head()\n\nelapsed_time = time.time() - start_time\nprint(f'time elapsed: {elapsed_time // 60:.0f} min {elapsed_time % 60:.0f} sec')","metadata":{"papermill":{"duration":0.07156,"end_time":"2021-10-27T20:24:59.914224","exception":false,"start_time":"2021-10-27T20:24:59.842664","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-13T20:27:17.093118Z","iopub.execute_input":"2021-11-13T20:27:17.093765Z","iopub.status.idle":"2021-11-13T20:27:17.131677Z","shell.execute_reply.started":"2021-11-13T20:27:17.093664Z","shell.execute_reply":"2021-11-13T20:27:17.130663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}