{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### I refer to them\n* [Sartorius Segmentation - Detectron2 [training]](https://www.kaggle.com/ammarnassanalhajali/sartorius-segmentation-detectron2-training) \n* [Sartorius Segmentation - Detectron2 [Inference]](https://www.kaggle.com/ammarnassanalhajali/sartorius-segmentation-detectron2-inference) \n* [K-fold CrossValidation COCO Dataset Generator](https://www.kaggle.com/ammarnassanalhajali/k-fold-crossvalidation-coco-dataset-generator) \n\n\n","metadata":{"papermill":{"duration":0.013174,"end_time":"2021-12-11T06:57:37.649999","exception":false,"start_time":"2021-12-11T06:57:37.636825","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Install and import libraries","metadata":{"papermill":{"duration":0.013137,"end_time":"2021-12-11T06:57:37.728835","exception":false,"start_time":"2021-12-11T06:57:37.715698","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install ../input/detectron-05/whls/pycocotools-2.0.2/dist/pycocotools-2.0.2.tar --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/fvcore-0.1.5.post20211019/fvcore-0.1.5.post20211019 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/antlr4-python3-runtime-4.8/antlr4-python3-runtime-4.8 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/detectron2-0.5/detectron2 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/ensemble-boxes-104/ensemble_boxes-1.0.4/ -f ./ --no-index","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":214.247843,"end_time":"2021-12-11T07:01:11.990284","exception":false,"start_time":"2021-12-11T06:57:37.742441","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-30T14:23:44.120205Z","iopub.execute_input":"2021-12-30T14:23:44.120836Z","iopub.status.idle":"2021-12-30T14:27:15.606838Z","shell.execute_reply.started":"2021-12-30T14:23:44.120742Z","shell.execute_reply":"2021-12-30T14:27:15.606037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport json\nimport time\nimport numpy as np\nimport pandas as pd\nimport torch\nimport detectron2\nfrom tqdm.auto import tqdm\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.data.datasets import register_coco_instances\nfrom detectron2.evaluation import inference_on_dataset\nfrom detectron2.evaluation.evaluator import DatasetEvaluator\nfrom detectron2.data import DatasetCatalog, build_detection_test_loader\nimport pycocotools.mask as mask_util\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom fastcore.all import *\nfrom ensemble_boxes import *\nos.environ['CUDA_VISIBLE_DEVICES'] = '0' \nif torch.cuda.is_available():\n    DEVICE = torch.device('cuda')\n    print('GPU is available')\nelse:\n    DEVICE = torch.device('cpu')\n    print('CPU is used')\nprint('detectron ver:', detectron2.__version__)","metadata":{"papermill":{"duration":1.794759,"end_time":"2021-12-11T07:01:13.818453","exception":false,"start_time":"2021-12-11T07:01:12.023694","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-30T14:27:15.609833Z","iopub.execute_input":"2021-12-30T14:27:15.610389Z","iopub.status.idle":"2021-12-30T14:27:17.664426Z","shell.execute_reply.started":"2021-12-30T14:27:15.610355Z","shell.execute_reply":"2021-12-30T14:27:17.663381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## My Models","metadata":{"papermill":{"duration":0.033373,"end_time":"2021-12-11T07:01:13.885547","exception":false,"start_time":"2021-12-11T07:01:13.852174","status":"completed"},"tags":[]}},{"cell_type":"code","source":"best_model=(\n    {'file': '3.0_model_0009679.pth','config_name':'COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml', 'LB score': 0.30,'ths':[.20, .40, .60]},\n    {'file': 'nonmymodel_best_4.pth','config_name':'COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml', 'LB score': 0,'ths':[.15, .35, .55]}, #thsnazo\n    {'file': 'nonmymodel_final.pth','config_name':'COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml', 'LB score': 0.284,'ths':[.17, .37, .57]} #thsnazo\n            )","metadata":{"papermill":{"duration":0.041421,"end_time":"2021-12-11T07:01:13.960561","exception":false,"start_time":"2021-12-11T07:01:13.91914","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-30T14:27:17.669650Z","iopub.execute_input":"2021-12-30T14:27:17.669950Z","iopub.status.idle":"2021-12-30T14:27:17.682903Z","shell.execute_reply.started":"2021-12-30T14:27:17.669914Z","shell.execute_reply":"2021-12-30T14:27:17.681863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"KAGGLE = True\nimport os\nimport cv2\nimport json\nimport time\nimport numpy as np\nimport pandas as pd\nimport torch\nimport detectron2\nfrom tqdm.auto import tqdm\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.data.datasets import register_coco_instances\nfrom detectron2.evaluation import inference_on_dataset\nfrom detectron2.evaluation.evaluator import DatasetEvaluator\nfrom detectron2.data import DatasetCatalog, build_detection_test_loader\nimport pycocotools.mask as mask_util\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom fastcore.all import *\nfrom ensemble_boxes import *\nos.environ['CUDA_VISIBLE_DEVICES'] = '0' if KAGGLE else '1'\nif torch.cuda.is_available():\n    DEVICE = torch.device('cuda')\n    print('GPU is available')\nelse:\n    DEVICE = torch.device('cpu')\n    print('CPU is used')\nprint('detectron ver:', detectron2.__version__)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T14:27:17.689604Z","iopub.execute_input":"2021-12-30T14:27:17.691985Z","iopub.status.idle":"2021-12-30T14:27:17.710862Z","shell.execute_reply.started":"2021-12-30T14:27:17.691945Z","shell.execute_reply":"2021-12-30T14:27:17.709889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#config_name = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\nmdl_path = \"../input/my-trained-models\"\nDATA_PATH = \"../input/sartorius-cell-instance-segmentation\"\nMODELS = []\nBEST_MODELS =[]\nTHSS = []\nID_TEST = 0\nSUBM_PATH = f'{DATA_PATH}/test'\nSINGLE_MODE = False\nNMS = True\nMIN_PIXELS = [75, 150, 75]\nIOU_TH = 0.60\nfor b_m in best_model:\n    model_name=b_m[\"file\"]\n    model_ths=b_m[\"ths\"]\n    config_name=b_m[\"config_name\"]\n    BEST_MODELS.append(model_name)\n    THSS.append(model_ths)\n    cfg = get_cfg()\n    cfg.MODEL.DEVICE = \"cpu\"\n    cfg.merge_from_file(model_zoo.get_config_file(config_name))\n    cfg.INPUT.MASK_FORMAT = 'bitmask'\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 3 \n    cfg.MODEL.WEIGHTS = f'{mdl_path}/{model_name}'\n    cfg.TEST.DETECTIONS_PER_IMAGE = 10000\n    MODELS.append(DefaultPredictor(cfg))\nprint(f'all loaded:\\nthresholds: {THSS}\\nmodels: {BEST_MODELS}')","metadata":{"papermill":{"duration":13.975435,"end_time":"2021-12-11T07:01:27.969039","exception":false,"start_time":"2021-12-11T07:01:13.993604","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-30T14:27:17.715826Z","iopub.execute_input":"2021-12-30T14:27:17.718231Z","iopub.status.idle":"2021-12-30T14:27:25.642757Z","shell.execute_reply.started":"2021-12-30T14:27:17.718179Z","shell.execute_reply":"2021-12-30T14:27:25.641901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODELS","metadata":{"papermill":{"duration":0.051272,"end_time":"2021-12-11T07:01:28.058814","exception":false,"start_time":"2021-12-11T07:01:28.007542","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-30T14:27:25.643935Z","iopub.execute_input":"2021-12-30T14:27:25.644178Z","iopub.status.idle":"2021-12-30T14:27:25.653500Z","shell.execute_reply.started":"2021-12-30T14:27:25.644143Z","shell.execute_reply":"2021-12-30T14:27:25.652568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utils","metadata":{"papermill":{"duration":0.038002,"end_time":"2021-12-11T07:01:28.134842","exception":false,"start_time":"2021-12-11T07:01:28.09684","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def rle_decode(mask_rle, shape=(520, 704)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) \n                       for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo : hi] = 1\n    return img.reshape(shape)  # Needed to align to RLE direction\n\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    \n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef pred_masks(file_name, path, model, ths, min_pixels):\n    img = cv2.imread(f'{path}/{file_name}')\n    output = model(img)\n    pred_classes = output['instances'].pred_classes.cpu().numpy().tolist()\n    pred_class = max(set(pred_classes), key=pred_classes.count)\n    take = output['instances'].scores >= ths[pred_class]\n    pred_masks = output['instances'].pred_masks[take]\n    pred_masks = pred_masks.cpu().numpy()\n    result = []\n    used = np.zeros(img.shape[:2], dtype=int) \n    for i, mask in enumerate(pred_masks):\n        mask = mask * (1 - used)\n        if mask.sum() >= min_pixels[pred_class]:\n            used += mask\n            result.append(rle_encode(mask))\n    return result\n\ndef ensemble_preds(file_name, path, models, ths):\n    img = cv2.imread(f'{path}/{file_name}')\n    classes = []\n    scores = []\n    bboxes = []\n    masks = []\n    for i, model in enumerate(models):\n        output = model(img)\n        pred_classes = output['instances'].pred_classes.cpu().numpy().tolist()\n        pred_class = max(set(pred_classes), key=pred_classes.count)\n        take = output['instances'].scores >= ths[i][pred_class]\n        classes.extend(output['instances'].pred_classes[take].cpu().numpy().tolist())\n        scores.extend(output['instances'].scores[take].cpu().numpy().tolist())\n        bboxes.extend(output['instances'].pred_boxes[take].tensor.cpu().numpy().tolist())\n        masks.extend(output['instances'].pred_masks[take].cpu().numpy())\n    assert len(classes) == len(masks) , 'ensemble lenght mismatch'\n    #scores, classes, bboxes, masks = zip(*sorted(zip(scores, classes, bboxes, masks),reverse=True))\n    return classes, scores, bboxes, masks\n\ndef nms_predictions(classes, scores, bboxes, masks, \n                    iou_th=0.55, shape=(520, 704)):\n    he, wd = shape[0], shape[1]\n    boxes_list = [[[x[0] / wd, x[1] / he, x[2] / wd, x[3] / he] for x in bboxes]]\n    scores_list = [[x for x in scores]]\n    classes_list = [[x for x in classes]]\n    nms_bboxes, nms_scores, nms_classes = non_maximum_weighted(\n        boxes_list, \n        scores_list, \n        classes_list, \n        weights=None,\n        iou_thr=0.55,skip_box_thr=0.0002 #0.3, 0.0001  \n    )\n    nms_masks = []\n    for s in nms_scores:\n        nms_masks.append(masks[scores.index(s)])\n    nms_scores, nms_classes, nms_masks = zip(*sorted(zip(nms_scores, nms_classes, nms_masks), reverse=True))\n    return nms_classes, nms_scores, nms_masks\n\ndef ensemble_pred_masks(masks, classes, min_pixels, shape=(520, 704)):\n    result = []\n    #pred_class = max(set(classes), key=classes.count)\n    pred_class = int(max(set(classes), key=classes.count).item())\n    used = np.zeros(shape, dtype=int) \n    for i, mask in enumerate(masks):\n        mask = mask * (1 - used)\n        if mask.sum() >= min_pixels[pred_class]:\n            used += mask\n            result.append(rle_encode(mask))\n    return result","metadata":{"papermill":{"duration":0.073286,"end_time":"2021-12-11T07:01:28.247215","exception":false,"start_time":"2021-12-11T07:01:28.173929","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-30T14:45:33.046836Z","iopub.execute_input":"2021-12-30T14:45:33.047109Z","iopub.status.idle":"2021-12-30T14:45:33.071112Z","shell.execute_reply.started":"2021-12-30T14:45:33.047079Z","shell.execute_reply":"2021-12-30T14:45:33.070400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Demo inference","metadata":{"papermill":{"duration":0.040295,"end_time":"2021-12-11T07:01:28.329733","exception":false,"start_time":"2021-12-11T07:01:28.289438","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_names = os.listdir(SUBM_PATH)\nprint('test images:', len(test_names))","metadata":{"papermill":{"duration":0.054704,"end_time":"2021-12-11T07:01:28.4275","exception":false,"start_time":"2021-12-11T07:01:28.372796","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-30T14:45:34.234328Z","iopub.execute_input":"2021-12-30T14:45:34.234899Z","iopub.status.idle":"2021-12-30T14:45:34.243800Z","shell.execute_reply.started":"2021-12-30T14:45:34.234863Z","shell.execute_reply":"2021-12-30T14:45:34.242760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_masks_single = pred_masks(\n    test_names[ID_TEST], \n    path=SUBM_PATH, \n    model=MODELS[0],\n    ths=THSS[0],\n    min_pixels=MIN_PIXELS\n)","metadata":{"papermill":{"duration":5.991517,"end_time":"2021-12-11T07:01:34.455532","exception":false,"start_time":"2021-12-11T07:01:28.464015","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-30T14:45:34.687840Z","iopub.execute_input":"2021-12-30T14:45:34.688236Z","iopub.status.idle":"2021-12-30T14:45:42.985111Z","shell.execute_reply.started":"2021-12-30T14:45:34.688203Z","shell.execute_reply":"2021-12-30T14:45:42.984351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes, scores, bboxes, masks = ensemble_preds(\n    file_name=test_names[ID_TEST] , \n    path=SUBM_PATH, \n    models=MODELS,\n    ths=THSS\n)\nif NMS:\n    classes, scores, masks = nms_predictions(\n        classes, \n        scores, \n        bboxes,\n        masks, iou_th=IOU_TH\n    )\nencoded_masks = ensemble_pred_masks(masks, classes, min_pixels=MIN_PIXELS)","metadata":{"papermill":{"duration":4.168061,"end_time":"2021-12-11T07:01:38.658349","exception":false,"start_time":"2021-12-11T07:01:34.490288","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-30T14:46:08.615494Z","iopub.execute_input":"2021-12-30T14:46:08.616224Z","iopub.status.idle":"2021-12-30T14:46:36.752862Z","shell.execute_reply.started":"2021-12-30T14:46:08.616163Z","shell.execute_reply":"2021-12-30T14:46:36.752051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, axs = plt.subplots(2, 2, figsize=(14, 8))\naxs[0][0].imshow(cv2.imread(f'{SUBM_PATH}/{test_names[ID_TEST]}'))\naxs[0][0].axis('off')\naxs[0][0].set_title(test_names[ID_TEST])\nfor en_mask in encoded_masks_single:\n    dec_mask = rle_decode(en_mask)\n    axs[0][1].imshow(np.ma.masked_where(dec_mask == 0, dec_mask))\n    axs[0][1].axis('off')\n    axs[0][1].set_title('single model')\naxs[1][0].imshow(cv2.imread(f'{SUBM_PATH}/{test_names[ID_TEST]}'))\naxs[1][0].axis('off')\naxs[1][0].set_title(test_names[ID_TEST])\nfor en_mask in encoded_masks:\n    dec_mask = rle_decode(en_mask)\n    axs[1][1].imshow(np.ma.masked_where(dec_mask == 0, dec_mask))\n    axs[1][1].axis('off')\n    axs[1][1].set_title('ensemble models')\nplt.show()","metadata":{"papermill":{"duration":16.87357,"end_time":"2021-12-11T07:01:55.566995","exception":false,"start_time":"2021-12-11T07:01:38.693425","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-30T14:46:36.755479Z","iopub.execute_input":"2021-12-30T14:46:36.756011Z","iopub.status.idle":"2021-12-30T14:46:52.759731Z","shell.execute_reply.started":"2021-12-30T14:46:36.755972Z","shell.execute_reply":"2021-12-30T14:46:52.759030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{"papermill":{"duration":0.038012,"end_time":"2021-12-11T07:01:55.642891","exception":false,"start_time":"2021-12-11T07:01:55.604879","status":"completed"},"tags":[]}},{"cell_type":"code","source":"subm_ids, subm_masks = [], []\nfor test_name in tqdm(test_names):\n    if SINGLE_MODE:\n        encoded_masks = pred_masks(\n            test_name, \n            path=SUBM_PATH, \n            model=MODELS[0],\n            ths=THSS[0],\n            min_pixels=MIN_PIXELS\n        )\n    else:\n        classes, scores, bboxes, masks = ensemble_preds(\n            file_name=test_name, \n            path=SUBM_PATH, \n            models=MODELS, \n            ths=THSS\n        )\n        if NMS:\n            classes, scores, masks = nms_predictions(\n                classes, \n                scores, \n                bboxes, \n                masks, \n                iou_th=IOU_TH\n            )\n        encoded_masks = ensemble_pred_masks(\n            masks, \n            classes, \n            min_pixels=MIN_PIXELS\n        )\n    for enc_mask in encoded_masks:\n        subm_ids.append(test_name[:test_name.find('.')])\n        subm_masks.append(enc_mask)","metadata":{"papermill":{"duration":2.980903,"end_time":"2021-12-11T07:01:58.661498","exception":false,"start_time":"2021-12-11T07:01:55.680595","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-30T14:46:52.761067Z","iopub.execute_input":"2021-12-30T14:46:52.761434Z","iopub.status.idle":"2021-12-30T14:48:06.197362Z","shell.execute_reply.started":"2021-12-30T14:46:52.761402Z","shell.execute_reply":"2021-12-30T14:48:06.196657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame({\n    'id': subm_ids, \n    'predicted': subm_masks\n}).to_csv('submission.csv', index=False)\npd.read_csv('submission.csv').head()","metadata":{"papermill":{"duration":0.131584,"end_time":"2021-12-11T07:01:58.870796","exception":false,"start_time":"2021-12-11T07:01:58.739212","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-30T14:48:06.199082Z","iopub.execute_input":"2021-12-30T14:48:06.199467Z","iopub.status.idle":"2021-12-30T14:48:06.220931Z","shell.execute_reply.started":"2021-12-30T14:48:06.199432Z","shell.execute_reply":"2021-12-30T14:48:06.220163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-12-30T14:48:22.359467Z","iopub.execute_input":"2021-12-30T14:48:22.359984Z","iopub.status.idle":"2021-12-30T14:48:22.376498Z","shell.execute_reply.started":"2021-12-30T14:48:22.359946Z","shell.execute_reply":"2021-12-30T14:48:22.375805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"allok\")","metadata":{"execution":{"iopub.status.busy":"2021-12-30T14:29:31.190414Z","iopub.execute_input":"2021-12-30T14:29:31.190684Z","iopub.status.idle":"2021-12-30T14:29:31.195454Z","shell.execute_reply.started":"2021-12-30T14:29:31.190640Z","shell.execute_reply":"2021-12-30T14:29:31.194659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# References\n1. https://www.kaggle.com/vgarshin/detectron2-inference-with-ensemble-and-nms","metadata":{"papermill":{"duration":0.069861,"end_time":"2021-12-11T07:01:59.019808","exception":false,"start_time":"2021-12-11T07:01:58.949947","status":"completed"},"tags":[]}}]}