{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":30201,"databundleVersionId":2750748,"sourceType":"competition"},{"sourceId":1986198,"sourceType":"datasetVersion","datasetId":1187413},{"sourceId":2724479,"sourceType":"datasetVersion","datasetId":1660485},{"sourceId":2880819,"sourceType":"datasetVersion","datasetId":1764632},{"sourceId":2988139,"sourceType":"datasetVersion","datasetId":1814566}],"dockerImageVersionId":30140,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is based on Slawek Biel's notebook (https://www.kaggle.com/slawekbiel/positive-score-with-detectron-3-3-inference)","metadata":{}},{"cell_type":"markdown","source":"## Inference and submission\nNow, we have trained model ([train with livecell](https://www.kaggle.com/markunys/sartorius-transfer-learning-train-with-livecell), [train](https://www.kaggle.com/markunys/sartorius-transfer-learning-train)). Let's inference with the model !!","metadata":{}},{"cell_type":"code","source":"!pip install ../input/detectron-05/whls/pycocotools-2.0.2/dist/pycocotools-2.0.2.tar --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/fvcore-0.1.5.post20211019/fvcore-0.1.5.post20211019 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/antlr4-python3-runtime-4.8/antlr4-python3-runtime-4.8 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/detectron2-0.5/detectron2 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/ensemble-boxes-104/ensemble_boxes-1.0.4/ -f ./ --no-index","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-30T16:05:54.422099Z","iopub.execute_input":"2021-12-30T16:05:54.422554Z","iopub.status.idle":"2021-12-30T16:09:23.867878Z","shell.execute_reply.started":"2021-12-30T16:05:54.422441Z","shell.execute_reply":"2021-12-30T16:09:23.86687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import detectron2\nimport torch\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom PIL import Image\nimport cv2\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom pathlib import Path\n# from cluster_utils import Path\nfrom glob import glob\n\nfrom ensemble_boxes import *","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:09:23.870189Z","iopub.execute_input":"2021-12-30T16:09:23.870611Z","iopub.status.idle":"2021-12-30T16:09:25.485402Z","shell.execute_reply.started":"2021-12-30T16:09:23.870554Z","shell.execute_reply":"2021-12-30T16:09:25.484651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataDir = Path('../input/sartorius-cell-instance-segmentation')","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:09:25.486994Z","iopub.execute_input":"2021-12-30T16:09:25.487283Z","iopub.status.idle":"2021-12-30T16:09:25.491303Z","shell.execute_reply.started":"2021-12-30T16:09:25.487246Z","shell.execute_reply":"2021-12-30T16:09:25.490655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# From https://www.kaggle.com/stainsby/fast-tested-rle\ndef rle_decode(mask_rle, shape=(520, 704)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape)  # Needed to align to RLE direction\n\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\n\ndef predict_masks(fn, overall_predictor):\n    im = cv2.imread(str(fn))\n    pred = overall_predictor(im)\n    image_class = torch.mode(pred['instances'].pred_classes)[0]\n    \n    score_arr = pred['instances'].scores.cpu()\n    class_arr = pred['instances'].pred_classes.cpu()        \n    correct_score_arr = score_arr * (class_arr==int(image_class.cpu()))\n    \n    take = correct_score_arr >= THRESHOLDS[image_class]\n    pred_masks = pred['instances'].pred_masks[take].cpu().numpy()\n\n    result = []\n    used = np.zeros(im.shape[:2], dtype=int)\n    for mask in pred_masks:\n        mask = mask * (1-used)\n        if mask.sum() >= MIN_PIXELS[image_class]: # skip predictions with small area\n            used += mask\n            result.append(rle_encode(mask))\n    return result\n","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:09:25.493239Z","iopub.execute_input":"2021-12-30T16:09:25.493793Z","iopub.status.idle":"2021-12-30T16:09:25.509194Z","shell.execute_reply.started":"2021-12-30T16:09:25.493756Z","shell.execute_reply":"2021-12-30T16:09:25.508563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collect_predictions(filename, predictor_list):\n    '''collect instance predictions of multiple predictors on one image'''\n    im = cv2.imread(str(filename))\n    score_list,bbox_list,mask_list = [],[],[]\n    \n    for pos,predictor in enumerate(predictor_list):\n        # perform single model prediction\n        prediction = predictor(im)\n        image_class = torch.mode(prediction['instances'].pred_classes)[0].cpu().item()\n        score_arr = prediction['instances'].scores.cpu()\n        class_arr = prediction['instances'].pred_classes.cpu()\n        correct_score_arr = score_arr * (class_arr==int(image_class)) # one class per image\n        take = correct_score_arr >= THRESHOLDS[image_class]\n        \n        # add instances with high confidence to the pool list\n        score_list.extend(score_arr[take].tolist())\n        bbox_list.extend(prediction['instances'].pred_boxes[take].tensor.cpu().numpy().tolist())\n        mask_list.extend(prediction['instances'].pred_masks[take].cpu().numpy())\n    \n    return score_list,bbox_list,mask_list, image_class\n\n\ndef NMS_ensemble_to_masks(score_list, bbox_list, mask_list, image_class,\n                               iou_th=0.5, shape=(520, 704)):\n    # use imported NMS algorithm to filter the instance candidates\n    height, width = shape\n    bbox_list = [[bbox[0] / width, bbox[1] / height, bbox[2] / width, bbox[3] / height]\n                  for bbox in bbox_list]\n    label_list = [image_class for _ in range(len(score_list))]\n\n    nms_bboxes,nms_scores,_ = non_maximum_weighted(\n        [bbox_list], \n        [score_list],\n        [label_list],\n        weights=None,\n        iou_thr=0.3,skip_box_thr=0.0001\n    )\n    \n    nms_masks = [\n        mask_list[score_list.index(s)] for s in nms_scores\n    ]\n    \n    # sort the nms resulting lists by confidence score\n    nms_scores,nms_masks = zip(\n        *sorted(\n            zip(nms_scores,nms_masks), reverse=True,\n        )\n    )\n\n    # generate non-overlapping masks\n    result = []\n    used = np.zeros(shape[:2], dtype=int)\n    for mask in nms_masks:\n        mask = mask * (1-used)\n        if mask.sum() >= MIN_PIXELS[image_class]: # skip predictions with small area\n            used += mask\n            result.append(rle_encode(mask))\n    return result\n    \n    \ndef predict_NMS_ensemble_masks(filename, predictor_list, iou_th=0.5, shape=(520, 704)):\n    score_list,bbox_list,mask_list, image_class = collect_predictions(filename, predictor_list)\n    final_masks = NMS_ensemble_to_masks(score_list, bbox_list, mask_list, image_class,\n                               iou_th, shape)\n    return final_masks\n    ","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:09:25.510673Z","iopub.execute_input":"2021-12-30T16:09:25.510945Z","iopub.status.idle":"2021-12-30T16:09:25.528656Z","shell.execute_reply.started":"2021-12-30T16:09:25.510906Z","shell.execute_reply":"2021-12-30T16:09:25.527962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_info_list = (\n    {'file': 'X101_bigstep2_nosh_12000.pth',\n     'architecture': 'mask_rcnn_X_101_32x8d_FPN_3x',\n    'LB': 0.307, 'note': 'no extra sh training'},\n    {'file': 'mask_rcnn_R_50_FPN_3x_overall_bigstep_18000.pth',\n     'architecture': 'mask_rcnn_R_50_FPN_3x',\n    'LB': 0.308, 'note': 'with extra sh training' },\n    {'file': 'R50_FPN_bigstep2_nosh_12000.pth',\n     'architecture': 'mask_rcnn_R_50_FPN_3x',\n    'LB': 0.309, 'note': 'no extra sh training'},\n)\n\n\npredictor_list = []\nfor model_dict in model_info_list:\n    cfg = get_cfg()\n    cfg.merge_from_file(model_zoo.get_config_file(f\"COCO-InstanceSegmentation/{model_dict['architecture']}.yaml\"))\n    cfg.INPUT.MASK_FORMAT = 'bitmask'\n    cfg.TEST.DETECTIONS_PER_IMAGE = 1000\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 3\n    cfg.MODEL.WEIGHTS = (f\"../input/sartorius-separated-models/{model_dict['file']}\")\n    predictor_list.append(DefaultPredictor(cfg))\n","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:19:10.249263Z","iopub.execute_input":"2021-12-30T16:19:10.249524Z","iopub.status.idle":"2021-12-30T16:19:14.455531Z","shell.execute_reply.started":"2021-12-30T16:19:10.249495Z","shell.execute_reply":"2021-12-30T16:19:14.454768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"THRESHOLDS = [.15, .35, .55] # \nMIN_PIXELS = [75, 150, 75]\nMAX_PIXELS = [2000, 10000, 2000]\n\nids, masks = [],[]\nimage_file_list = glob(f'{dataDir}/test/*') # + glob(f'{dataDir}/train/*')\n# display(image_file_list[:2])\n\nfor i_file in image_file_list:\n    i_image_id = i_file.split('/')[-1].split('.')[0]\n#     encoded_masks = predict_masks(i_file, predictor_list[0])\n    encoded_masks = predict_NMS_ensemble_masks(i_file, predictor_list)\n    for enc in encoded_masks:\n        ids.append(i_image_id)\n        masks.append(enc)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:19:17.58589Z","iopub.execute_input":"2021-12-30T16:19:17.586549Z","iopub.status.idle":"2021-12-30T16:19:20.295292Z","shell.execute_reply.started":"2021-12-30T16:19:17.586508Z","shell.execute_reply":"2021-12-30T16:19:20.294596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for submission\npd.DataFrame({'id':ids, 'predicted':masks}).to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:19:22.56436Z","iopub.execute_input":"2021-12-30T16:19:22.564649Z","iopub.status.idle":"2021-12-30T16:19:22.582814Z","shell.execute_reply.started":"2021-12-30T16:19:22.564617Z","shell.execute_reply":"2021-12-30T16:19:22.582102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('submission.csv').shape","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:19:37.557409Z","iopub.execute_input":"2021-12-30T16:19:37.557876Z","iopub.status.idle":"2021-12-30T16:19:37.569808Z","shell.execute_reply.started":"2021-12-30T16:19:37.557837Z","shell.execute_reply":"2021-12-30T16:19:37.569003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('submission.csv').sample(5)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:19:50.81913Z","iopub.execute_input":"2021-12-30T16:19:50.819812Z","iopub.status.idle":"2021-12-30T16:19:50.83519Z","shell.execute_reply.started":"2021-12-30T16:19:50.819775Z","shell.execute_reply":"2021-12-30T16:19:50.834576Z"},"trusted":true},"execution_count":null,"outputs":[]}]}