{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%set_env CUDA_HOME=/usr/local/cuda-11.8/","metadata":{"execution":{"iopub.status.busy":"2023-06-15T12:34:46.402314Z","iopub.execute_input":"2023-06-15T12:34:46.402664Z","iopub.status.idle":"2023-06-15T12:34:46.417041Z","shell.execute_reply.started":"2023-06-15T12:34:46.402635Z","shell.execute_reply":"2023-06-15T12:34:46.415980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install git+https://github.com/facebookresearch/detectron2.git","metadata":{"execution":{"iopub.status.busy":"2023-06-15T12:34:46.622910Z","iopub.execute_input":"2023-06-15T12:34:46.623290Z","iopub.status.idle":"2023-06-15T12:34:46.627538Z","shell.execute_reply.started":"2023-06-15T12:34:46.623242Z","shell.execute_reply":"2023-06-15T12:34:46.626438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/ensemble-boxes/ensemble_boxes-1.0.9-py3-none-any.whl --no-index --find-links /kaggle/input/ensemble-boxes\n!pip install /kaggle/input/detectron2-wheel-cuda118/detectron2/detectron2-0.6-cp310-cp310-linux_x86_64.whl --no-index --find-links=/kaggle/input/detectron2-wheel-cuda118/detectron2","metadata":{"execution":{"iopub.status.busy":"2023-06-15T12:34:46.831481Z","iopub.execute_input":"2023-06-15T12:34:46.832382Z","iopub.status.idle":"2023-06-15T12:35:13.037787Z","shell.execute_reply.started":"2023-06-15T12:34:46.832340Z","shell.execute_reply":"2023-06-15T12:35:13.036541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport json\nimport time\nimport numpy as np\nimport pandas as pd\nimport torch\nimport detectron2\nfrom tqdm.auto import tqdm\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.data.datasets import register_coco_instances\nfrom detectron2.evaluation import inference_on_dataset\nfrom detectron2.evaluation.evaluator import DatasetEvaluator\nfrom detectron2.data import DatasetCatalog, build_detection_test_loader\nimport pycocotools.mask as mask_util\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom fastcore.all import *\nfrom ensemble_boxes import *\nos.environ['CUDA_VISIBLE_DEVICES'] = '0' \nif torch.cuda.is_available():\n    DEVICE = torch.device('cuda')\n    print('GPU is available')\nelse:\n    DEVICE = torch.device('cpu')\n    print('CPU is used')\nprint('detectron ver:', detectron2.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T12:35:13.040296Z","iopub.execute_input":"2023-06-15T12:35:13.040675Z","iopub.status.idle":"2023-06-15T12:35:17.902941Z","shell.execute_reply.started":"2023-06-15T12:35:13.040635Z","shell.execute_reply":"2023-06-15T12:35:17.901942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mdl_path = \"../input/hhvweightsdetectron2\"\nDATA_PATH = \"../input/hubmap-hacking-the-human-vasculature\"\nMODELS = []\nBEST_MODELS =[]\nTHSS = []\nID_TEST = 0\nSUBM_PATH = f'{DATA_PATH}/test'\nSINGLE_MODE = False\nNMS = True\nIOU_TH = .6\n\nTHRESHOLDS = [.01, .99, .99]\nMIN_PIXELS = [1, 60, 60]\n\nconfig_names = [\n    \"COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x.yaml\",\n    \"COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x.yaml\",\n#     \"Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv.yaml\",\n#     \"Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv.yaml\",\n    \"Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv.yaml\",\n    \"Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv.yaml\",\n]\nbest_model = [\n    \"model_best_old.pth\",\n    \"model_best_old.pth\",\n#     \"model_best_34.67.pth\",\n#     \"model_best_34.67.pth\",\n    \"model_best.pth\",\n    \"model_best.pth\",\n]\ntest_sizes = [\n    640,\n    1280,\n    640,\n    1280,\n#     800,\n#     1280\n]\n\nfor i, b_m in enumerate(best_model):\n    model_name=b_m\n    model_ths=THRESHOLDS\n    BEST_MODELS.append(model_name)\n    THSS.append(model_ths)\n    cfg = get_cfg()\n    cfg.merge_from_file(model_zoo.get_config_file(config_names[i]))\n    cfg.INPUT.MASK_FORMAT = 'bitmask'\n    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1\n    cfg.MODEL.WEIGHTS = f'{mdl_path}/{model_name}'\n\n#     cfg.MODEL.ANCHOR_GENERATOR.SIZES = [[16], [32], [64], [128], [256]]\n#     cfg.MODEL.ASPECT_RATIOS = [[0.25, 0.5, 1.0, 2.0, 4.0]]\n#     cfg.MODEL.RPN.IN_FEATURES = ['p2', 'p3', 'p4', 'p5', 'p6']\n#     cfg.MODEL.SEM_SEG_HEAD.IN_FEATURES = ['p2', 'p3', 'p4', 'p5', 'p6']\n\n    cfg.INPUT.MIN_SIZE_TEST = test_sizes[i]\n    cfg.INPUT.MAX_SIZE_TEST = test_sizes[i]\n\n    cfg.TEST.DETECTIONS_PER_IMAGE = 1000\n    MODELS.append(DefaultPredictor(cfg))\nprint(f'all loaded:\\nthresholds: {THSS}\\nmodels: {BEST_MODELS}')\nMODELS","metadata":{"execution":{"iopub.status.busy":"2023-06-15T12:36:22.300092Z","iopub.execute_input":"2023-06-15T12:36:22.300465Z","iopub.status.idle":"2023-06-15T12:36:56.362053Z","shell.execute_reply.started":"2023-06-15T12:36:22.300437Z","shell.execute_reply":"2023-06-15T12:36:56.360988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_decode(mask_rle, shape=(512, 512)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) \n                       for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo : hi] = 1\n    return img.reshape(shape)  # Needed to align to RLE direction\n\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    \n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef pred_masks(file_name, path, model, ths, min_pixels):\n    img = cv2.imread(f'{path}/{file_name}')\n    output = model(img)\n    pred_classes = output['instances'].pred_classes.cpu().numpy().tolist()\n    pred_class = max(set(pred_classes), key=pred_classes.count)\n    take = output['instances'].scores >= ths[pred_class]\n    pred_masks = output['instances'].pred_masks[take]\n    pred_masks = pred_masks.cpu().numpy()\n    result = []\n    used = np.zeros(img.shape[:2], dtype=int) \n    for i, mask in enumerate(pred_masks):\n        mask = mask * (1 - used)\n        if mask.sum() >= min_pixels[pred_class]:\n            used += mask\n            result.append(rle_encode(mask))\n    return result\n\ndef ensemble_preds(file_name, path, models, ths):\n    img = cv2.imread(f'{path}/{file_name}')\n    \n    outputs = []\n    \n    classes = []\n    scores = []\n    bboxes = []\n    masks = []\n    \n    pred_classes_gros = []\n    \n    for i, model in enumerate(models):\n        output = model(img)\n        outputs.append(output)\n        \n        pred_classes = output['instances'].pred_classes.cpu().numpy().tolist()\n        if len(pred_classes) > 0:\n            pred_class = max(set(pred_classes), key=pred_classes.count)\n            print(f\"old model {i} predict class {pred_class}\")\n            pred_classes_gros.append(pred_class)\n    \n    if len(pred_classes_gros) == 0:\n        return classes, scores, bboxes, masks\n\n    pred_class_final = max(set(pred_classes_gros), key=pred_classes_gros.count)\n    print(f\"final class {pred_class_final}\")\n    \n    for c, output in zip(pred_classes_gros, outputs):\n        if c != pred_class_final:\n            continue\n        take = output['instances'].scores >= ths[i][pred_class_final]\n        classes.extend(output['instances'].pred_classes[take].cpu().numpy().tolist())\n        scores.extend(output['instances'].scores[take].cpu().numpy().tolist())\n        bboxes.extend(output['instances'].pred_boxes[take].tensor.cpu().numpy().tolist())\n        masks.extend(output['instances'].pred_masks[take].cpu().numpy())\n\n    assert len(classes) == len(masks) , 'ensemble lenght mismatch'\n    return classes, scores, bboxes, masks\n\ndef nms_predictions(classes, scores, bboxes, masks, \n                    iou_th=.5, shape=(512, 512)):\n    he, wd = shape[0], shape[1]\n    boxes_list = [[[x[0] / wd, x[1] / he, x[2] / wd, x[3] / he] for x in bboxes]]\n    scores_list = [[x for x in scores]]\n    classes_list = [[x for x in classes]]\n    nms_bboxes, nms_scores, nms_classes = non_maximum_weighted(\n        boxes_list, \n        scores_list, \n        classes_list, \n        weights=None,\n        iou_thr=IOU_TH,\n        skip_box_thr=0.0001,\n    )\n    nms_masks = []\n    for s in nms_scores:\n        nms_masks.append(masks[scores.index(s)])\n    nms_scores, nms_classes, nms_masks = zip(*sorted(zip(nms_scores, nms_classes, nms_masks), reverse=True))\n    return nms_classes, nms_scores, nms_masks\n\ndef ensemble_pred_masks(masks, classes, min_pixels, shape=(512, 512)):\n    result = []\n    #pred_class = max(set(classes), key=classes.count)\n    pred_class = int(max(set(classes), key=classes.count).item())\n    used = np.zeros(shape, dtype=int) \n    for i, mask in enumerate(masks):\n        mask = mask * (1 - used)\n        if mask.sum() >= min_pixels[pred_class]:\n            used += mask\n            result.append(rle_encode(mask))\n    return result","metadata":{"execution":{"iopub.status.busy":"2023-06-15T12:40:29.283375Z","iopub.execute_input":"2023-06-15T12:40:29.283845Z","iopub.status.idle":"2023-06-15T12:40:29.319087Z","shell.execute_reply.started":"2023-06-15T12:40:29.283809Z","shell.execute_reply":"2023-06-15T12:40:29.318032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_names = os.listdir(SUBM_PATH)\nprint('test images:', len(test_names))\n\nencoded_masks_single = pred_masks(\n    test_names[ID_TEST], \n    path=SUBM_PATH, \n    model=MODELS[0],\n    ths=THSS[ID_TEST],\n    min_pixels=MIN_PIXELS\n)\n\nclasses, scores, bboxes, masks = ensemble_preds(\n    file_name=test_names[ID_TEST] , \n    path=SUBM_PATH, \n    models=MODELS, \n    ths=THSS\n)\nif NMS:\n    classes, scores, masks = nms_predictions(\n        classes, \n        scores, \n        bboxes,\n        masks, iou_th=IOU_TH\n    )\nencoded_masks = ensemble_pred_masks(masks, classes, min_pixels=MIN_PIXELS)\n\n_, axs = plt.subplots(2, 2, figsize=(16, 12))\naxs[0][0].imshow(cv2.imread(f'{SUBM_PATH}/{test_names[ID_TEST]}'))\naxs[0][0].axis('on')\naxs[0][0].set_title(test_names[ID_TEST])\nfor en_mask in encoded_masks_single:\n    dec_mask = rle_decode(en_mask)\n    axs[0][1].imshow(np.ma.masked_where(dec_mask == 0, dec_mask))\n    axs[0][1].axis('on')\n    axs[0][1].set_title('single model')\naxs[1][0].imshow(cv2.imread(f'{SUBM_PATH}/{test_names[ID_TEST]}'))\naxs[1][0].axis('on')\naxs[1][0].set_title(test_names[ID_TEST])\nfor en_mask in encoded_masks:\n    dec_mask = rle_decode(en_mask)\n    axs[1][1].imshow(np.ma.masked_where(dec_mask == 0, dec_mask))\n    axs[1][1].axis('on')\n    axs[1][1].set_title('ensemble models')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-15T12:40:29.454052Z","iopub.execute_input":"2023-06-15T12:40:29.454783Z","iopub.status.idle":"2023-06-15T12:40:44.271014Z","shell.execute_reply.started":"2023-06-15T12:40:29.454745Z","shell.execute_reply":"2023-06-15T12:40:44.269797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\n\nimport base64\nimport numpy as np\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib\n\ndef encode_binary_mask(mask: np.ndarray) -> t.Text:\n    # check input mask --\n    if mask.dtype != bool:\n        raise ValueError(\n            \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n            mask.dtype)\n\n    mask = np.squeeze(mask)\n    if len(mask.shape) != 2:\n        raise ValueError(\n            \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n            mask.shape)\n\n    # convert input mask to expected COCO API input --\n    mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n    mask_to_encode = mask_to_encode.astype(np.uint8)\n    mask_to_encode = np.asfortranarray(mask_to_encode)\n\n    # RLE encode mask --\n    encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n    # compress and base64 encoding --\n    binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n    base64_str = base64.b64encode(binary_str)\n    return base64_str\n\ntest_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/test\"\nsample_submission = pd.read_csv('/kaggle/input/hubmap-hacking-the-human-vasculature/sample_submission.csv')\nids = []\nheights = []\nwidths = []\nprediction_strings = []\n\nfor test_name in tqdm(test_names):\n    img = cv2.imread(f'{SUBM_PATH}/{test_name}')\n    h, w, _ = img.shape\n    if SINGLE_MODE:\n        encoded_masks = pred_masks(\n            test_name, \n            path=SUBM_PATH, \n            model=MODELS[0],\n            ths=THSS[0],\n            min_pixels=MIN_PIXELS\n        )\n    else:\n        classes, scores, bboxes, masks = ensemble_preds(\n            file_name=test_name, \n            path=SUBM_PATH, \n            models=MODELS, \n            ths=THSS\n        )\n        if NMS and len(classes) and len(scores) and len(masks):\n            classes, scores, masks = nms_predictions(\n                classes, \n                scores, \n                bboxes, \n                masks, \n                iou_th=IOU_TH\n            )\n        pred_string = \"\"\n        for i, mask in enumerate(masks):\n            if int(classes[i]) != 0: continue\n            if mask.sum() < MIN_PIXELS[int(classes[i])]: continue\n            \n            kernel = np.ones((3, 3), np.uint8)\n            mask = cv2.dilate(mask.astype(np.uint8), kernel, iterations=1) == 1\n\n            encoded = encode_binary_mask(mask)\n            if i == 0:\n                pred_string += f\"{int(classes[i])} {scores[i]} {encoded.decode('utf-8')}\"\n            else:\n                pred_string += f\" {int(classes[i])} {scores[i]} {encoded.decode('utf-8')}\"\n        ids.append(test_name.split('.')[0])\n        heights.append(h)\n        widths.append(w)\n        prediction_strings.append(pred_string)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-06-15T12:40:44.272539Z","iopub.execute_input":"2023-06-15T12:40:44.273121Z","iopub.status.idle":"2023-06-15T12:40:47.266654Z","shell.execute_reply.started":"2023-06-15T12:40:44.273092Z","shell.execute_reply":"2023-06-15T12:40:47.265724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['id'] = ids\nsubmission['height'] = heights\nsubmission['width'] = widths\nsubmission['prediction_string'] = prediction_strings\nsubmission = submission.set_index('id')\nsubmission.to_csv(\"submission.csv\")\nprint(submission)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T12:40:47.268311Z","iopub.execute_input":"2023-06-15T12:40:47.272965Z","iopub.status.idle":"2023-06-15T12:40:47.301965Z","shell.execute_reply.started":"2023-06-15T12:40:47.272930Z","shell.execute_reply":"2023-06-15T12:40:47.300842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}