{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!cp -r /kaggle/input/pycocotools/ /kaggle/working/pycocotools\n!pip install /kaggle/working/pycocotools/pycocotools-2.0.6  --no-index --find-links=/kaggle/working/pycocotools/ \n!pip install -qqq /kaggle/input/weighted-boxes-fusion/ensemble_b1nboxes-1.0.9-py3-none-any.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-31T10:20:36.381148Z","iopub.execute_input":"2023-07-31T10:20:36.381519Z","iopub.status.idle":"2023-07-31T10:21:33.237097Z","shell.execute_reply.started":"2023-07-31T10:20:36.381468Z","shell.execute_reply":"2023-07-31T10:21:33.235791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# install mmdet\n!pip install -qqq /kaggle/input/mmdetv3-env/archive/addict-2.4.0-py3-none-any.whl\n!pip install -qqq /kaggle/input/mmdetv3-env/archive/mmengine-0.7.4-py3-none-any.whl\n!pip install -qqq /kaggle/input/mmdetv3-env/archive/mmcv-2.0.0-cp310-cp310-linux_x86_64.whl\n!pip install -qqq /kaggle/input/mmdetv3-env/archive/terminaltables-3.1.10-py2.py3-none-any.whl\n!pip install -qqq /kaggle/input/mmdetection-3-1-evn/src/mmdet-3.1.0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2023-07-31T10:21:33.239609Z","iopub.execute_input":"2023-07-31T10:21:33.239988Z","iopub.status.idle":"2023-07-31T10:24:16.051888Z","shell.execute_reply.started":"2023-07-31T10:21:33.239953Z","shell.execute_reply":"2023-07-31T10:24:16.050595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# yolov7 历史文件夹\nimport sys\nsys.path.append('/kaggle/input/hubmap-eda-coco-yolov7-train-and-validate') \nsys.path.append('/kaggle/input/hubmap-eda-coco-yolov7-train-and-validate/yolov7/seg') \n#sys.path.append('../input/ensemble-boxes/')\n\nimport torch\nprint('torch',torch.__version__)\n\nimport pandas as pd\nimport numpy as np\nfrom glob import glob\n\nimport base64\nfrom pycocotools import _mask as coco_mask\nimport typing as typ\nimport zlib\nimport cv2\n\nprint('numpy',np.__version__)\nfrom yolov7.seg.models.common import DetectMultiBackend\nfrom yolov7.seg.utils.general import non_max_suppression,xywh2xyxy\nfrom yolov7.seg.utils.segment.general import process_mask, scale_masks,crop\nfrom yolov7.seg.utils.torch_utils import select_device\n#from ensemble_boxes import weighted_boxes_fusion, weighted_masks_fusion\n\n\nimport mmdet, mmcv, mmengine\nfrom mmengine.config import Config\nfrom mmengine.runner import Runner\nfrom mmdet.utils import register_all_modules\nfrom mmdet.apis import init_detector, inference_detector\nfrom mmengine.visualization import Visualizer\n\nimport matplotlib\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nprint(mmdet.__version__)\nprint(mmcv.__version__)\nprint(mmengine.__version__)\nprint('IMPORT OK!!!')","metadata":{"execution":{"iopub.status.busy":"2023-07-31T10:24:16.054991Z","iopub.execute_input":"2023-07-31T10:24:16.055801Z","iopub.status.idle":"2023-07-31T10:24:21.071535Z","shell.execute_reply.started":"2023-07-31T10:24:16.055760Z","shell.execute_reply":"2023-07-31T10:24:21.069786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encode_binary_mask(mask: np.ndarray) -> typ.Text:\n    \"\"\"Converts a binary mask into OID challenge encoding ascii text.\"\"\"\n\n    # check input mask --\n    #if mask.dtype != np.bool:\n    if mask.dtype != bool:\n        raise ValueError(\n            \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n            mask.dtype)\n\n    mask = np.squeeze(mask)\n    if len(mask.shape) != 2:\n        raise ValueError(\n            \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n            mask.shape)\n\n    # convert input mask to expected COCO API input --\n    mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n    mask_to_encode = mask_to_encode.astype(np.uint8)\n    mask_to_encode = np.asfortranarray(mask_to_encode)\n\n    # RLE encode mask --\n    encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n    # compress and base64 encoding --\n    binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n    base64_str = base64.b64encode(binary_str)\n    return base64_str\n\n# 提交trick，对输出结果进行腐蚀或膨胀\n# https://www.kaggle.com/competitions/hubmap-hacking-the-human-vasculature/discussion/416901\ndef dilate_predict_mask(out_mask):\n    for i in range(len(out_mask)):\n        kernel = np.ones(shape=(3, 3), dtype=np.uint8)\n        out_mask[i] = cv2.dilate(out_mask[i], kernel, 3)\n    return out_mask\n\ndef augment_hsv(img, h_gain=1.0, s_gain=1.0, v_gain=1.0):\n    img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV).astype(np.int16)\n    # img_hsv.shape = (450, 720, 3)\n\n    img_hsv[..., 0] = (img_hsv[..., 0] * h_gain) % 180\n    img_hsv[..., 1] = np.clip(img_hsv[..., 1] * s_gain, 0, 255)\n    img_hsv[..., 2] = np.clip(img_hsv[..., 2] * v_gain, 0, 255)\n    \n    aug_img = cv2.cvtColor(img_hsv.astype(img.dtype), cv2.COLOR_HSV2BGR, dst=img)\n    return aug_img\n\n# 单词进行预测\ndef predict_one(model_f, image_file, device, augment = False, hsv_h = 1.0, hsv_s = 1.0, hsv_v = 1.0):\n    model = DetectMultiBackend(model_f, device=device, dnn=False, data=data_yaml, fp16=False)\n    model.warmup(imgsz=(1,3,512,512))  \n    \n    im0 = cv2.imread(image_file)  # BGR\n    if augment:\n        im0 = augment_hsv(im0, h_gain=hsv_h, s_gain=hsv_s, v_gain=hsv_v)\n    # Resize and pad image while meeting stride-multiple constraints\n    #im = letterbox(im0, self.img_size, stride=self.stride, auto=self.auto)[0]  # padded resize\n    im = im0\n    im = im.transpose((2, 0, 1))[::-1]  # HWC to CHW, BGR to RGB\n    im = np.ascontiguousarray(im)  # contiguous\n\n    im = torch.from_numpy(im).to(device)\n    im = im.float()  # uint8 to fp16/32\n    im /= 255  # 0 - 255 to 0.0 - 1.0\n    im = im[None]  # expand for batch dim\n    \n    \n\n    # inference\n    pred, out = model(im, augment=False, visualize=False)\n    proto = out[1]\n    \n    #pred.shape: torch.Size([1, 16128, 40])\n\n    # NMS\n    pred = non_max_suppression(pred, conf_thres=0.001, iou_thres=0.6, classes=0, agnostic=False, max_det=1000, nm=32) #nm = num_mask?\n    # now pred is list\n        \n    out_mask = []\n    out_conf = []\n    for i, det in enumerate(pred):  # per image\n        if len(det):\n            print(\"det\",det.shape)\n            masks = process_mask(proto[i], det[:, 6:], det[:, :4], im.shape[2:], upsample=True)  # HWC\n            confs = det[:, 4]\n            clasf = det[:, 5]\n\n            out_conf = confs.data.cpu().numpy()\n\n            # https://www.kaggle.com/competitions/hubmap-hacking-the-human-vasculature/discussion/416901\n            for mask, confidence, classification in zip(masks, confs, clasf):\n                binary_mask = mask.cpu().numpy() \n                out_mask.append(binary_mask)\n                \n    torch.cuda.empty_cache()\n  \n    return out_mask, out_conf","metadata":{"execution":{"iopub.status.busy":"2023-07-31T10:24:21.074993Z","iopub.execute_input":"2023-07-31T10:24:21.075352Z","iopub.status.idle":"2023-07-31T10:24:21.100537Z","shell.execute_reply.started":"2023-07-31T10:24:21.075317Z","shell.execute_reply":"2023-07-31T10:24:21.096020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 用于集成的函数：","metadata":{}},{"cell_type":"code","source":"import warnings\nimport torch.nn.functional as F\nimport time\ndef prefilter_boxes_faster(boxes, scores, labels, weights, thr = 0.7):\n    # ->torch.tensor\n    # Create dict with boxes stored by its label\n    # only suits one label\n\n    new_boxes = []\n    # start = time.time()\n\n    for t in range(len(boxes)):\n        ######## t-th model ########\n        if len(boxes[t]) != len(scores[t]):\n            raise KeyError(\n                'Error. Length of boxes arrays not equal to length of scores array: {} != {}'.format(len(boxes[t]),\n                                                                                                     len(scores[t])))\n        if len(boxes[t]) != len(labels[t]):\n            raise KeyError(\n                'Error. Length of boxes arrays not equal to length of labels array: {} != {}'.format(len(boxes[t]),\n                                                                                                     len(labels[t])))\n\n        # type tensor\n        score = scores[t]\n        pick = torch.ones_like(score, device=score.device).bool()\n\n        label = labels[t]\n        box_part = boxes[t]\n\n        box_part[box_part < 0] = 0.0\n        box_part[box_part > 1] = 1.0\n\n        # Box data checks\n        false_box = box_part[:, 0] > box_part[:, 2]\n        if (box_part[false_box].shape[0] > 0):\n            mid = box_part[false_box, 0].clone()\n            box_part[false_box, 0] = box_part[false_box, 2]\n            box_part[false_box, 2] = mid\n\n        false_box = box_part[:, 1] > box_part[:, 3]\n        if (box_part[false_box].shape[0] > 0):\n            mid = box_part[false_box, 1].clone()\n            box_part[false_box, 1] = box_part[false_box, 3]\n            box_part[false_box, 3] = mid\n\n        del false_box\n        pick[(box_part[:, 0] - box_part[:, 2]) * (box_part[:, 1] - box_part[:, 3]) == 0] = False\n        # print(label[:,None].shape, (score * weights[t])[:,None].shape, (torch.ones_like(score)*weights[t])[:,None].shape,\n        # (torch.ones_like(score)*t)[:,None].shape, torch.arange(0,score.shape[0])[:,None].shape,box_part.shape)\n\n        b = torch.cat([label[:, None], (score * weights[t])[:, None],\n                       (torch.ones_like(score, device=score.device) * weights[t])[:, None],\n                       (torch.ones_like(score, device=score.device) * t)[:, None],\n                       torch.arange(0, score.shape[0], device=score.device)[:, None], box_part], dim=1)\n        # print(b.shape)\n        new_boxes.append(b[pick])\n\n    new_boxes = torch.cat(new_boxes, dim=0)\n\n    new_boxes = new_boxes[new_boxes[:, 1].argsort(descending=True)]\n\n    return new_boxes  # box:[label, score, weight, model index, box index, x1, y1, x2, y2]\n\n\n\ndef tta_box_preprocecss(tta_detection,mode,weights):\n    new_boxes = []\n    N_model = len(tta_detection)\n    for index,tta_i in enumerate(tta_detection):\n        # start = time.time()\n\n            # type tensor\n        score = tta_i[:, 4]\n        label = tta_i[:, 5]\n        box_part = tta_i[:, :4]\n        pick = torch.ones_like(score, device=score.device).bool()\n\n\n        box_part[box_part < 0] = 0.0\n        box_part[box_part > 1] = 1.0\n\n        if (mode == \"ud\"):\n            box_part[:, 0] = 1 - box_part[:, 0]\n            box_part[:, 2] = 1 - box_part[:, 2]\n        elif (mode == \"rl\"):\n            box_part[:, 1] = 1 - box_part[:, 1]\n            box_part[:, 3] = 1 - box_part[:, 3]\n        else:\n            raise KeyError(\"mode is wrong\")\n\n        # Box data checks\n        false_box = box_part[:, 0] > box_part[:, 2]\n        if (box_part[false_box].shape[0] > 0):\n            mid = box_part[false_box, 0].clone()\n            box_part[false_box, 0] = box_part[false_box, 2]\n            box_part[false_box, 2] = mid\n\n        false_box = box_part[:, 1] > box_part[:, 3]\n        if (box_part[false_box].shape[0] > 0):\n            mid = box_part[false_box, 1].clone()\n            box_part[false_box, 1] = box_part[false_box, 3]\n            box_part[false_box, 3] = mid\n\n        del false_box\n        pick[(box_part[:, 0] - box_part[:, 2]) * (box_part[:, 1] - box_part[:, 3]) == 0] = False\n        # print(label[:,None].shape, (score * weights[t])[:,None].shape, (torch.ones_like(score)*weights[t])[:,None].shape,\n        # (torch.ones_like(score)*t)[:,None].shape, torch.arange(0,score.shape[0])[:,None].shape,box_part.shape)\n\n        if(mode==\"rl\"):\n            b = torch.cat([label[:, None], (score * weights[index])[:, None],\n                       (torch.ones_like(score, device=score.device) * weights[index])[:, None],\n                       (torch.ones_like(score, device=score.device) * index + N_model)[:, None],\n                       torch.arange(0, score.shape[0], device=score.device)[:, None], box_part], dim=1)\n        elif(mode==\"ud\"):\n            b = torch.cat([label[:, None], (score * weights[index])[:, None],\n                           (torch.ones_like(score, device=score.device) * weights[index])[:, None],\n                           (torch.ones_like(score, device=score.device) * index + N_model*2)[:, None],\n                           torch.arange(0, score.shape[0], device=score.device)[:, None], box_part], dim=1)\n        else:\n            raise KeyError(\"mode is wrong\")\n        # print(b.shape)\n        new_boxes.append(b[pick].cpu().numpy())\n\n        # new_boxes = torch.cat(new_boxes, dim=0)\n        # new_boxes = new_boxes[new_boxes[:, 1].argsort(descending=True)]\n    return new_boxes\n\n\ndef preprocess_mmdet_box_faster(mmdet_pred,model_index,img_shape,weights=1.0):\n    # mmdet_pred :list\n    #     scores = mmdet_pred.scores.cpu().numpy()\n    #     bboxes = mmdet_pred.bboxes.cpu().numpy()\n    #     masks = mmdet_pred.masks.cpu().numpy()\n    #     labels = mmdet_pred.labels.cpu().numpy()\n    # box:[label, score, weight, model index, box index, x1, y1, x2, y2]\n\n    ih, iw = img_shape\n    score = mmdet_pred.scores\n    box_part = mmdet_pred.bboxes\n    label = mmdet_pred.labels\n    mask = mmdet_pred.masks\n\n    pick = torch.ones_like(score, device=score.device).bool()\n\n    box_part[:, 0] = box_part[:, 0] / iw\n    box_part[:, 2] = box_part[:, 2] / iw\n    box_part[:, 1] = box_part[:, 1] / ih\n    box_part[:, 3] = box_part[:, 3] / ih\n\n    box_part[box_part < 0] = 0.0\n    box_part[box_part > 1] = 1.0\n\n    # Box data checks\n    false_box = box_part[:, 0] > box_part[:, 2]\n    if (box_part[false_box].shape[0] > 0):\n        mid = box_part[false_box, 0].clone()\n        box_part[false_box, 0] = box_part[false_box, 2]\n        box_part[false_box, 2] = mid\n\n    false_box = box_part[:, 1] > box_part[:, 3]\n    if (box_part[false_box].shape[0] > 0):\n        mid = box_part[false_box, 1].clone()\n        box_part[false_box, 1] = box_part[false_box, 3]\n        box_part[false_box, 3] = mid\n\n    del false_box\n    pick[(box_part[:, 0] - box_part[:, 2]) * (box_part[:, 1] - box_part[:, 3]) == 0] = False\n\n    b = torch.cat([label[:, None], (score * weights)[:, None],\n                   (torch.ones_like(score, device=score.device) * weights)[:, None],\n                   (torch.ones_like(score, device=score.device) * model_index + 10)[:, None],\n                   torch.arange(0, score.shape[0], device=score.device)[:, None], box_part], dim=1)\n\n\n    new_boxes = b[pick].cpu().numpy()\n    #new_boxes = new_boxes[new_boxes[:, 1].argsort(descending=True)]\n\n    return new_boxes,mask[pick].cpu().numpy()      # np.array\n\n\ndef get_weighted_box_faster(boxes):\n    \"\"\"\n    Create weighted box for set of boxes\n    :param boxes: set of boxes to fuse\n    :param conf_type: type of confidence one of 'avg' or 'max'\n    :return: weighted box (label, score, weight, model index, x1, y1, x2, y2)\n    \"\"\"\n    # box:[label, score, weight, model index, box index, x1, y1, x2, y2]\n\n    box = np.zeros(9, dtype=np.float32)\n    # w = 0\n    #boxes = np.array(boxes[boxes[3]<3 & boxes[3]>=10])\n    boxes = np.array(boxes)\n    box[5:] = (boxes[:, 1].reshape(-1, 1) * boxes[:, 5:]).sum(axis=0)\n    conf = boxes[:, 1].sum()\n\n    box[0] = boxes[0, 0]\n    box[1] = conf / boxes.shape[0]\n    box[2] = boxes[:, 2].sum()\n    box[3] = -1\n    box[4] = -1\n    if (conf == 0): print(conf)\n    box[5:] /= conf\n\n    return box\n\ndef find_matching_box_fast(boxes_list, new_box, match_iou,flip=False):\n    \"\"\"\n        Reimplementation of find_matching_box with numpy instead of loops. Gives significant speed up for larger arrays\n        (~100x). This was previously the bottleneck since the function is called for every entry in the array.\n    \"\"\"\n\n    def bb_iou_array(boxesi, new_boxi):\n        xA = np.maximum(boxesi[:, 0], new_boxi[0])\n        yA = np.maximum(boxesi[:, 1], new_boxi[1])\n        xB = np.minimum(boxesi[:, 2], new_boxi[2])\n        yB = np.minimum(boxesi[:, 3], new_boxi[3])\n\n        interArea = np.maximum(xB - xA, 0) * np.maximum(yB - yA, 0)\n\n        # compute the area of both the prediction and ground-truth rectangles\n        boxAArea = (boxesi[:, 2] - boxesi[:, 0]) * (boxesi[:, 3] - boxesi[:, 1])\n        boxBArea = (new_boxi[2] - new_boxi[0]) * (new_boxi[3] - new_boxi[1])\n\n        iou = interArea / (boxAArea + boxBArea - interArea)\n\n        return iou\n\n    if len(boxes_list) == 0:\n        return -1, match_iou\n\n    if(flip):\n        boxes = boxes_list\n        #print(\"fliped box shape\",boxes.shape)\n    else:\n        boxes = np.array(boxes_list)\n        #print(\"normal box shape\",boxes.shape)\n        \n    ious = bb_iou_array(boxes[:, 5:], new_box[5:])\n    ious[boxes[:, 0] != new_box[0]] = -1\n    best_idx = np.argmax(ious)\n    best_iou = ious[best_idx]\n\n    if best_iou <= match_iou:\n        best_iou = match_iou\n        best_idx = -1\n\n    return best_idx, best_iou\n\n\n\n\ndef one_process_mask(protos, masks_in, bboxes, shape, upsample=True):\n    \"\"\"\n    Crop before upsample.\n    proto_out: [mask_dim, mask_h, mask_w]\n    out_masks: [n, mask_dim], n is number of masks after nms\n    bboxes: [n, 4], n is number of masks after nms\n    shape:input_image_size, (h, w)\n\n    return: h, w, n\n    \"\"\"\n\n    c, mh, mw = protos.shape  # CHW\n\n    ih, iw = shape\n    masks = (masks_in @ protos.float().view(c, -1)).sigmoid().view(-1, mh, mw)  # CHW\n\n    if(isinstance(bboxes,np.ndarray)):\n        downsampled_bboxes = bboxes.copy()\n    else:\n        downsampled_bboxes = bboxes.clone()\n    downsampled_bboxes[0] *= mw\n    downsampled_bboxes[2] *= mw\n    downsampled_bboxes[3] *= mh\n    downsampled_bboxes[1] *= mh\n\n\n    x1, y1, x2, y2 = torch.chunk(downsampled_bboxes[:, None], 4, 0)  # x1 shape(1,1,n)\n    r = torch.arange(mw, device=masks.device, dtype=x1.dtype)[None, :]  # rows shape(1,w,1)\n    c = torch.arange(mh, device=masks.device, dtype=x1.dtype)[:, None]  # cols shape(h,1,1)\n    masks = masks[0] * (r >= x1) * (r < x2) * (c >= y1) * (c < y2)\n    # print(\"mid mask shape\",masks.shape)\n\n    if upsample:\n        masks = F.interpolate(masks[None, None, :, :], shape, mode='bilinear', align_corners=False)[0]\n    # print(\"then mask shape\",masks.shape)\n    return masks.gt_(0.5).cpu().numpy()\n\n\ndef yolo_get_weighted_mask(proto, detection, cluster_boxes, img_shape):\n    mask = np.zeros(img_shape, dtype=np.float32)[None, :, :]\n    # mask = torch.zeros(img_shape, dtype=proto[0].dtype, device = proto[0].device)\n    conf = 0\n    # conf_list = []\n\n    for b in cluster_boxes:\n        s = b[1]\n        m = one_process_mask(proto[int(b[3])], detection[int(b[3])][int(b[4]), 6:], detection[int(b[3])][int(b[4]), :4],\n                             img_shape)\n        mask = mask + s * m\n        conf += s\n        # conf_list.append(s)\n\n    # score = np.max(conf_list)\n    mask = mask / conf\n\n    return mask  # , score, conf_list\n\ndef all_get_weighted_mask(proto, detection, cluster_boxes,mmdet_mask, img_shape):\n    mask = np.zeros(img_shape, dtype=np.float32)[None, :, :]\n    # mask = torch.zeros(img_shape, dtype=proto[0].dtype, device = proto[0].device)\n    conf = 0\n    # conf_list = []\n\n    for b in cluster_boxes:\n        s = b[1]\n        if(int(b[3])<10):\n            m = one_process_mask(proto[int(b[3])], detection[int(b[3])][int(b[4]), 6:], detection[int(b[3])][int(b[4]), :4],\n                             img_shape)\n        else:\n            # model_index > 10 is mmdet model\n            m = mmdet_mask[int(b[3])-10][int(b[4])]\n\n        mask = mask + s * m\n        conf += s\n        # conf_list.append(s)\n\n    # score = np.max(conf_list)\n    mask = mask / conf\n\n    return mask  # , score, conf_list\n\n\n\ndef yolo_weighted_boxes_fusion_faster(\n        detection,\n        proto,\n        img_shape,\n        weights=None,\n        iou_thr=0.55,\n        skip_box_thr=0.0,\n        conf_type='avg',\n        allows_overflow=False\n):\n    '''\n    detection:[[box(x1,y1,x2,y2),conf,cls],...]\n    proto:[proto0,...]\n    yolo_weighted_boxes_fusion(det,proto,im.shape[2:])\n\n    :param weights: list of weights for each model. Default: None, which means weight == 1 for each model\n    :param iou_thr: IoU value for boxes to be a match\n    :param skip_box_thr: exclude boxes with score lower than this variable\n    :param conf_type: how to calculate confidence in weighted boxes.\n        'avg': average value,\n        'max': maximum value,\n        'box_and_model_avg': box and model wise hybrid weighted average,\n        'absent_model_aware_avg': weighted average that takes into account the absent model.\n    :param allows_overflow: false if we want confidence score not exceed 1.0\n\n    :return: boxes: boxes coordinates (Order of boxes: x1, y1, x2, y2).\n    :return: scores: confidence scores\n    :return: labels: boxes labels\n    '''\n\n    # masks = process_mask(proto[i], det[:, 6:], det[:, :4], im.shape[2:], upsample=True)\n    # raw_mask = detection[:, 6:]\n    # print(\"orig:\",detection[0][0,:4])\n\n    ih, iw = img_shape\n    norm_box = [i[:, :4] for i in detection]\n    # box = detection[:, :4]\n    scores_list = [i[:, 4] for i in detection]\n    labels_list = [i[:, 5] for i in detection]\n\n    for i in range(len(norm_box)):\n        norm_box[i][:, 0] /= iw\n        norm_box[i][:, 2] /= iw\n        norm_box[i][:, 1] /= ih\n        norm_box[i][:, 3] /= ih\n\n    # print(\"after norm:\",detection[0][0,:4])\n\n    if weights is None:\n        weights = np.ones(len(norm_box))\n    if len(weights) != len(norm_box):\n        print('Warning: incorrect number of weights {}. Must be: {}. Set weights equal to 1.'.format(len(weights),\n                                                                                                     len(norm_box)))\n        weights = np.ones(len(norm_box))\n    weights = np.array(weights)\n\n    if conf_type not in ['avg', 'max', 'box_and_model_avg', 'absent_model_aware_avg']:\n        print('Unknown conf_type: {}. Must be \"avg\", \"max\" or \"box_and_model_avg\", or \"absent_model_aware_avg\"'.format(\n            conf_type))\n        exit()\n\n    filtered_boxes = prefilter_boxes_faster(norm_box, scores_list, labels_list, weights, skip_box_thr)\n    # filtered_boxes:tensor[[label, score, weight, model index, box index, x1, y1, x2, y2],...]\n\n    if filtered_boxes.shape[0] == 0:\n        return np.zeros((0, 4)), np.zeros((0,)), np.zeros((0,))\n\n    boxes = filtered_boxes.cpu().numpy()\n    new_boxes = []\n    weighted_boxes = []\n\n    # Clusterize boxes\n\n    for j in range(0, boxes.shape[0]):\n        # boxes[j] is every box whose cls = label\n        # boxes[j]:[label, score, weight, model index, box index, x1, y1, x2, y2]\n        index, best_iou = find_matching_box_fast(weighted_boxes, boxes[j], iou_thr)\n\n        # new_boxes:[[box,box2],[box1,box5]...]\n        # box:[label, score, weight, model index, box index, x1, y1, x2, y2]\n        if index != -1:\n            new_boxes[index].append(boxes[j])\n            weighted_boxes[index] = get_weighted_box_faster(new_boxes[index])\n            # weighted_masks[index] = yolo_get_weighted_mask(proto, detection, new_boxes[index], img_shape)\n\n\n        else:\n            new_boxes.append([boxes[j]])\n            weighted_boxes.append(boxes[j])\n\n            # new_mask = one_process_mask(proto[int(boxes[j][3])], detection[int(boxes[j][3])][int(boxes[j][4]), 6:],\n            # detection[int(boxes[j][3])][int(boxes[j][4]), :4], img_shape)\n            # weighted_masks = np.vstack((weighted_masks, new_mask))\n\n            # weighted_boxes:[[label, score, weight, model index, box index, x1, y1, x2, y2]]\n\n    weighted_boxes = np.array(weighted_boxes)\n    weighted_masks = []\n    for i in range(len(new_boxes)):\n        # len(new_boxes) is num of cluster\n        # start = time.time()\n        clustered_boxes = np.array(new_boxes[i])\n        new_mask = yolo_get_weighted_mask(proto, detection, clustered_boxes, img_shape)\n        weighted_masks.append(new_mask)\n\n        weighted_boxes[i, 1] = weighted_boxes[i, 1] * clustered_boxes.shape[0] / weights.sum()  # avg\n        # end = time.time()\n        # print(f\"faster time {clustered_boxes.shape[0]} boxes:\",end-start)\n\n    weighted_masks = np.concatenate(weighted_masks, axis=0)\n    pick_arg = weighted_boxes[:, 1].argsort()[::-1]\n    weighted_boxes = weighted_boxes[pick_arg]\n    weighted_masks = weighted_masks[pick_arg]\n    print(\"overall shape:\", weighted_boxes.shape, weighted_masks.shape)\n    # box,score,label,mask\n\n    return weighted_boxes[:, 5:], weighted_boxes[:, 1], weighted_boxes[:, 0], weighted_masks\n\n\ndef All_weighted_boxes_fusion_faster(\n        detection,\n        proto,\n        img_shape,\n        mmdet_boxes,\n        mmdet_masks,\n        weights_mm = None,\n        weights_yolo=None,\n        iou_thr=0.55,\n        skip_box_thr=0.0\n):\n    '''\n    detection:[[box(x1,y1,x2,y2),conf,cls],...]\n    proto:[proto0,...]\n    yolo_weighted_boxes_fusion(det,proto,im.shape[2:])\n    mmdet_res:list[pr_i]\n\n    :param weights: list of weights for each model. Default: None, which means weight == 1 for each model\n    :param iou_thr: IoU value for boxes to be a match\n    :param skip_box_thr: exclude boxes with score lower than this variable\n    :param conf_type: how to calculate confidence in weighted boxes.\n        'avg': average value,\n        'max': maximum value,\n        'box_and_model_avg': box and model wise hybrid weighted average,\n        'absent_model_aware_avg': weighted average that takes into account the absent model.\n    :param allows_overflow: false if we want confidence score not exceed 1.0\n\n    :return: boxes: boxes coordinates (Order of boxes: x1, y1, x2, y2).\n    :return: scores: confidence scores\n    :return: labels: boxes labels\n    '''\n\n    # masks = process_mask(proto[i], det[:, 6:], det[:, :4], im.shape[2:], upsample=True)\n    # raw_mask = detection[:, 6:]\n    # print(\"orig:\",detection[0][0,:4])\n\n    ih, iw = img_shape\n    norm_box = [i[:, :4] for i in detection]\n    # box = detection[:, :4]\n    scores_list = [i[:, 4] for i in detection]\n    labels_list = [i[:, 5] for i in detection]\n\n    for i in range(len(norm_box)):\n        norm_box[i][:, 0] /= iw\n        norm_box[i][:, 2] /= iw\n        norm_box[i][:, 1] /= ih\n        norm_box[i][:, 3] /= ih\n\n    # print(\"after norm:\",detection[0][0,:4])\n\n    if weights_yolo is None:\n        weights_yolo = np.ones(len(norm_box))\n    if len(weights_yolo) != len(norm_box):\n        print('Warning: incorrect number of weights {}. Must be: {}. Set weights equal to 1.'.format(len(weights_yolo),\n                                                                                                     len(norm_box)))\n        weights_yolo = np.ones(len(norm_box))\n\n    weights_yolo = np.array(weights_yolo)\n\n    if weights_mm is None:\n        weights_mm = np.ones(len(mmdet_boxes))\n    if len(weights_mm) != len(mmdet_boxes):\n        print('Warning: incorrect number of weights {}. Must be: {}. Set weights equal to 1.'.format(len(weights_mm),\n                                                                                                     len(mmdet_boxes)))\n        weights_mm = np.ones(len(mmdet_boxes))\n\n    weights_mm = np.array(weights_mm)\n\n\n    filtered_boxes = prefilter_boxes_faster(norm_box, scores_list, labels_list, weights_yolo, skip_box_thr)\n    # filtered_boxes:tensor[[label, score, weight, model index, box index, x1, y1, x2, y2],...]\n\n    if filtered_boxes.shape[0] == 0:\n        return np.zeros((0, 4)), np.zeros((0,)), np.zeros((0,))\n\n    boxes = filtered_boxes.cpu().numpy()\n    new_boxes = []\n    weighted_boxes = []\n\n    # Clusterize boxes\n\n    for j in range(0, boxes.shape[0]):\n        # boxes[j] is every box whose cls = label\n        # boxes[j]:[label, score, weight, model index, box index, x1, y1, x2, y2]\n        index, best_iou = find_matching_box_fast(weighted_boxes, boxes[j], iou_thr)\n\n        # new_boxes:[[box,box2],[box1,box5]...]\n        # box:[label, score, weight, model index, box index, x1, y1, x2, y2]\n        if index != -1:\n            new_boxes[index].append(boxes[j])\n            weighted_boxes[index] = get_weighted_box_faster(new_boxes[index])\n            # weighted_masks[index] = yolo_get_weighted_mask(proto, detection, new_boxes[index], img_shape)\n\n\n        else:\n            new_boxes.append([boxes[j]])\n            weighted_boxes.append(boxes[j])\n    del boxes\n\n\n    ###################### mmdet #####################\n    # mmdet_boxes is list\n    for idx,mmdet_boxi in enumerate(mmdet_boxes):\n        # mmdet_boxi(np.array) is the boxes generated by i-th mmdet\n        for j in range(0, mmdet_boxi.shape[0]):\n            index, best_iou = find_matching_box_fast(weighted_boxes, mmdet_boxi[j], iou_thr)\n\n            if index != -1:\n                new_boxes[index].append(mmdet_boxi[j])\n                weighted_boxes[index] = get_weighted_box_faster(new_boxes[index])\n            else:\n                new_boxes.append([mmdet_boxi[j]])\n                weighted_boxes.append(mmdet_boxi[j])\n\n    weighted_boxes = np.array(weighted_boxes)\n    weighted_masks = []\n\n    for i in range(len(new_boxes)):\n        clustered_boxes = np.array(new_boxes[i])\n        new_mask = all_get_weighted_mask(proto, detection,clustered_boxes,mmdet_masks,img_shape)\n        weighted_masks.append(new_mask)\n        # score average\n        weighted_boxes[i, 1] = weighted_boxes[i, 1] * clustered_boxes.shape[0] / (weights_yolo.sum()+weights_mm.sum())\n\n    weighted_masks = np.concatenate(weighted_masks, axis=0)\n    pick_arg = weighted_boxes[:, 1].argsort()[::-1]\n    weighted_boxes = weighted_boxes[pick_arg]\n    weighted_masks = weighted_masks[pick_arg]\n    print(\"overall shape:\", weighted_boxes.shape, weighted_masks.shape)\n    # box,score,label,mask\n\n    return weighted_boxes[:, 5:], weighted_boxes[:, 1], weighted_boxes[:, 0], weighted_masks\n\ndef tta_process_mask(protos, masks_in, bboxes, shape, upsample=True,flip=\"ori\"):\n    \"\"\"\n    Crop before upsample.\n    proto_out: [mask_dim, mask_h, mask_w]\n    out_masks: [n, mask_dim], n is number of masks after nms\n    bboxes: [n, 4], n is number of masks after nms\n    shape:input_image_size, (h, w)\n\n    return: h, w, n\n    \"\"\"\n\n    c, mh, mw = protos.shape  # CHW\n    if flip=='rl':\n        protos=torch.flip(protos,dims=[1]) # flip W\n    elif flip=='ud':\n        protos=torch.flip(protos,dims=[2]) # flip W\n    masks = (masks_in @ protos.float().view(c, -1)).sigmoid().view(-1, mh, mw)  # CHW\n    \n    \n    if(isinstance(bboxes,np.ndarray)):\n        bboxes = torch.from_numpy(bboxes).to(protos.device)\n    \n    downsampled_bboxes = bboxes.clone()\n    downsampled_bboxes[0] *= mw\n    downsampled_bboxes[2] *= mw\n    downsampled_bboxes[3] *= mh\n    downsampled_bboxes[1] *= mh\n\n    x1, y1, x2, y2 = torch.chunk(downsampled_bboxes[:, None], 4, 0)  # x1 shape(1,1,n)\n    r = torch.arange(mw, device=masks.device, dtype=x1.dtype)[None, :]  # rows shape(1,w,1)\n    c = torch.arange(mh, device=masks.device, dtype=x1.dtype)[:, None]  # cols shape(h,1,1)\n    masks = masks[0] * (r >= x1) * (r < x2) * (c >= y1) * (c < y2)\n\n    if upsample:\n        masks = F.interpolate(masks[None, None, :, :], shape, mode='bilinear', align_corners=False)[0]\n\n    return masks.gt_(0.5).cpu().numpy()\n\ndef tta_get_weighted_mask(proto, detection, cluster_boxes, mmdet_mask, img_shape, wbox, N_tta_model=3):\n    mask = np.zeros(img_shape, dtype=np.float32)[None, :, :]\n    # mask = torch.zeros(img_shape, dtype=proto[0].dtype, device = proto[0].device)\n    mask_conf = 0\n    conf = 0\n    model_num = 0\n    # model_num = len(cluster_boxes)\n    # conf_list = []\n\n    for index, b in enumerate(cluster_boxes):\n        s = b[1]\n        model_index = int(b[3])\n        if (model_index < N_tta_model):\n            # print(\"weighted normal mask\")\n            m = tta_process_mask(proto[model_index], detection[model_index][int(b[4]), 6:],\n                                 detection[model_index][int(b[4]), :4],\n                                 img_shape)\n            conf += s\n            model_num += 1\n        elif (model_index < 2 * N_tta_model):\n            # print(\"weighted rl mask\")\n            m = tta_process_mask(proto[model_index], detection[model_index][int(b[4]), 6:], wbox[index][5:],\n                                 img_shape, flip=\"rl\")\n            # print(\"weighted ud mask\")\n        elif (model_index < 3 * N_tta_model):\n            m = tta_process_mask(proto[model_index], detection[model_index][int(b[4]), 6:], wbox[index][5:],\n                                 img_shape, flip=\"ud\")\n        else:\n            # model_index > 10 is mmdet model\n            m = mmdet_mask[model_index - 10][int(b[4])]\n            conf += s\n            model_num += 1\n\n        mask = mask + s * m\n        mask_conf += s\n        # conf_list.append(s)\n\n    # score = np.max(conf_list)\n    mask = mask / mask_conf\n    \n\n    return mask,conf/model_num,model_num   #  conf, model_num  # , score, conf_list\n\n\n\n\n\ndef All_fusion_tta_faster(\n        detection,\n        proto,\n        img_shape,\n        mmdet_boxes,\n        mmdet_masks,\n        N_tta_model = 3,\n        weights_mm = None,\n        weights_yolo = None,\n        iou_thr=0.55,\n        skip_box_thr=0.0\n):\n    '''\n\n    N_tta_model: the number of model using TTA\n    detection:[[box(x1,y1,x2,y2),conf,cls,32],...]\n    proto:[proto0,...]\n    yolo_weighted_boxes_fusion(det,proto,im.shape[2:])\n    mmdet_res:list[pr_i]\n\n    :param weights: list of weights for each model. Default: None, which means weight == 1 for each model\n    :param iou_thr: IoU value for boxes to be a match\n    :param skip_box_thr: exclude boxes with score lower than this variable\n    :param conf_type: how to calculate confidence in weighted boxes.\n        'avg': average value,\n        'max': maximum value,\n        'box_and_model_avg': box and model wise hybrid weighted average,\n        'absent_model_aware_avg': weighted average that takes into account the absent model.\n    :param allows_overflow: false if we want confidence score not exceed 1.0\n\n    :return: boxes: boxes coordinates (Order of boxes: x1, y1, x2, y2).\n    :return: scores: confidence scores\n    :return: labels: boxes labels\n    '''\n\n    # masks = process_mask(proto[i], det[:, 6:], det[:, :4], im.shape[2:], upsample=True)\n    # raw_mask = detection[:, 6:]\n    # print(\"orig:\",detection[0][0,:4])\n\n\n    ih, iw = img_shape\n    # box normalization\n    for i in range(len(detection)):\n        detection[i][:, 0] /= iw\n        detection[i][:, 2] /= iw\n        detection[i][:, 1] /= ih\n        detection[i][:, 3] /= ih\n        \n    '''for i in range(N_tta_model,len(detection)-N_tta_model):\n        detection[i][:, 0] = detection[i][:, 0]\n        detection[i][:, 2] = detection[i][:, 2]\n    for i in range(N_tta_model*2,len(detection)):\n        detection[i][:, 1] = detection[i][:, 1]\n        detection[i][:, 3] = detection[i][:, 3]'''\n    \n    \n    ### box/scores/label for original image\n    norm_box = [i[:, :4] for i in detection[:N_tta_model]]\n    scores_list = [i[:, 4] for i in detection[:N_tta_model]]\n    labels_list = [i[:, 5] for i in detection[:N_tta_model]]\n\n\n    if weights_yolo is None:\n        weights_yolo = np.ones(len(norm_box))\n    if len(weights_yolo) != len(norm_box):\n        print('Warning: incorrect number of weights {}. Must be: {}. Set weights equal to 1.'.format(len(weights_yolo),\n                                                                                                     len(norm_box)))\n        weights_yolo = np.ones(len(norm_box))\n    weights_yolo = np.array(weights_yolo)\n\n\n\n    if weights_mm is None:\n        weights_mm = np.ones(len(mmdet_boxes))\n    if len(weights_mm) != len(mmdet_boxes):\n        print('Warning: incorrect number of weights {}. Must be: {}. Set weights equal to 1.'.format(len(weights_mm),\n                                                                                                     len(mmdet_boxes)))\n        weights_mm = np.ones(len(mmdet_boxes))\n    weights_mm = np.array(weights_mm)\n\n\n    filtered_boxes = prefilter_boxes_faster(norm_box, scores_list, labels_list, weights_yolo)\n\n\n    if filtered_boxes.shape[0] == 0:\n        return np.zeros((0, 4)), np.zeros((0,)), np.zeros((0,))\n\n    boxes = filtered_boxes.cpu().numpy()\n\n    clusters_list = []\n    weighted_boxes = []\n\n    # Clusterize boxes\n    for j in range(0, boxes.shape[0]):\n        index, best_iou = find_matching_box_fast(weighted_boxes, boxes[j], iou_thr)\n        if index != -1:\n            clusters_list[index].append(boxes[j])\n            weighted_boxes[index] = get_weighted_box_faster(clusters_list[index])\n        else:\n            clusters_list.append([boxes[j]])\n            weighted_boxes.append(boxes[j])\n    del boxes\n\n    # TTA cluster\n    TTA_iou = 0.80\n    rl_boxes_list = tta_box_preprocecss(detection[N_tta_model:2*N_tta_model],mode=\"rl\",weights = weights_yolo)\n    ud_boxes_list = tta_box_preprocecss(detection[N_tta_model*2:],mode = \"ud\",weights = weights_yolo)\n    #print(\"rlboxes:\",rl_boxes_list[0][:5,:])\n\n    for idx,c_boxes in enumerate(clusters_list):\n        #print(\"before TTA:\",len(clusters_list[idx]))\n        for i_box in c_boxes.copy():\n            # box:[label, score, weight, model index, box index, x1, y1, x2, y2]\n\n            model_index = int(i_box[3])\n\n\n            index, best_iou = find_matching_box_fast(rl_boxes_list[model_index], i_box, iou_thr)\n            if index != -1:\n                if (best_iou > TTA_iou):\n                    #print(\"rlbox:\",rl_boxes_list[model_index][index].shape)\n                    clusters_list[idx].append(rl_boxes_list[model_index][index])\n\n            index, best_iou = find_matching_box_fast(ud_boxes_list[model_index], i_box, iou_thr)\n            if index != -1:\n                if(best_iou > TTA_iou):\n                    #print(\"udbox:\", ud_boxes_list[model_index][index].shape)\n                    clusters_list[idx].append(ud_boxes_list[model_index][index])\n\n        #print(\"after TTA:\", len(clusters_list[idx]))\n        weighted_boxes[idx] = get_weighted_box_faster(clusters_list[idx])\n        '''for index,wbox in enumerate(clusters_list[idx]):\n            print(f\"{index}-th box:\",wbox)'''\n\n\n        #print(\"after TTA:\",len(c_boxes))\n\n\n    ###################### mmdet #####################\n    # mmdet_boxes is list\n    for idx,mmdet_boxi in enumerate(mmdet_boxes):\n        # mmdet_boxi(np.array) is the boxes generated by i-th mmdet\n        for j in range(0, mmdet_boxi.shape[0]):\n            index, best_iou = find_matching_box_fast(weighted_boxes, mmdet_boxi[j], iou_thr)\n\n            if index != -1:\n                clusters_list[index].append(mmdet_boxi[j])\n                weighted_boxes[index] = get_weighted_box_faster(clusters_list[index])\n            else:\n                clusters_list.append([mmdet_boxi[j]])\n                weighted_boxes.append(mmdet_boxi[j])\n\n    weighted_boxes = np.array(weighted_boxes)\n    weighted_masks = []\n\n    for i in range(len(clusters_list)):\n        clustered_boxes = np.array(clusters_list[i])\n        \n        \n        new_mask,new_conf,model_num = tta_get_weighted_mask(proto, detection, clustered_boxes, mmdet_masks, img_shape, weighted_boxes)\n        '''for cb in clustered_boxes:\n            print(f\"{i}-th box:\",cb)\n        print(\"new score:\",new_conf,model_num)'''\n        \n        weighted_masks.append(new_mask)\n        # score average\n        # weighted_boxes[i, 1] = new_conf * model_num / (weights_yolo.sum()+weights_mm.sum())\n        weighted_boxes[i, 1] = new_conf\n        #if(weighted_boxes[i, 1]>1): weighted_boxes[i, 1] = 1.0\n\n    weighted_masks = np.concatenate(weighted_masks, axis=0)\n    pick_arg = weighted_boxes[:, 1].argsort()[::-1]\n    weighted_boxes = weighted_boxes[pick_arg]\n    weighted_masks = weighted_masks[pick_arg]\n    print(\"overall shape:\", weighted_boxes.shape, weighted_masks.shape)\n    # box,score,label,mask\n    \n    \n    \n    '''mid_mask = [] \n    for index,rlb in enumerate(rl_boxes_list[1]):\n        m = tta_process_mask(proto[4], detection[4][int(rlb[4]), 6:], torch.from_numpy(rlb[-4:]).to(proto[4].device),img_shape,flip=\"rl\") \n        mid_mask.append(m)\n    mid_mask = np.concatenate(mid_mask, axis=0)\n    mid_mask = dilate_predict_mask(mid_mask)\n\n    all_x = np.zeros((512,512), dtype=np.uint8)\n    if len(mid_mask)>0:\n        num_mask = len(mid_mask)\n        \n        for i in range(num_mask):\n            m = mid_mask[i]>0\n            #if all_x[m].mean() > 0.7: continue\n            all_x[m] = 1\n    \n    plt.figure()\n    plt.imshow(all_x)\n    plt.show()\n    \n    \n    mid_mask = [] \n    for index,rlb in enumerate(ud_boxes_list[1]):\n        m = tta_process_mask(proto[4], detection[4][int(rlb[4]), 6:], detection[4][int(rlb[4]), :4],img_shape,flip=\"rl\") \n        mid_mask.append(m)\n    mid_mask = np.concatenate(mid_mask, axis=0)\n    mid_mask = dilate_predict_mask(mid_mask)\n\n    all_x = np.zeros((512,512), dtype=np.uint8)\n    if len(mid_mask)>0:\n        num_mask = len(mid_mask)\n        \n        for i in range(num_mask):\n            m = mid_mask[i]>0\n            #if all_x[m].mean() > 0.7: continue\n            all_x[m] = 1\n    \n    plt.figure()\n    plt.imshow(all_x)\n    plt.show()'''\n    \n\n    return weighted_boxes[:, 5:], weighted_boxes[:, 1], weighted_boxes[:, 0], weighted_masks","metadata":{"execution":{"iopub.status.busy":"2023-07-31T13:08:12.917162Z","iopub.execute_input":"2023-07-31T13:08:12.917579Z","iopub.status.idle":"2023-07-31T13:08:13.042854Z","shell.execute_reply.started":"2023-07-31T13:08:12.917545Z","shell.execute_reply":"2023-07-31T13:08:13.041754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_all(model_files,mmdet_checkpoints,mmdet_cfg,image_file, device, augment = False, hsv_h = 1.0, hsv_s = 1.0, hsv_v = 1.0):\n    im0 = cv2.imread(image_file)  # BGR\n    if augment:\n        im0 = augment_hsv(im0, h_gain=hsv_h, s_gain=hsv_s, v_gain=hsv_v)\n    # Resize and pad image while meeting stride-multiple constraints\n    #im = letterbox(im0, self.img_size, stride=self.stride, auto=self.auto)[0]  # padded resize\n    im = im0\n    im = im.transpose((2, 0, 1))[::-1]  # HWC to CHW, BGR to RGB\n    im = np.ascontiguousarray(im)  # contiguous\n\n    im = torch.from_numpy(im).to(device)\n    im = im.float()  # uint8 to fp16/32\n    im /= 255  # 0 - 255 to 0.0 - 1.0\n    image_shape = im.shape[1:]\n    print(image_shape)\n    im = im[None]  # expand for batch dim\n    \n    \n    plt.figure()\n    plt.subplot(1,3,1)\n    plt.imshow(im0)\n    plt.subplot(1,3,2)\n    plt.imshow(torch.flip(im[0],dims=[2]).cpu().numpy().transpose((1, 2, 0)))\n    plt.subplot(1,3,3)\n    plt.imshow(torch.flip(im[0],dims=[1]).cpu().numpy().transpose((1, 2, 0)))\n    plt.show()\n    \n\n    predictions = []\n    protos = []\n    N_model = len(model_files)\n    ########################## Yolo models ###########################\n    for idx,model_f in enumerate(model_files):\n        model_i = DetectMultiBackend(model_f, device=device, dnn=False, data=data_yaml, fp16=False)\n        model_i.warmup(imgsz=(1,3,512,512))  \n        pred, out = model_i(im, augment=False, visualize=False)\n        proto = out[1]\n        pred_masks = non_max_suppression(pred, conf_thres=0.001, iou_thres=0.6, classes=0, agnostic=False, max_det=1000, nm=32) #nm = num_mask\n        print(\"ori_num:\",pred_masks[0].shape)\n        predictions.append(pred_masks[0])\n        protos.append(proto[0])\n        \n        torch.cuda.empty_cache()\n        del model_i\n        del out,pred,pred_masks\n        \n    for idx,model_f in enumerate(model_files):\n        model_i = DetectMultiBackend(model_f, device=device, dnn=False, data=data_yaml, fp16=False)\n        model_i.warmup(imgsz=(1,3,512,512))   \n        pred_lr, out_lr = model_i(torch.flip(im,dims=[2]), augment=False, visualize=False)\n        proto_lr = out_lr[1]\n        pred_masks_lr = non_max_suppression(pred_lr, conf_thres=0.001, iou_thres=0.6, classes=0, agnostic=False, max_det=1000, nm=32)\n        print(\"filplr_num:\",pred_masks_lr[0].shape)\n        predictions.append(pred_masks_lr[0])\n        protos.append(proto_lr[0])\n        torch.cuda.empty_cache()\n        del model_i\n        del out_lr,pred_lr,pred_masks_lr\n        \n    for idx,model_f in enumerate(model_files):\n        model_i = DetectMultiBackend(model_f, device=device, dnn=False, data=data_yaml, fp16=False)\n        model_i.warmup(imgsz=(1,3,512,512)) \n        #  print(proto[0,:5,:5],torch.flip(proto_lr,dims=[2])[0,:5,:5])\n        pred_ud, out_ud = model_i(torch.flip(im,dims=[1]), augment=False, visualize=False)\n        proto_ud = out_ud[1]\n        pred_masks_ud = non_max_suppression(pred_ud, conf_thres=0.001, iou_thres=0.6, classes=0, agnostic=False, max_det=1000, nm=32)\n        print(\"filpud_num:\",pred_masks_ud[0].shape)\n        predictions.append(pred_masks_ud[0])\n        protos.append(proto_ud[0])\n        torch.cuda.empty_cache()\n        del model_i\n        del out_ud,pred_ud,pred_masks_ud\n    \n    #print(\"yolo predictions:\",len(predictions))\n    pred_count = 0\n    for i in range(len(model_files)):\n        pred_count+=predictions[i].shape[0]\n    print(\"yolo pred number:\",pred_count)\n    \n    \n    \n    ########################## mmdet models ###########################\n    mmdet_boxes = []\n    mmdet_masks = []\n    \n    for idx,model_f in enumerate(mmdet_checkpoints):\n        model_i = init_detector(mmdet_cfg, checkpoint=model_f,device=\"cuda:0\")\n        result = inference_detector(model_i, imgs=image_file)\n        \n        pr_i = result.pred_instances\n        \n        mmdet_b,mmdet_m = preprocess_mmdet_box_faster(pr_i,idx,image_shape,weights=1.0)\n        \n        mmdet_boxes.append(mmdet_b)\n        mmdet_masks.append(mmdet_m)\n        torch.cuda.empty_cache()\n        \n        del model_i,result \n        \n    print(\"mmdet0 pred number:\",mmdet_boxes[0].shape[0])\n    \n    ########################## fusion #############################\n    _,out_conf,_,out_mask = All_fusion_tta_faster(predictions,protos,image_shape,mmdet_boxes,mmdet_masks,weights_yolo = [1,1,1],iou_thr=0.7)\n\n\n    print(\"WBF out_mask,out_conf\",out_mask.shape,out_conf.shape)\n    \n    \n\n        \n  \n    return out_mask, out_conf\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-31T12:26:57.561514Z","iopub.execute_input":"2023-07-31T12:26:57.562396Z","iopub.status.idle":"2023-07-31T12:26:57.586982Z","shell.execute_reply.started":"2023-07-31T12:26:57.562356Z","shell.execute_reply":"2023-07-31T12:26:57.585660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#############################################################################################################\nmode = 'submit' #'submit' #debug\n\nif mode=='debug':\n    image_dir = '/kaggle/input/hubmap-hacking-the-human-vasculature/train'\n    image_id = glob(f'{image_dir}/*.tif')\n    image_id = [f.split('/')[-1][:-4] for f in image_id] #[:25]\n    \n\nif mode=='submit': \n    image_dir = '/kaggle/input/hubmap-hacking-the-human-vasculature/test' \n    valid_df = pd.read_csv('/kaggle/input/hubmap-hacking-the-human-vasculature/sample_submission.csv')\n    #image_id = valid_df['id'].values\n    \n    image_id = glob(f'{image_dir}/*.tif')\n    image_id = [f.split('/')[-1][:-4] for f in image_id]\n    \nprint('image_dir', image_dir)\nprint(len(image_id),image_id )\n\n\ndata_yaml='/kaggle/working/hubmap-predict.yaml'\n\n# Create a yaml file as expected by YOLOv7 (and others)\nyaml_text = \"\"\"\n# class names\nnames: \n  0: blood_vessel\n  1: glomerulus\n  2: unsure\n\"\"\"\nwith open(data_yaml, 'w') as text_file:\n    text_file.write(yaml_text)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-31T12:26:59.806604Z","iopub.execute_input":"2023-07-31T12:26:59.807678Z","iopub.status.idle":"2023-07-31T12:26:59.825904Z","shell.execute_reply.started":"2023-07-31T12:26:59.807640Z","shell.execute_reply":"2023-07-31T12:26:59.824758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 多个YOLO模型","metadata":{}},{"cell_type":"code","source":"model_files = ['/kaggle/input/yolov7-512-weights/fold1_best.pt','/kaggle/input/yolov7-512-weights/v7x_best.pt','/kaggle/input/yolov7-512-weights/fold4_best.pt']\n\ndevice = select_device('0') ","metadata":{"execution":{"iopub.status.busy":"2023-07-31T12:27:03.326567Z","iopub.execute_input":"2023-07-31T12:27:03.326919Z","iopub.status.idle":"2023-07-31T12:27:03.338844Z","shell.execute_reply.started":"2023-07-31T12:27:03.326890Z","shell.execute_reply":"2023-07-31T12:27:03.337524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 多个mmdet模型","metadata":{}},{"cell_type":"code","source":"mmdet_cfg = Config.fromfile(\"/kaggle/input/hubmap-mmdet-fold-1/custom_config.py\")\nmmdet_cfg.work_dir = \"/kaggle/working/work_dir_test\"\nmmdet_vis_backends = [dict(type='LocalVisBackend')]\nmmdet_cfg.visualizer = dict(type='DetLocalVisualizer', vis_backends=mmdet_vis_backends, name='visualizer')","metadata":{"execution":{"iopub.status.busy":"2023-07-31T10:24:21.320067Z","iopub.execute_input":"2023-07-31T10:24:21.320319Z","iopub.status.idle":"2023-07-31T10:24:21.350453Z","shell.execute_reply.started":"2023-07-31T10:24:21.320297Z","shell.execute_reply":"2023-07-31T10:24:21.349624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mmdet_checkpoint_files = [\"/kaggle/input/hubmap-mmdet-fold-1/best_coco_segm_mAP_epoch_16.pth\",\"/kaggle/input/saved-model/best_coco_segm_mAP_epoch_15.pth/best_coco_segm_mAP_epoch_15.pth\"]","metadata":{"execution":{"iopub.status.busy":"2023-07-31T10:24:21.354762Z","iopub.execute_input":"2023-07-31T10:24:21.355580Z","iopub.status.idle":"2023-07-31T10:24:21.359797Z","shell.execute_reply.started":"2023-07-31T10:24:21.355549Z","shell.execute_reply":"2023-07-31T10:24:21.358722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 运行集成&生成结果：","metadata":{}},{"cell_type":"code","source":"submission=[] \n\n\nprint(\"image number:\",len(image_id))\nfor imt,id in enumerate(image_id):\n    print(imt,id) \n    \n    prediction_string = ''\n    image_file = f'{image_dir}/{id}.tif' \n    \n    \n    predict_start = time.time()\n    mask, conf = predict_all(model_files,mmdet_checkpoint_files,mmdet_cfg,image_file, device, augment = True,hsv_s = 1.1,hsv_v = 1.1)\n    predict_end = time.time()\n    print(\"predict_all time:\",predict_end - predict_start)\n\n\n    mask = dilate_predict_mask(mask)\n    # masks.append(mask)\n    # confs.append(conf)\n\n    all_x = np.zeros((512,512), dtype=np.uint8)\n    if len(mask)>0:\n        num_mask = len(mask)\n        for i in range(num_mask):\n            m = mask[i]>0\n            #if all_x[m].mean() > 0.7: continue\n            all_x[m] = 1\n\n            e = encode_binary_mask(m)\n            if i == 0:\n                prediction_string = f'0 {conf[i]} {e.decode(\"utf-8\")}'\n            else:\n                prediction_string += f' 0 {conf[i]} {e.decode(\"utf-8\")}'\n\n        print(\"mask number:\",num_mask)\n\n\n    del mask\n        \n        \n        \n    submission.append({\n        'id':id,\n        'height':512,\n        'width':512,\n        'prediction_string':prediction_string,\n        \n    })    \n\n\n    #print(prediction_string)","metadata":{"execution":{"iopub.status.busy":"2023-07-31T13:08:17.573362Z","iopub.execute_input":"2023-07-31T13:08:17.574055Z","iopub.status.idle":"2023-07-31T13:08:29.756710Z","shell.execute_reply.started":"2023-07-31T13:08:17.574020Z","shell.execute_reply":"2023-07-31T13:08:29.755571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.imshow(all_x)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-31T13:04:10.559472Z","iopub.execute_input":"2023-07-31T13:04:10.559872Z","iopub.status.idle":"2023-07-31T13:04:10.814058Z","shell.execute_reply.started":"2023-07-31T13:04:10.559837Z","shell.execute_reply":"2023-07-31T13:04:10.813122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame(submission)\nsubmission_df.to_csv('submission.csv',index=False)\nprint(submission_df)\nprint('SUBMIT OK !!!')","metadata":{"execution":{"iopub.status.busy":"2023-07-31T13:09:17.349302Z","iopub.execute_input":"2023-07-31T13:09:17.349724Z","iopub.status.idle":"2023-07-31T13:09:17.367963Z","shell.execute_reply.started":"2023-07-31T13:09:17.349688Z","shell.execute_reply":"2023-07-31T13:09:17.366646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}