{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport numpy as np\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\nimport glob\nimport shutil\nimport sys\nsys.path.append('../input/tensorflow-great-barrier-reef')\nimport torch\nfrom PIL import Image\nimport ast\nsys.path.append('../input/weightedboxesfusion/Weighted-Boxes-Fusion')\n\n# 1280\n# x1.25 = 1600\n# x1.5  = 1920\n# x1.75 = 2240\n# x2    = 2560\n# x2.25 = 2880\n# x2.5  = 3200\n# x3    = 3840\n# x3.25 = 4160\n# x3.5  = 4480\n# x3.75 = 4800\n\n\nBest_Models = ['../input/m6-2560-20ep-weights/fold0.pt',\n               '../input/m6-2560-20ep-randomresize-weights/fold1.pt',\n               '../input/m6-2560-20ep-weights/fold2.pt',\n               '../input/m6-2560-20ep-randomresize-weights/fold3.pt',\n               '../input/m6-2560-20ep-randomresize-weights/fold4.pt'] + \\\n               glob.glob('../input/m6-2880-15epfinetune/*pt')\n#glob.glob('../input/m6-2560-20ep-video-weights/*pt') + \\\n\nprint(Best_Models)\n\n# best params in CV\n# Input:2560\n# TTA:True\n# NMS-IoU:0.35\n# Expansion:-0.04\n\n#IMG_SIZES=[3840]*len(Best_Models)\nIMG_SIZES=[2560]*5 + [2880]*5\n\n#TTAs=[False]*len(Best_Models)\nTTAs=[True]*5 + [True]*5\n\nIOU=0.4\nWBF_IOU=0.475\nENSEMBLE_METHOD='WBF'#NMS,NMW,WBF\n\n# use custom tta\nREPO_DIR_PATH = '../input/custom-tta/yolo_repo_vh'# hflip TTA\n\nBOX_EXPANSION=-0.04\n\nCONFS = [0.4]*len(Best_Models)\n#CONFS = [0.134, 0.156, 0.234, 0.235, 0.16]\nFINAL_CONF = 0#0.4#0.4\n\nTRAIN_PATH = '/kaggle/input/tensorflow-great-barrier-reef'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-15T16:48:54.45244Z","iopub.execute_input":"2022-02-15T16:48:54.452775Z","iopub.status.idle":"2022-02-15T16:48:55.968978Z","shell.execute_reply.started":"2022-02-15T16:48:54.452695Z","shell.execute_reply":"2022-02-15T16:48:55.968276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert ENSEMBLE_METHOD in [\"WBF\", \"NMW\", \"NMS\"]","metadata":{"execution":{"iopub.status.busy":"2022-02-13T12:14:03.794191Z","iopub.execute_input":"2022-02-13T12:14:03.794449Z","iopub.status.idle":"2022-02-13T12:14:03.812108Z","shell.execute_reply.started":"2022-02-13T12:14:03.794422Z","shell.execute_reply":"2022-02-13T12:14:03.811192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def voc2yolo(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    voc  => [x1, y1, x2, y1]\n    yolo => [xmid, ymid, w, h] (normalized)\n    \"\"\"\n    \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]/ image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]/ image_height\n    \n    w = bboxes[..., 2] - bboxes[..., 0]\n    h = bboxes[..., 3] - bboxes[..., 1]\n    \n    bboxes[..., 0] = bboxes[..., 0] + w/2\n    bboxes[..., 1] = bboxes[..., 1] + h/2\n    bboxes[..., 2] = w\n    bboxes[..., 3] = h\n    \n    return bboxes\n\ndef yolo2voc(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    voc  => [x1, y1, x2, y1]\n    \n    \"\"\" \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height\n    \n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]\n    \n    return bboxes\n\ndef coco2yolo(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    coco => [xmin, ymin, w, h]\n    yolo => [xmid, ymid, w, h] (normalized)\n    \"\"\"\n    \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    # normolizinig\n    bboxes[..., [0, 2]]= bboxes[..., [0, 2]]/ image_width\n    bboxes[..., [1, 3]]= bboxes[..., [1, 3]]/ image_height\n    \n    # converstion (xmin, ymin) => (xmid, ymid)\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]/2\n    \n    return bboxes\n\ndef yolo2coco(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    coco => [xmin, ymin, w, h]\n    \n    \"\"\" \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    # denormalizing\n    bboxes[..., [0, 2]]= bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]]= bboxes[..., [1, 3]]* image_height\n    \n    # converstion (xmid, ymid) => (xmin, ymin) \n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    \n    return bboxes\n\ndef voc2coco(bboxes, image_height=720, image_width=1280):\n    bboxes  = voc2yolo(bboxes, image_height, image_width)\n    bboxes  = yolo2coco(bboxes, image_height, image_width)\n    return bboxes\n\ndef coco2voc(bboxes, image_height=720, image_width=1280):\n    bboxes  = coco2yolo(bboxes, image_height, image_width)\n    bboxes  = yolo2voc(bboxes, image_height, image_width)\n    return bboxes\n\n\ndef load_image(image_path):\n    return cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n\n\ndef plot_one_box(x, img, color=None, label=None, line_thickness=None):\n    # Plots one bounding box on image img\n    tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1  # line/font thickness\n    color = color or [random.randint(0, 255) for _ in range(3)]\n    c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\n    cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)\n    if label:\n        tf = max(tl - 1, 1)  # font thickness\n        t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\n        c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\n        cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA)  # filled\n        cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)\n\ndef draw_bboxes(img, bboxes, classes, class_ids, colors = None, show_classes = None, bbox_format = 'yolo', class_name = False, line_thickness = 2):  \n     \n    image = img.copy()\n    show_classes = classes if show_classes is None else show_classes\n    colors = (0, 255 ,0) if colors is None else colors\n    \n    if bbox_format == 'yolo':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes:\n            \n                x1 = round(float(bbox[0])*image.shape[1])\n                y1 = round(float(bbox[1])*image.shape[0])\n                w  = round(float(bbox[2])*image.shape[1]/2) #w/2 \n                h  = round(float(bbox[3])*image.shape[0]/2)\n\n                voc_bbox = (x1-w, y1-h, x1+w, y1+h)\n                plot_one_box(voc_bbox, \n                             image,\n                             color = color,\n                             label = cls if class_name else str(get_label(cls)),\n                             line_thickness = line_thickness)\n            \n    elif bbox_format == 'coco':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes:            \n                x1 = int(round(bbox[0]))\n                y1 = int(round(bbox[1]))\n                w  = int(round(bbox[2]))\n                h  = int(round(bbox[3]))\n\n                voc_bbox = (x1, y1, x1+w, y1+h)\n                plot_one_box(voc_bbox, \n                             image,\n                             color = color,\n                             label = cls if class_name else str(cls_id),\n                             line_thickness = line_thickness)\n\n    elif bbox_format == 'voc_pascal':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes: \n                x1 = int(round(bbox[0]))\n                y1 = int(round(bbox[1]))\n                x2 = int(round(bbox[2]))\n                y2 = int(round(bbox[3]))\n                voc_bbox = (x1, y1, x2, y2)\n                plot_one_box(voc_bbox, \n                             image,\n                             color = color,\n                             label = cls if class_name else str(cls_id),\n                             line_thickness = line_thickness)\n    else:\n        raise ValueError('wrong bbox format')\n\n    return image\n\ndef get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndef get_imgsize(row):\n    row['width'], row['height'] = imagesize.get(row['image_path'])\n    return row\n\n\nnp.random.seed(8)\ncolors = (np.random.randint(0, 255), np.random.randint(0, 255), np.random.randint(0, 255))\ncolors=(255,0,0)","metadata":{"execution":{"iopub.status.busy":"2022-02-08T16:58:14.425134Z","iopub.execute_input":"2022-02-08T16:58:14.425531Z","iopub.status.idle":"2022-02-08T16:58:14.46414Z","shell.execute_reply.started":"2022-02-08T16:58:14.42549Z","shell.execute_reply":"2022-02-08T16:58:14.463354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolo-arial/Arial.ttf /root/.config/Ultralytics/","metadata":{"execution":{"iopub.status.busy":"2022-02-08T16:58:14.466615Z","iopub.execute_input":"2022-02-08T16:58:14.466872Z","iopub.status.idle":"2022-02-08T16:58:15.870004Z","shell.execute_reply.started":"2022-02-08T16:58:14.466835Z","shell.execute_reply":"2022-02-08T16:58:15.869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(Best_Model, conf=0.25, iou=0.50):\n    model = torch.hub.load(REPO_DIR_PATH,\n                           'custom',\n                           path=Best_Model,\n                           source='local',\n                           force_reload=True)  # local repo\n    model.conf = conf  # NMS confidence threshold\n    model.iou  = iou  # NMS IoU threshold\n    model.classes = None   # (optional list) filter by class, i.e. = [0, 15, 16] for persons, cats and dogs\n    model.multi_label = False  # NMS multiple labels per box\n    model.max_det = 1000  # maximum number of detections per image\n    return model\n\n\ndef predict(model, img, size=768, augment=False):\n    height, width = img.shape[:2]\n    results = model(img, size=size, augment=augment)  # custom inference size\n    preds   = results.pandas().xyxy[0]\n    bboxes  = preds[['xmin','ymin','xmax','ymax']].values\n    if len(bboxes):\n        bboxes  = voc2coco(bboxes,height,width).astype(int)\n        confs   = preds.confidence.values\n        return bboxes, confs\n    else:\n        return [],[]\n    \ndef format_prediction(bboxes, confs):\n    annot = ''\n    if len(bboxes)>0:\n        for idx in range(len(bboxes)):\n            xmin, ymin, w, h = bboxes[idx]\n            conf             = confs[idx]\n            annot += f'{conf} {xmin} {ymin} {w} {h}'\n            annot +=' '\n        annot = annot.strip(' ')\n    return annot\n\ndef show_img(img, bboxes, bbox_format='yolo'):\n    names  = ['starfish']*len(bboxes)\n    labels = [0]*len(bboxes)\n    img    = draw_bboxes(img = img,\n                           bboxes = bboxes, \n                           classes = names,\n                           class_ids = labels,\n                           class_name = True, \n                           colors = colors, \n                           bbox_format = bbox_format,\n                           line_thickness = 2)\n    return Image.fromarray(img).resize((800, 400))","metadata":{"execution":{"iopub.status.busy":"2022-02-08T16:58:15.87258Z","iopub.execute_input":"2022-02-08T16:58:15.872866Z","iopub.status.idle":"2022-02-08T16:58:15.884585Z","shell.execute_reply.started":"2022-02-08T16:58:15.872827Z","shell.execute_reply":"2022-02-08T16:58:15.883899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def expand_bboxes(bboxes, expansion=0.1):\n    if len(bboxes) == 0:\n        return bboxes\n    else:\n        new_bboxes = []\n        for bbox in bboxes:\n            xmin, ymin, w, h = bbox\n            delta_w = w*expansion*0.5\n            delta_h = h*expansion*0.5\n            \n            new_xmin = int(round(np.clip(xmin - delta_w, 0, 1280)))\n            new_ymin = int(round(np.clip(ymin - delta_h, 0, 720)))\n            new_w = int(round(w + 2.0*delta_w))\n            new_h = int(round(h + 2.0*delta_h))\n            \n            new_bboxes.append([new_xmin, new_ymin, new_w, new_h])\n            \n        return np.array(new_bboxes)\n    \ndef filter_bboxes(bboxes, confs, conf_thresh):\n    if len(confs) == 0:\n        return [], []\n    else:\n        filtered_bboxes, filtered_confs = [], []\n        for bbox, conf in zip(bboxes, confs):\n            if conf > conf_thresh:\n                filtered_bboxes.append(bbox)\n                filtered_confs.append(conf)\n        return filtered_bboxes, filtered_confs","metadata":{"execution":{"iopub.status.busy":"2022-02-08T16:58:15.885921Z","iopub.execute_input":"2022-02-08T16:58:15.886474Z","iopub.status.idle":"2022-02-08T16:58:15.900646Z","shell.execute_reply.started":"2022-02-08T16:58:15.886436Z","shell.execute_reply":"2022-02-08T16:58:15.900013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ensemble_boxes import *\n\ndef run_wbf(bboxes, confs, image_size=1280, iou_thr=0.5, skip_box_thr=0.001, weights=None, method='WBF'):\n    num_ensemble = len(bboxes)\n    boxes =  [bbox/(image_size) for bbox in bboxes]\n    \n    scores = [conf for conf in confs]\n    labels = [np.ones(conf.shape[0]) for conf in confs]\n    \n    if method == 'WBF':\n        boxes, scores, labels = weighted_boxes_fusion(boxes,\n                                                      scores,\n                                                      labels,\n                                                      weights=[1]*num_ensemble,\n                                                      iou_thr=iou_thr,\n                                                      skip_box_thr=skip_box_thr)\n    elif method == 'NMW':\n        boxes, scores, labels = non_maximum_weighted(boxes,\n                                                     scores,\n                                                     labels,\n                                                     weights=[1]*num_ensemble,\n                                                     iou_thr=iou_thr,\n                                                     skip_box_thr=skip_box_thr)\n    elif method == 'NMS':\n        boxes, scores, labels = nms(boxes,\n                                    scores,\n                                    labels,\n                                    weights=[1]*num_ensemble,\n                                    iou_thr=iou_thr)\n    else:\n        0/0\n        \n    \n    boxes = boxes*(image_size-1)\n    boxes=voc2coco(boxes).astype(int)\n    \n    return boxes, scores, labels","metadata":{"execution":{"iopub.status.busy":"2022-02-08T16:58:15.902339Z","iopub.execute_input":"2022-02-08T16:58:15.902829Z","iopub.status.idle":"2022-02-08T16:58:16.740011Z","shell.execute_reply.started":"2022-02-08T16:58:15.902779Z","shell.execute_reply":"2022-02-08T16:58:16.739167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()# initialize the environment\niter_test = env.iter_test()      # an iterator which loops over the test set and sample submission","metadata":{"execution":{"iopub.status.busy":"2022-02-08T16:58:16.75208Z","iopub.execute_input":"2022-02-08T16:58:16.752761Z","iopub.status.idle":"2022-02-08T16:58:16.79839Z","shell.execute_reply.started":"2022-02-08T16:58:16.752715Z","shell.execute_reply":"2022-02-08T16:58:16.792296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = []\nfor Best_Model, CONF in zip(Best_Models, CONFS):\n    models.append(load_model(Best_Model, conf=CONF, iou=IOU))","metadata":{"execution":{"iopub.status.busy":"2022-02-08T16:58:16.801992Z","iopub.execute_input":"2022-02-08T16:58:16.802536Z","iopub.status.idle":"2022-02-08T16:58:31.243787Z","shell.execute_reply.started":"2022-02-08T16:58:16.802494Z","shell.execute_reply":"2022-02-08T16:58:31.242822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, (img, pred_df) in enumerate(tqdm(iter_test)):\n    bboxes_all = []\n    confs_all = []\n    for model, IMG_SIZE, TTA in zip(models, IMG_SIZES, TTAs):\n        bboxes, confs  = predict(model, img, size=IMG_SIZE, augment=TTA)\n        \n        if len(bboxes) > 0:\n            bboxes_all.append(coco2voc(bboxes).astype(int))\n            confs_all.append(confs)\n        \n    if len(bboxes_all) > 1:\n        bboxes_wbf, confs_wbf, _ = run_wbf(bboxes_all,\n                                           confs_all,\n                                           image_size=1280,\n                                           iou_thr=WBF_IOU,\n                                           method=ENSEMBLE_METHOD)\n        if FINAL_CONF != 0:\n            bboxes_wbf, confs_wbf = filter_bboxes(bboxes_wbf, confs_wbf, FINAL_CONF)\n\n    else:\n        bboxes_wbf = []\n        confs_wbf = []\n    \n    if BOX_EXPANSION != 0:\n        bboxes_wbf = expand_bboxes(bboxes_wbf, BOX_EXPANSION)\n        \n    annot = format_prediction(bboxes_wbf, confs_wbf)\n    pred_df['annotations'] = annot\n    env.predict(pred_df)\n    \n    if idx<3:\n        display(show_img(img, bboxes_wbf, bbox_format='coco'))","metadata":{"execution":{"iopub.status.busy":"2022-02-08T16:58:31.24749Z","iopub.execute_input":"2022-02-08T16:58:31.247745Z","iopub.status.idle":"2022-02-08T16:58:43.96055Z","shell.execute_reply.started":"2022-02-08T16:58:31.247697Z","shell.execute_reply":"2022-02-08T16:58:43.959719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv('submission.csv')\nsub_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-08T16:58:43.961832Z","iopub.execute_input":"2022-02-08T16:58:43.962595Z","iopub.status.idle":"2022-02-08T16:58:43.97921Z","shell.execute_reply.started":"2022-02-08T16:58:43.962555Z","shell.execute_reply":"2022-02-08T16:58:43.978375Z"},"trusted":true},"execution_count":null,"outputs":[]}]}