{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 📚 Import Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\nimport glob\nimport shutil\nimport sys\nsys.path.append('../input/tensorflow-great-barrier-reef')\nimport torch\nfrom PIL import Image\nimport ast","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-07T18:23:24.709178Z","iopub.execute_input":"2022-02-07T18:23:24.709577Z","iopub.status.idle":"2022-02-07T18:23:26.494575Z","shell.execute_reply.started":"2022-02-07T18:23:24.709486Z","shell.execute_reply":"2022-02-07T18:23:26.493896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Please Upvote if you find this Helpful","metadata":{}},{"cell_type":"code","source":"ROOT_DIR  = '/kaggle/input/tensorflow-great-barrier-reef/'\nMODEL_DIR = '/kaggle/input/barrier-models'\n\nCKPT_PATHS = [f'{MODEL_DIR}/yolov5l6_epoch19_video1_r_708_p_868_m_38_s_642_size_3100_all_data.pt',\n              f'../input/asnorkin-gbreef-yolov5/yolov5l6_2s1_gbr_video_v0_fold2_obj8.0_yanc_strong_noise_enhance_fixedv6_1280_4b10e.pt',\n            ]\n\nAUGMENT    = [True, False]\nIMG_SIZES  = [3100, 1920]\nCONFS      = [0.15, 0.2]\nIOUS       = [0.3, 0.3]\nWEIGHTS    = [0.55, 0.45]\nPRED_CONF  = 0.15\n\nTRACKING = True","metadata":{"execution":{"iopub.status.busy":"2022-02-07T18:23:28.133148Z","iopub.execute_input":"2022-02-07T18:23:28.133735Z","iopub.status.idle":"2022-02-07T18:23:28.138981Z","shell.execute_reply.started":"2022-02-07T18:23:28.133698Z","shell.execute_reply":"2022-02-07T18:23:28.138078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_path(row):\n    row['image_path'] = f'{ROOT_DIR}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    return row","metadata":{"execution":{"iopub.status.busy":"2022-02-07T18:23:29.218177Z","iopub.execute_input":"2022-02-07T18:23:29.218427Z","iopub.status.idle":"2022-02-07T18:23:29.222479Z","shell.execute_reply.started":"2022-02-07T18:23:29.218398Z","shell.execute_reply":"2022-02-07T18:23:29.221568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def voc2yolo(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    voc  => [x1, y1, x2, y1]\n    yolo => [xmid, ymid, w, h] (normalized)\n    \"\"\"\n    \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]/ image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]/ image_height\n    \n    w = bboxes[..., 2] - bboxes[..., 0]\n    h = bboxes[..., 3] - bboxes[..., 1]\n    \n    bboxes[..., 0] = bboxes[..., 0] + w/2\n    bboxes[..., 1] = bboxes[..., 1] + h/2\n    bboxes[..., 2] = w\n    bboxes[..., 3] = h\n    \n    return bboxes\n\ndef yolo2voc(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    voc  => [x1, y1, x2, y1]\n    \n    \"\"\" \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height\n    \n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]\n    \n    return bboxes\n\ndef coco2yolo(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    coco => [xmin, ymin, w, h]\n    yolo => [xmid, ymid, w, h] (normalized)\n    \"\"\"\n    \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    # normolizinig\n    bboxes[..., [0, 2]]= bboxes[..., [0, 2]]/ image_width\n    bboxes[..., [1, 3]]= bboxes[..., [1, 3]]/ image_height\n    \n    # converstion (xmin, ymin) => (xmid, ymid)\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]/2\n    \n    return bboxes\n\ndef yolo2coco(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    coco => [xmin, ymin, w, h]\n    \n    \"\"\" \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    # denormalizing\n    bboxes[..., [0, 2]]= bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]]= bboxes[..., [1, 3]]* image_height\n    \n    # converstion (xmid, ymid) => (xmin, ymin) \n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    \n    return bboxes\n\ndef voc2coco(bboxes, image_height=720, image_width=1280):\n    bboxes  = voc2yolo(bboxes, image_height, image_width)\n    bboxes  = yolo2coco(bboxes, image_height, image_width)\n    return bboxes\n\n\ndef load_image(image_path):\n    return cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n\n\ndef plot_one_box(x, img, color=None, label=None, line_thickness=None):\n    # Plots one bounding box on image img\n    tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1  # line/font thickness\n    color = color or [random.randint(0, 255) for _ in range(3)]\n    c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\n    cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)\n    if label:\n        tf = max(tl - 1, 1)  # font thickness\n        t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\n        c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\n        cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA)  # filled\n        cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)\n\ndef draw_bboxes(img, bboxes, classes, class_ids, colors = None, show_classes = None, bbox_format = 'yolo', class_name = False, line_thickness = 2):  \n     \n    image = img.copy()\n    show_classes = classes if show_classes is None else show_classes\n    colors = (0, 255 ,0) if colors is None else colors\n    \n    if bbox_format == 'yolo':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes:\n            \n                x1 = round(float(bbox[0])*image.shape[1])\n                y1 = round(float(bbox[1])*image.shape[0])\n                w  = round(float(bbox[2])*image.shape[1]/2) #w/2 \n                h  = round(float(bbox[3])*image.shape[0]/2)\n\n                voc_bbox = (x1-w, y1-h, x1+w, y1+h)\n                plot_one_box(voc_bbox, \n                             image,\n                             color = color,\n                             label = cls if class_name else str(get_label(cls)),\n                             line_thickness = line_thickness)\n            \n    elif bbox_format == 'coco':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes:            \n                x1 = int(round(bbox[0]))\n                y1 = int(round(bbox[1]))\n                w  = int(round(bbox[2]))\n                h  = int(round(bbox[3]))\n\n                voc_bbox = (x1, y1, x1+w, y1+h)\n                plot_one_box(voc_bbox, \n                             image,\n                             color = color,\n                             label = cls if class_name else str(cls_id),\n                             line_thickness = line_thickness)\n\n    elif bbox_format == 'voc_pascal':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes: \n                x1 = int(round(bbox[0]))\n                y1 = int(round(bbox[1]))\n                x2 = int(round(bbox[2]))\n                y2 = int(round(bbox[3]))\n                voc_bbox = (x1, y1, x2, y2)\n                plot_one_box(voc_bbox, \n                             image,\n                             color = color,\n                             label = cls if class_name else str(cls_id),\n                             line_thickness = line_thickness)\n    else:\n        raise ValueError('wrong bbox format')\n\n    return image\n\ndef get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndef get_imgsize(row):\n    row['width'], row['height'] = imagesize.get(row['image_path'])\n    return row\n\nnp.random.seed(32)\ncolors = [(np.random.randint(255), np.random.randint(255), np.random.randint(255))\\\n          for idx in range(1)]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-07T18:23:29.544298Z","iopub.execute_input":"2022-02-07T18:23:29.544555Z","iopub.status.idle":"2022-02-07T18:23:29.581877Z","shell.execute_reply.started":"2022-02-07T18:23:29.544522Z","shell.execute_reply":"2022-02-07T18:23:29.581145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_uint8(img):\n    return np.clip(img, 0, 255).astype(np.uint8)\n\n\ndef channel_stretching(img):\n    I_min = np.min(img)\n    I_max = np.max(img)\n    I_mean = np.mean(img)\n    return (img - I_min) * (1 / max(1, (I_max - I_min)))\n\n\ndef enchance(img):\n    # TO HSV\n    hsv_img = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)\n\n    # Histogram equalisation on the V-channel\n    hsv_img[:, :, 2] = cv2.equalizeHist(hsv_img[:, :, 2])\n\n    # CLAHE\n    h, s, v = hsv_img[:, :, 0], hsv_img[:, :, 1], hsv_img[:, :, 2]\n    clahe = cv2.createCLAHE(clipLimit=15.0, tileGridSize=(20, 20))\n    v = clahe.apply(v)\n\n    # HSVStretching\n    s = channel_stretching(s)\n    v = channel_stretching(v)\n\n    # TO RGB\n    hsv_img = np.dstack((h, s, v))\n    hsv_img = to_uint8(hsv_img)\n    out_img = cv2.cvtColor(hsv_img, cv2.COLOR_HSV2RGB)\n    out_img = to_uint8(out_img)\n\n    # Gamma correction\n    R = 255.0\n    out_img = (R * np.power(img.astype(np.uint32) / R, 1.5))\n\n    return to_uint8(out_img)","metadata":{"execution":{"iopub.status.busy":"2022-02-07T18:23:29.785401Z","iopub.execute_input":"2022-02-07T18:23:29.785888Z","iopub.status.idle":"2022-02-07T18:23:29.797074Z","shell.execute_reply.started":"2022-02-07T18:23:29.785854Z","shell.execute_reply":"2022-02-07T18:23:29.796306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-07T18:23:29.985228Z","iopub.execute_input":"2022-02-07T18:23:29.985482Z","iopub.status.idle":"2022-02-07T18:23:31.320549Z","shell.execute_reply.started":"2022-02-07T18:23:29.985453Z","shell.execute_reply":"2022-02-07T18:23:31.31956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(ckpt_path, conf=0.25, iou=0.50):\n    model = torch.hub.load('/kaggle/input/yolov5-lib-ds',\n                           'custom',\n                           path=ckpt_path,\n                           source='local',\n                           force_reload=True)  # local repo\n    model.conf = conf  # NMS confidence threshold\n    model.iou  = iou  # NMS IoU threshold\n    model.classes = None   # (optional list) filter by class, i.e. = [0, 15, 16] for persons, cats and dogs\n    model.multi_label = False  # NMS multiple labels per box\n    model.max_det = 1000  # maximum number of detections per image\n    return model","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-07T18:23:31.323504Z","iopub.execute_input":"2022-02-07T18:23:31.323992Z","iopub.status.idle":"2022-02-07T18:23:31.330335Z","shell.execute_reply.started":"2022-02-07T18:23:31.32395Z","shell.execute_reply":"2022-02-07T18:23:31.329686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔭 Inference","metadata":{}},{"cell_type":"markdown","source":"## Helper","metadata":{}},{"cell_type":"code","source":"def predict(model, img, size=768, augment=False, idx=0):\n    height, width = img.shape[:2]\n    results = model(img, size=size)  # custom inference size\n    preds   = results.pandas().xyxy[0]\n    bboxes  = preds[['xmin','ymin','xmax','ymax']].values\n    if len(bboxes):\n        bboxes  = voc2coco(bboxes,height,width).astype(int)\n        confs   = preds.confidence.values\n        return bboxes, confs\n    else:\n        return [],[]\n    \ndef format_prediction(bboxes, confs):\n    annot = ''\n    if len(bboxes)>0:\n        for idx in range(len(bboxes)):\n            xmin, ymin, w, h = bboxes[idx]\n            conf             = confs[idx]\n            annot += f'{conf} {xmin} {ymin} {w} {h}'\n            annot +=' '\n        annot = annot.strip(' ')\n    return annot\n\ndef show_img(img, bboxes, bbox_format='yolo'):\n    names  = ['starfish']*len(bboxes)\n    labels = [0]*len(bboxes)\n    img    = draw_bboxes(img = img,\n                           bboxes = bboxes, \n                           classes = names,\n                           class_ids = labels,\n                           class_name = True, \n                           colors = colors, \n                           bbox_format = bbox_format,\n                           line_thickness = 2)\n    return Image.fromarray(img).resize((800, 400))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-07T18:23:31.332655Z","iopub.execute_input":"2022-02-07T18:23:31.332948Z","iopub.status.idle":"2022-02-07T18:23:31.346916Z","shell.execute_reply.started":"2022-02-07T18:23:31.332922Z","shell.execute_reply":"2022-02-07T18:23:31.346102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## WBF","metadata":{}},{"cell_type":"code","source":"import sys; sys.path.append('/kaggle/input/weightedboxesfusion/')","metadata":{"execution":{"iopub.status.busy":"2022-02-07T18:23:31.853168Z","iopub.execute_input":"2022-02-07T18:23:31.853798Z","iopub.status.idle":"2022-02-07T18:23:31.859071Z","shell.execute_reply.started":"2022-02-07T18:23:31.85376Z","shell.execute_reply":"2022-02-07T18:23:31.858327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = '../input/tensorflow-great-barrier-reef'","metadata":{"execution":{"iopub.status.busy":"2022-02-07T18:23:32.137975Z","iopub.execute_input":"2022-02-07T18:23:32.138962Z","iopub.status.idle":"2022-02-07T18:23:32.142761Z","shell.execute_reply.started":"2022-02-07T18:23:32.138917Z","shell.execute_reply":"2022-02-07T18:23:32.142009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_boxes(bboxes_1):\n    \n    bboxes_1_coco = bboxes_1.copy() \n    if len(bboxes_1) > 0:\n        bboxes_1_coco[:,3] = bboxes_1[:,3]+bboxes_1[:,1]\n        bboxes_1_coco[:,2] = bboxes_1[:,2]+bboxes_1[:,0]\n    return bboxes_1_coco","metadata":{"execution":{"iopub.status.busy":"2022-02-07T18:23:32.45714Z","iopub.execute_input":"2022-02-07T18:23:32.45741Z","iopub.status.idle":"2022-02-07T18:23:32.462403Z","shell.execute_reply.started":"2022-02-07T18:23:32.45738Z","shell.execute_reply":"2022-02-07T18:23:32.461689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_path(row):\n    row['image_path'] = f'{ROOT_DIR}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    return row\n\n# Train Data\ndf = pd.read_csv(f'{ROOT_DIR}/train.csv')\ndf = df.progress_apply(get_path, axis=1)\n# Train Data\ndf['old_image_path'] = f'{ROOT_DIR}/train_images/video_'+df.video_id.astype(str)+'/'+df.video_frame.astype(str)+'.jpg'\ndf['annotations'] = df['annotations'].progress_apply(eval)\ndisplay(df.head(2))\ndf['num_bbox'] = df['annotations'].progress_apply(lambda x: len(x))\ndata = (df.num_bbox>0).value_counts()/len(df)*100","metadata":{"execution":{"iopub.status.busy":"2022-02-07T18:23:33.075568Z","iopub.execute_input":"2022-02-07T18:23:33.076307Z","iopub.status.idle":"2022-02-07T18:23:48.770368Z","shell.execute_reply.started":"2022-02-07T18:23:33.076271Z","shell.execute_reply":"2022-02-07T18:23:48.769598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ensemble_boxes import *\n\ndef run_wbf(bboxes, confs, image_size=512, iou_thr=0.55, skip_box_thr=0.7, weights=None):\n    boxes =  [bbox/(image_size-1) for bbox in bboxes]\n    scores = [conf for conf in confs]    \n    labels = [np.ones(conf.shape[0]) for conf in confs]\n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    boxes = boxes*(image_size-1)\n    return boxes, scores, labels","metadata":{"execution":{"iopub.status.busy":"2022-02-07T18:23:48.772225Z","iopub.execute_input":"2022-02-07T18:23:48.772715Z","iopub.status.idle":"2022-02-07T18:23:48.810236Z","shell.execute_reply.started":"2022-02-07T18:23:48.772677Z","shell.execute_reply":"2022-02-07T18:23:48.80951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nmodel_1 = load_model(CKPT_PATHS[0], conf=CONFS[0], iou=IOUS[0])\nmodel_2 = load_model(CKPT_PATHS[1], conf=CONFS[1], iou=IOUS[1])\n\nimage_paths = df[df.num_bbox>1].sample(100).image_path.tolist()\nfor idx, path in enumerate(image_paths):\n    image_np = cv2.imread(path)[...,::-1]\n    \n    bboxes_1, confis_1 = predict(model_1, image_np, size=IMG_SIZES[0], augment=AUGMENT)   \n    bboxes_2, confis_2 = predict(model_2, image_np, size=IMG_SIZES[1], augment=AUGMENT)\n    boxes = [prepare_boxes(bb) for bb in [bboxes_1, bboxes_2]]\n    confs = [c for i,c in enumerate([confis_1, confis_2]) if len(boxes[i])>0]\n    boxes = [b for b in boxes if len(b)>0]\n    \n    bboxes = []\n    scores = []\n    \n    if len(boxes)>0:\n        bboxes, scores, labels = run_wbf(boxes, confs, \n                                             image_size = 1280, iou_thr=0.6, skip_box_thr=0.05, weights=WEIGHTS)\n        if len(bboxes)>0:\n            bboxes[:,3] = bboxes[:,3] - bboxes[:,1]\n            bboxes[:,2] = bboxes[:,2] - bboxes[:,0]\n\n            bboxes = bboxes.round().astype(int).tolist()\n    \n    annot          = format_prediction(bboxes, scores)\n    print('\\n\\nEnsemble (WBF) Predictions: ')\n    display(show_img(image_np, bboxes, bbox_format='coco'))\n    \n    if idx>5:\n        break","metadata":{"execution":{"iopub.status.busy":"2022-02-07T18:53:51.848656Z","iopub.execute_input":"2022-02-07T18:53:51.849015Z","iopub.status.idle":"2022-02-07T18:54:03.207668Z","shell.execute_reply.started":"2022-02-07T18:53:51.848977Z","shell.execute_reply":"2022-02-07T18:54:03.206749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tracking","metadata":{}},{"cell_type":"code","source":"if TRACKING:\n    # norfair dependencies\n    %cd /kaggle/input/norfair031py3/\n    !pip install commonmark-0.9.1-py2.py3-none-any.whl -f ./ --no-index\n    !pip install rich-9.13.0-py3-none-any.whl\n\n    !mkdir /kaggle/working/tmp\n    !cp -r /kaggle/input/norfair031py3/filterpy-1.4.5/filterpy-1.4.5/ /kaggle/working/tmp/\n    %cd /kaggle/working/tmp/filterpy-1.4.5/\n    !pip install .\n    !rm -rf /kaggle/working/tmp\n\n    # norfair\n    %cd /kaggle/input/norfair031py3/\n    !pip install norfair-0.3.1-py3-none-any.whl -f ./ --no-index\n    %cd /kaggle/working\n    \n    import cv2\n    import numpy as np\n    from norfair import Detection, Tracker\n    from norfair.tracker import TrackedObject\n\n    # Helper to convert bbox in format [x_min, y_min, x_max, y_max, score] to norfair.Detection class\n    def to_norfair(detects, frame_id):\n        result = []\n        for x_min, y_min, x_max, y_max, score in detects:\n            xc, yc = (x_min + x_max) / 2, (y_min + y_max) / 2\n            w, h = x_max - x_min, y_max - y_min\n            result.append(\n                Detection(\n                    points=np.array([xc, yc]), \n                    scores=np.array([score]), \n                    data=np.array([w, h, frame_id])\n                )\n            )\n\n        return result\n\n    # Euclidean distance function to match detections on this frame with tracked_objects from previous frames\n    def euclidean_distance(detection, tracked_object):\n        return np.linalg.norm(detection.points - tracked_object.estimate)\n\n\n    def calculate_homography(image, next_image, min_matches=21, topk=100, alpha=0.7, algo=\"orb\", debug=False):\n        if algo == \"sift\":\n            descriptor = cv2.SIFT_create()\n        elif algo == \"orb\":\n            descriptor = cv2.ORB_create()\n        else:\n            raise ValueError(f\"Unexpected algorithm: {algo}\")\n\n        # find the keypoints and descriptors\n        kp1, des1 = descriptor.detectAndCompute(image, None)\n        kp2, des2 = descriptor.detectAndCompute(next_image, None)\n        des1 = np.float32(des1)\n        des2 = np.float32(des2)\n\n        FLANN_INDEX_KDTREE = 1\n        index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)\n        search_params = dict(checks=50)\n        flann = cv2.FlannBasedMatcher(index_params, search_params)\n        matches = flann.knnMatch(des1, des2, k=2)\n\n        scores = [m.distance / n.distance for m, n in matches]\n        indices = np.argsort(scores)\n        good = [matches[i][0] for i in indices[:topk] if scores[i] < alpha]\n        if len(good) >= min_matches:\n            src_pts = np.float32([ kp1[m.queryIdx].pt for m in good ]).reshape(-1,1,2)\n            dst_pts = np.float32([ kp2[m.trainIdx].pt for m in good ]).reshape(-1,1,2)\n            M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)\n            matchesMask = mask.ravel().tolist()\n        else:\n            print(f\"Not enough matches: {len(good)} for {algo}\")\n            M, matchesMask = None, None\n\n            # Try to improve using SIFT\n            if algo == \"orb\":\n                M, kp1, kp2, good, matchesMask = calculate_homography(\n                    image, next_image, min_matches=11, topk=topk, alpha=alpha, algo=\"sift\", debug=True)\n\n        if debug:\n            return M, kp1, kp2, good, matchesMask\n\n        return M\n\n\n    def transform(points, H):\n        points = np.asarray(points, dtype=float).reshape(1, -1, 2)\n        return cv2.perspectiveTransform(points, H)[0]\n\n\n    def split(bboxes, confs, confthresh):\n        weak_detects = [[x, y, x + w, y + h, conf] for (x, y, w, h), conf in zip(bboxes, confs)]\n        _bboxes, _confs = [], []\n        for bb, conf in zip(bboxes, confs):\n            if conf >= confthresh:\n                _bboxes.append(bb)\n                _confs.append(conf)\n        bboxes, confs = np.asarray(_bboxes), np.asarray(_confs)\n        return bboxes, confs, weak_detects\n\n\n    class HomographyEstimator:\n        def __init__(self, min_matches=21, topk=100, alpha=0.7):\n            self.min_matches = min_matches\n            self.topk = topk\n            self.alpha = alpha\n\n            self.prev_frame = None\n\n        def estimate(self, frame):\n            H = None\n            if self.prev_frame is not None:\n                try:\n                    H = calculate_homography(\n                        self.prev_frame, \n                        frame, \n                        min_matches=self.min_matches, \n                        topk=self.topk, \n                        alpha=self.alpha\n                    )\n                except Exception as e:\n                    print(f\"Failed to calculate homography: {e}\")\n\n            self.prev_frame = frame\n\n            return H\n\n\n    class MatchingCOTSTracker(Tracker):\n        def weak_update(self, detections=None, period=1, confthresh=0.0):\n            self.period = period\n\n            # Remove stale trackers and make candidate object real if it has hit inertia\n            self.tracked_objects = [o for o in self.tracked_objects if o.has_inertia]\n\n            # Update tracker\n            for obj in self.tracked_objects:\n                obj.tracker_step()\n\n            # Update initialized tracked objects with detections\n            unmatched_detections = self.update_objects_in_place(\n                [o for o in self.tracked_objects if not o.is_initializing], detections\n            )\n\n            # Filter out detections with confidence < confthresh\n            unmatched_detections = [\n                d for d in unmatched_detections if d.scores[0] >= confthresh\n            ]\n\n            # Update not yet initialized tracked objects with yet unmatched detections\n            unmatched_detections = self.update_objects_in_place(\n                [o for o in self.tracked_objects if o.is_initializing], unmatched_detections\n            )\n\n            # Create new tracked objects from remaining unmatched detections\n            for detection in unmatched_detections:\n                self.tracked_objects.append(\n                    TrackedObject(\n                        detection,\n                        self.hit_inertia_min,\n                        self.hit_inertia_max,\n                        self.initialization_delay,\n                        self.detection_threshold,\n                        self.period,\n                        self.point_transience,\n                        self.filter_setup,\n                    )\n                )\n\n            return [p for p in self.tracked_objects if not p.is_initializing]\n\n        def update(self, detections, frame_id, H=None, period=1, frame_width=1280, frame_height=720, confthresh=0.0):\n            if H is not None:\n                self.shift_tracks(H)\n\n            detections = to_norfair(detections, frame_id)\n    #         tracked_objects = super().update(detections, period)\n            tracked_objects = self.weak_update(detections, period, confthresh)\n\n            def _inside(tobj):\n                w, h, _ = tobj.last_detection.data\n                xc, yc = tobj.estimate[0]\n\n                alpha = 0.5  # Part should be inside\n                dw = alpha / 2 * w\n                dh = alpha / 2 * h\n                x_inside = 0 < xc - dw and xc + dw < frame_width\n                y_inside = 0 < yc - dh and yc + dh < frame_height\n                return x_inside and y_inside\n\n            result = []\n            for tobj in tracked_objects:\n                w, h, last_detected_frame_id = tobj.last_detection.data\n                conf = tobj.last_detection.scores[0]\n                frames_diff = frame_id - last_detected_frame_id\n                old_case = (frames_diff in {1, 2} and _inside(tobj))\n                new_case = (conf < confthresh) and (frames_diff == 0)\n                if old_case or new_case:\n                    xc, yc = tobj.estimate[0]\n                    x_min, y_min = int(round(xc - w / 2)), int(round(yc - h / 2))\n                    bbox = [x_min, y_min, int(w), int(h), conf]\n                    result.append(bbox)\n\n            return np.asarray(result)\n\n        def shift_tracks(self, H):\n            for i, tobj in enumerate(self.tracked_objects):\n                new_xc, new_yc = transform(tobj.estimate, H)[0]\n                self.tracked_objects[i].filter.x[0] = new_xc\n                self.tracked_objects[i].filter.x[1] = new_yc\nelse:\n    print(\"No Tracking\")","metadata":{"execution":{"iopub.status.busy":"2022-02-07T18:24:22.556485Z","iopub.execute_input":"2022-02-07T18:24:22.557153Z","iopub.status.idle":"2022-02-07T18:25:37.920304Z","shell.execute_reply.started":"2022-02-07T18:24:22.557106Z","shell.execute_reply":"2022-02-07T18:25:37.919477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Init `Env`","metadata":{}},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()# initialize the environment\niter_test = env.iter_test()      # an iterator which loops over the test set and sample submission","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-07T18:25:37.922234Z","iopub.execute_input":"2022-02-07T18:25:37.922777Z","iopub.status.idle":"2022-02-07T18:25:37.95231Z","shell.execute_reply.started":"2022-02-07T18:25:37.922735Z","shell.execute_reply":"2022-02-07T18:25:37.951426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Run Inference on **Test**","metadata":{}},{"cell_type":"code","source":"submission_dict = {\n    'id': [],\n    'prediction_string': [],\n}\nmodel_1 = load_model(CKPT_PATHS[0], conf=CONFS[0], iou=IOUS[0])\nmodel_2 = load_model(CKPT_PATHS[1], conf=CONFS[1], iou=IOUS[1])\n\nif TRACKING:\n    tracker_params = {\n        \"distance_function\": euclidean_distance,\n        \"distance_threshold\": 20,\n        \"hit_inertia_min\": 3,\n        \"hit_inertia_max\": 6,\n        \"initialization_delay\": 1,\n    }\n\n    homography = HomographyEstimator()\n    tracker = MatchingCOTSTracker(**tracker_params)\n\nfor idx, (image_np, sample_prediction_df) in enumerate(iter_test):\n    bboxes_1, confis_1 = predict(model_1, image_np, size=IMG_SIZES[0], augment=AUGMENT[0])   \n    bboxes_2, confis_2 = predict(model_2, enchance(image_np), size=IMG_SIZES[1], augment=AUGMENT[1])\n    boxes = [prepare_boxes(bb) for bb in [bboxes_1, bboxes_2]]\n    confs = [c for i,c in enumerate([confis_1, confis_2]) if len(boxes[i])>0]\n    boxes = [b for b in boxes if len(b)>0]\n    \n    bboxes = []\n    scores = []\n    \n    if len(boxes)>0:\n        bboxes, scores, labels = run_wbf(boxes, confs, \n                                             image_size = 1280, iou_thr=0.6, skip_box_thr=0.05, weights=WEIGHTS)\n        if len(bboxes)>0:\n            bboxes[:,3] = bboxes[:,3] - bboxes[:,1]\n            bboxes[:,2] = bboxes[:,2] - bboxes[:,0]\n\n#             bboxes = bboxes.round().astype(int).tolist()   \n            bboxes = bboxes.astype(int).tolist()\n            scores = scores.tolist()\n        else:\n            bboxes, scores = [], []\n            \n    if TRACKING:\n        try:\n            H = homography.estimate(image_np)\n            if H is None:  # Reset tracker if homography is broken or zero frame\n                tracker = MatchingCOTSTracker(**tracker_params)\n\n            imh, imw = image_np.shape[:2]\n            detects = [[x, y, x + w, y + h, score] for (x, y, w, h), score in zip(bboxes, scores)]\n            tracked = tracker.update(detects, idx, H, frame_width=imw, frame_height=imh, confthresh=0.0)  # Init all detections\n            for x, y, w, h, score in tracked:\n                bboxes.append([x, y, w, h])\n                scores.append(score)\n        except Exception as e:\n            print(f\"[WARNING] TRACKING FAILED: {e}\")\n\n    predictions = []\n    for i in range(len(bboxes)):\n        box = bboxes[i]        \n        score = scores[i]\n        if score < PRED_CONF:\n            continue\n\n        x_min = int(box[0])\n        y_min = int(box[1])\n        bbox_width = int(box[2])\n        bbox_height = int(box[3])\n\n        predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n\n    prediction_str = ' '.join(predictions)\n    \n    sample_prediction_df['annotations'] = prediction_str\n    env.predict(sample_prediction_df)\n    if idx<3:\n        display(show_img(image_np, bboxes, bbox_format='coco'))\n        print('Prediction:', prediction_str)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-07T18:25:37.954044Z","iopub.execute_input":"2022-02-07T18:25:37.954558Z","iopub.status.idle":"2022-02-07T18:25:45.880659Z","shell.execute_reply.started":"2022-02-07T18:25:37.954518Z","shell.execute_reply":"2022-02-07T18:25:45.879724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 👀 Check Submission","metadata":{}},{"cell_type":"code","source":"sub_df = pd.read_csv('submission.csv')\nsub_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-07T18:27:06.350822Z","iopub.execute_input":"2022-02-07T18:27:06.351178Z","iopub.status.idle":"2022-02-07T18:27:06.37236Z","shell.execute_reply.started":"2022-02-07T18:27:06.351141Z","shell.execute_reply":"2022-02-07T18:27:06.371619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Debug","metadata":{}},{"cell_type":"code","source":"# bboxes = [[0, 0, 20, 20], [50, 50, 70, 70]]\n# scores = [0.5, 0.1]\n# idx = 0\n\n# detects = [[x, y, x + w, y + h, score] for (x, y, w, h), score in zip(bboxes, scores)]\n# tracked = tracker.update(detects, idx, None, frame_width=imw, frame_height=imh, confthresh=0.0)\n# for x, y, w, h, score in tracked:\n#     bboxes.append([x, y, w, h])\n#     scores.append(score)\n    \n# tracked","metadata":{"execution":{"iopub.status.busy":"2022-02-07T18:59:09.114992Z","iopub.execute_input":"2022-02-07T18:59:09.115269Z","iopub.status.idle":"2022-02-07T18:59:09.126224Z","shell.execute_reply.started":"2022-02-07T18:59:09.115239Z","shell.execute_reply":"2022-02-07T18:59:09.125359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bboxes = [[0, 0, 20, 20], [50, 50, 70, 70]]\n# scores = [0.5, 0.35]\n# idx = 1\n\n# detects = [[x, y, x + w, y + h, score] for (x, y, w, h), score in zip(bboxes, scores)]\n# tracked = tracker.update(detects, idx, None, frame_width=imw, frame_height=imh, confthresh=0.0)\n# for x, y, w, h, score in tracked:\n#     bboxes.append([x, y, w, h])\n#     scores.append(score)\n    \n# tracked","metadata":{"execution":{"iopub.status.busy":"2022-02-07T19:05:29.932606Z","iopub.execute_input":"2022-02-07T19:05:29.933283Z","iopub.status.idle":"2022-02-07T19:05:29.949243Z","shell.execute_reply.started":"2022-02-07T19:05:29.933247Z","shell.execute_reply":"2022-02-07T19:05:29.948419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bboxes = [[0, 0, 20, 20], [50, 50, 70, 70]]\n# scores = [0.2, 0.4]\n# idx = 2\n\n# detects = [[x, y, x + w, y + h, score] for (x, y, w, h), score in zip(bboxes, scores)]\n# tracked = tracker.update(detects, idx, None, frame_width=imw, frame_height=imh, confthresh=0.0)\n# for x, y, w, h, score in tracked:\n#     bboxes.append([x, y, w, h])\n#     scores.append(score)\n    \n# tracked","metadata":{"execution":{"iopub.status.busy":"2022-02-07T19:06:02.907875Z","iopub.execute_input":"2022-02-07T19:06:02.908451Z","iopub.status.idle":"2022-02-07T19:06:02.922013Z","shell.execute_reply.started":"2022-02-07T19:06:02.908413Z","shell.execute_reply":"2022-02-07T19:06:02.921015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bboxes = [[0, 0, 20, 20], [50, 50, 70, 70]]\n# scores = [0.1, 0.6]\n# idx = 3\n\n# detects = [[x, y, x + w, y + h, score] for (x, y, w, h), score in zip(bboxes, scores)]\n# tracked = tracker.update(detects, idx, None, frame_width=imw, frame_height=imh, confthresh=0.0)\n# for x, y, w, h, score in tracked:\n#     bboxes.append([x, y, w, h])\n#     scores.append(score)\n    \n# tracked","metadata":{"execution":{"iopub.status.busy":"2022-02-07T19:06:26.905138Z","iopub.execute_input":"2022-02-07T19:06:26.905836Z","iopub.status.idle":"2022-02-07T19:06:26.917515Z","shell.execute_reply.started":"2022-02-07T19:06:26.905797Z","shell.execute_reply":"2022-02-07T19:06:26.916693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bboxes = [[50, 50, 70, 70]]\n# scores = [0.6]\n# idx = 4\n\n# detects = [[x, y, x + w, y + h, score] for (x, y, w, h), score in zip(bboxes, scores)]\n# tracked = tracker.update(detects, idx, None, frame_width=imw, frame_height=imh, confthresh=0.0)\n# for x, y, w, h, score in tracked:\n#     bboxes.append([x, y, w, h])\n#     scores.append(score)\n    \n# tracked","metadata":{"execution":{"iopub.status.busy":"2022-02-07T19:06:55.557108Z","iopub.execute_input":"2022-02-07T19:06:55.557725Z","iopub.status.idle":"2022-02-07T19:06:55.575013Z","shell.execute_reply.started":"2022-02-07T19:06:55.557681Z","shell.execute_reply":"2022-02-07T19:06:55.574253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}