{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Example Code of Seq-NMS\n\nI will share part of my evaluation code.\nAlthough the prediction code is private, you can imagine how to use Seq-NMS[1] in real-time situation.\n\nI used implementation [2]. You also can integrate it with your own project.\n\n* [1] https://arxiv.org/abs/1602.08465\n* [2] https://github.com/tmoopenn/seq-nms","metadata":{}},{"cell_type":"markdown","source":"# Environment Setup","metadata":{}},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\n\nuser_secrets = UserSecretsClient()\n\npersonal_key_for_api = user_secrets.get_secret(\"github-api-token\")\n\n! git clone https://{personal_key_for_api}:x-oauth-basic@github.com/bilzard/great-barrier-reef.git /tmp/great-barrier-reef\n! cd /tmp/great-barrier-reef && git log | head","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-16T02:13:32.164110Z","iopub.execute_input":"2022-02-16T02:13:32.164436Z","iopub.status.idle":"2022-02-16T02:13:35.384482Z","shell.execute_reply.started":"2022-02-16T02:13:32.164355Z","shell.execute_reply":"2022-02-16T02:13:35.383673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\ncat << 'EOF' > /tmp/run.bash\nPIP_DEP_PATH='/kaggle/input/starfish-build-yolov5-runtime-environment/pip_deps'\necho ${PIP_DEP_PATH}\npip install ${PIP_DEP_PATH}/* -f ./ --no-index --no-deps --find-links=\"${PIP_DEP_PATH}\"\n\nEOF\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:13:35.386406Z","iopub.execute_input":"2022-02-16T02:13:35.386856Z","iopub.status.idle":"2022-02-16T02:13:35.412737Z","shell.execute_reply.started":"2022-02-16T02:13:35.386817Z","shell.execute_reply":"2022-02-16T02:13:35.411892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!time /tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-16T02:13:35.414780Z","iopub.execute_input":"2022-02-16T02:13:35.415039Z","iopub.status.idle":"2022-02-16T02:14:00.413209Z","shell.execute_reply.started":"2022-02-16T02:13:35.415004Z","shell.execute_reply":"2022-02-16T02:14:00.412423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp ../input/yolov5-runtime-environment/arial.ttf /root/.config/Ultralytics/Arial.ttf","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:00.415624Z","iopub.execute_input":"2022-02-16T02:14:00.415890Z","iopub.status.idle":"2022-02-16T02:14:01.744735Z","shell.execute_reply.started":"2022-02-16T02:14:00.415854Z","shell.execute_reply":"2022-02-16T02:14:01.743823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# setup seq_nms","metadata":{}},{"cell_type":"code","source":"%%bash\n\ncat << 'EOF' > /tmp/run.bash\n\ncd /tmp/great-barrier-reef/seq_nms\npython setup.py build_ext --inplace\n\nEOF\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:01.748139Z","iopub.execute_input":"2022-02-16T02:14:01.748371Z","iopub.status.idle":"2022-02-16T02:14:01.769914Z","shell.execute_reply.started":"2022-02-16T02:14:01.748343Z","shell.execute_reply":"2022-02-16T02:14:01.768881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-16T02:14:01.771688Z","iopub.execute_input":"2022-02-16T02:14:01.771973Z","iopub.status.idle":"2022-02-16T02:14:05.690037Z","shell.execute_reply.started":"2022-02-16T02:14:01.771937Z","shell.execute_reply":"2022-02-16T02:14:05.689173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare Inference Utilities","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nfrom collections import deque\nfrom importlib import reload\n\nsys.path.append('/tmp/great-barrier-reef')\nsys.path.append('/tmp/great-barrier-reef/seq_nms')\nsys.path.append('/tmp/great-barrier-reef/yolov5')\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom omegaconf import OmegaConf\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\n\nfrom seq_nms import seq_nms\n\n%load_ext autoreload\n%autoreload 2","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:05.694676Z","iopub.execute_input":"2022-02-16T02:14:05.694951Z","iopub.status.idle":"2022-02-16T02:14:07.185302Z","shell.execute_reply.started":"2022-02-16T02:14:05.694913Z","shell.execute_reply":"2022-02-16T02:14:07.184555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR  = '/kaggle/input/tensorflow-great-barrier-reef/'\nCKPT_PATH = '../input/starfishyolov5models/01_split_by_partition_400_10_fold_img_640_l_heavyx2_g0_pw15_iou20_5x3_neg03fold_8_best.pt'\nAUGMENT   = False\nCONF = 0.001\nMAX_DET = 200\nIOU = 0.3\nIMG_SIZE  = 2560\nCONF_THRE = 0.458\nP_KEEP = 1.0\nRANDOM_SEED = 123\n\n# ===========\n# around-edge, low-confidence boxes removal\n# ===========\nEDGE_PIXELS = 60\nEDGE_SCORE_THR = 0.2\n\n# ===========\n# Seq-NMS\n# ===========\nSEQ_NMS_NMS_THR = 0.3\nSEQ_NMS_NUM_FRAMES = 20\nSEQ_NMS_LK_THR = 0.3\nSEQ_NMS_SCORE_METRIC = 'max'\nSEQ_NMS_DECAY = 0.9","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:07.187886Z","iopub.execute_input":"2022-02-16T02:14:07.188140Z","iopub.status.idle":"2022-02-16T02:14:07.223237Z","shell.execute_reply.started":"2022-02-16T02:14:07.188105Z","shell.execute_reply":"2022-02-16T02:14:07.222539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# generate fold split","metadata":{}},{"cell_type":"code","source":"import os\nos.environ['METADATA_PATH'] = '/kaggle/input/tensorflow-great-barrier-reef'\nos.environ['WORK_DIR'] = '/tmp'\nos.environ['IMAGE_DIR'] = '/kaggle/input/tensorflow-great-barrier-reef/train_images'\n!export | grep MEDATADA_PATH\n!export | grep WORK_DIR\n!export | grep IMAGE_DIR","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:07.226288Z","iopub.execute_input":"2022-02-16T02:14:07.226546Z","iopub.status.idle":"2022-02-16T02:14:09.276280Z","shell.execute_reply.started":"2022-02-16T02:14:07.226513Z","shell.execute_reply":"2022-02-16T02:14:09.275521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!echo $METADATA_PATH\n!echo $WORK_DIR\n!echo $IMAGE_DIR","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:09.277937Z","iopub.execute_input":"2022-02-16T02:14:09.278203Z","iopub.status.idle":"2022-02-16T02:14:11.291796Z","shell.execute_reply.started":"2022-02-16T02:14:09.278167Z","shell.execute_reply":"2022-02-16T02:14:11.290900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\ncd /tmp/great-barrier-reef/etl\npython 01_fold_split_by_partition.py 10","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:11.293328Z","iopub.execute_input":"2022-02-16T02:14:11.293625Z","iopub.status.idle":"2022-02-16T02:14:13.719535Z","shell.execute_reply.started":"2022-02-16T02:14:11.293587Z","shell.execute_reply":"2022-02-16T02:14:13.718776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Data\ndf = pd.read_csv(f'/tmp/great-barrier-reef/data/01_split_by_partition_400_10_fold.csv')\ndf['image_path'] = (\n    f'{ROOT_DIR}/train_images/video_' + \n    df['video_id'].astype(str) + \n    '/' + \n    df['video_frame'].astype(str) +\n    '.jpg'\n)\ndf['annotations'] = df['annotations'].apply(lambda x: eval(x))\ndisplay(df.head(2))","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:13.723988Z","iopub.execute_input":"2022-02-16T02:14:13.726052Z","iopub.status.idle":"2022-02-16T02:14:14.320636Z","shell.execute_reply.started":"2022-02-16T02:14:13.726008Z","shell.execute_reply":"2022-02-16T02:14:14.319960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['num_bbox'] = df['annotations'].apply(lambda x: len(x))\ndata = (df.num_bbox>0).value_counts()/len(df)*100\nprint(f\"No BBox: {data[0]:0.2f}% | With BBox: {data[1]:0.2f}%\")","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:14.321934Z","iopub.execute_input":"2022-02-16T02:14:14.322183Z","iopub.status.idle":"2022-02-16T02:14:14.373059Z","shell.execute_reply.started":"2022-02-16T02:14:14.322149Z","shell.execute_reply":"2022-02-16T02:14:14.371893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize_voc(bboxes, image_height=720, image_width=1280):\n    '''\n    voc => [x1, y1, x2, y2]\n    normalized voc => [sx1, sy1, sx2, sy2]\n    '''\n    bboxes[..., 0::2] /= image_width\n    bboxes[..., 1::2] /= image_height\n    return bboxes\n\n    \ndef renormalize_voc(bboxes, image_height=720, image_width=1280):\n    '''\n    normalized voc => [sx1, sy1, sx2, sy2]\n    voc => [x1, y1, x2, y2]\n    '''\n    bboxes[..., 0::2] *= image_width\n    bboxes[..., 1::2] *= image_height\n    bboxes = bboxes.astype(int)\n    return bboxes\n\n\ndef voc2coco(bboxes, image_height=720, image_width=1280):\n    '''\n    voc  => [x1, y1, x2, y2]\n    coco => [xmin, ymin, w, h]\n    '''\n    bboxes[..., 2:] = bboxes[..., 2:] - bboxes[..., :2]\n    return bboxes\n\n\ndef load_image(image_path):\n    return cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n\n\ndef plot_one_box(x, img, color=None, label=None, line_thickness=None):\n    # Plots one bounding box on image img\n    tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1  # line/font thickness\n    color = color or [random.randint(0, 255) for _ in range(3)]\n    c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\n    cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)\n    if label:\n        tf = max(tl - 1, 1)  # font thickness\n        t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\n        c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\n        cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA)  # filled\n        cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, \n                    [63, 63, 63], thickness=tf, lineType=cv2.LINE_AA)\n\n        \ndef draw_bboxes(img, bboxes, classes, class_ids, colors=None, show_classes=None, bbox_format='yolo', class_name=False, line_thickness=2):  \n     \n    image = img.copy()\n    show_classes = classes if show_classes is None else show_classes\n    colors = (0, 255 ,0) if colors is None else colors\n\n    if bbox_format == 'yolo':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes:\n            \n                x1 = round(float(bbox[0])*image.shape[1])\n                y1 = round(float(bbox[1])*image.shape[0])\n                w  = round(float(bbox[2])*image.shape[1]/2) #w/2 \n                h  = round(float(bbox[3])*image.shape[0]/2)\n\n                voc_bbox = (x1-w, y1-h, x1+w, y1+h)\n                plot_one_box(voc_bbox, \n                             image,\n                             color=color,\n                             label=cls if class_name else str(get_label(cls)),\n                             line_thickness=line_thickness)\n            \n    elif bbox_format == 'coco':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes:            \n                x1 = int(round(bbox[0]))\n                y1 = int(round(bbox[1]))\n                w  = int(round(bbox[2]))\n                h  = int(round(bbox[3]))\n\n                voc_bbox = (x1, y1, x1+w, y1+h)\n                plot_one_box(voc_bbox, \n                             image,\n                             color=color,\n                             label=cls if class_name else str(cls_id),\n                             line_thickness=line_thickness)\n\n    elif bbox_format == 'voc_pascal':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes: \n                x1 = int(round(bbox[0]))\n                y1 = int(round(bbox[1]))\n                x2 = int(round(bbox[2]))\n                y2 = int(round(bbox[3]))\n                voc_bbox = (x1, y1, x2, y2)\n                plot_one_box(voc_bbox, \n                             image,\n                             color = color,\n                             label = cls if class_name else str(cls_id),\n                             line_thickness = line_thickness)\n    else:\n        raise ValueError('wrong bbox format')\n\n    return image\n\ndef get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndef get_imgsize(row):\n    row['width'], row['height'] = imagesize.get(row['image_path'])\n    return row\n\nnp.random.seed(32)\ncolors = [\n    (255, 0, 0),  # red\n    (0, 255, 0),  # green\n    (255, 255, 0), # yellow\n    (0, 127, 255), # cyan\n]","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-02-16T02:14:14.374487Z","iopub.execute_input":"2022-02-16T02:14:14.374764Z","iopub.status.idle":"2022-02-16T02:14:14.432031Z","shell.execute_reply.started":"2022-02-16T02:14:14.374726Z","shell.execute_reply":"2022-02-16T02:14:14.431200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(ckpt_path, conf, iou):\n    model = torch.hub.load('/tmp/great-barrier-reef/yolov5',\n                           'custom',\n                           path=ckpt_path,\n                           source='local',\n                           force_reload=True,\n                           tile_helper=None,\n                          )  # local repo\n    model.conf = conf  # NMS confidence threshold\n    model.iou  = iou  # NMS IoU threshold\n    model.classes = None   # (optional list) filter by class, i.e. = [0, 15, 16] for persons, cats and dogs\n    model.multi_label = False  # NMS multiple labels per box\n    model.max_det = MAX_DET  # maximum number of detections per image\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:14.433231Z","iopub.execute_input":"2022-02-16T02:14:14.433478Z","iopub.status.idle":"2022-02-16T02:14:14.467209Z","shell.execute_reply.started":"2022-02-16T02:14:14.433443Z","shell.execute_reply":"2022-02-16T02:14:14.466454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drop_pred(bboxes, scores, p_keep):\n    '''\n    randomly drop prediction for the probability (1 - p_keep).\n    '''\n    if p_keep == 1:\n        return bboxes, scores\n    bboxes = bboxes.copy()\n    scores = scores.copy()\n    assert len(bboxes) == len(scores)\n    pp = np.random.uniform(size=len(bboxes))\n    bboxes = bboxes[pp <= p_keep]\n    scores = scores[pp <= p_keep]\n    return bboxes, scores\n\n\ndef filter_by_threshold(bboxes, scores, conf_thre, verbose=0):\n    bboxes = bboxes.copy()\n    scores = scores.copy()\n    \n    filter_by_score = scores >= conf_thre\n    bboxes, scores = bboxes[filter_by_score], scores[filter_by_score]\n    assert len(bboxes) == len(scores)\n    if verbose >= 2:\n        print('final predictions:')\n        print(bboxes, scores)\n    return bboxes, scores\n    \n\ndef predict(model, img, size=768, augment=False, verbose=0):\n    results = model(img, size=size, augment=augment)  # custom inference size\n    preds = results.pandas().xyxy[0]\n    bboxes = preds[['xmin','ymin','xmax','ymax']].values\n    if len(bboxes) == 0:\n        bboxes, scores = np.zeros((0, 4), dtype=int), np.zeros((0), dtype=int)\n\n    scores = preds.confidence.values\n\n    return bboxes, scores\n    \ndef format_prediction(bboxes, scores):\n    annot = ''\n    if len(bboxes) > 0:\n        for idx in range(len(bboxes)):\n            xmin, ymin, w, h = bboxes[idx]\n            conf = scores[idx]\n            annot += f'{conf:.8f} {xmin} {ymin} {w} {h}'\n            annot +=' '\n        annot = annot.strip(' ')\n    return annot\n\ndef show_img(img, bboxes, scores, gt_boxes, image_id, newly_detected, newly_suppressed, bbox_format='yolo', img_size=(800, 400)):\n    labels = [0] * len(bboxes)\n    img = draw_bboxes(img=img,\n                      bboxes=gt_boxes,\n                      classes=[\"GT\" for _ in range(len(gt_boxes))],\n                      class_ids=[1] * len(gt_boxes),\n                      class_name=True,\n                      colors=colors, \n                      bbox_format=bbox_format,\n                      line_thickness=2)\n    img = draw_bboxes(img=img,\n                      bboxes=bboxes,\n                      classes=[f\"Pred:{score:.2f}\" for score in scores],\n                      class_ids=labels,\n                      class_name=True, \n                      colors=colors, \n                      bbox_format=bbox_format,\n                      line_thickness=2)\n    img = draw_bboxes(img=img,\n                      bboxes=newly_detected[0],\n                      classes=[f\"Det:{score:.2f}\" for score in newly_detected[1]],\n                      class_ids=[2] * len(newly_detected[0]),\n                      class_name=True, \n                      colors=colors, \n                      bbox_format=bbox_format,\n                      line_thickness=2)\n    img = draw_bboxes(img=img,\n                      bboxes=newly_suppressed[0],\n                      classes=[f\"Sup:{score:.2f}\" for score in newly_suppressed[1]],\n                      class_ids=[3] * len(newly_suppressed[0]),\n                      class_name=True, \n                      colors=colors, \n                      bbox_format=bbox_format,\n                      line_thickness=2)\n    cv2.putText(img, image_id, (0, 25), 0, 1, [255, 255, 255], thickness=1, lineType=cv2.LINE_AA)\n    return Image.fromarray(img).resize(img_size)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:14.470044Z","iopub.execute_input":"2022-02-16T02:14:14.470645Z","iopub.status.idle":"2022-02-16T02:14:14.515363Z","shell.execute_reply.started":"2022-02-16T02:14:14.470607Z","shell.execute_reply":"2022-02-16T02:14:14.514547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SeqNmsPostProcessor:\n    def __init__(self, num_frames=5, decay=0.9, verbose=0):\n        self.queue = deque(maxlen=num_frames)\n        self.verbose = verbose\n        self.num_frames = num_frames\n        self.decay = decay\n        if self.verbose:\n            print(f\"SeqNmsPostProcessor(num_frames={num_frames} decay={decay})\")\n        \n    def _pad_sequence(self, frame_bboxes, frame_scores):\n        \"\"\"\n        pad bboxes and frame_scores in order to shape (f, b, 4) and (f, b) respectively\n        \"\"\"\n        max_len = max([len(item) for item in frame_scores])\n        for idx, (bboxes, scores) in enumerate(zip(frame_bboxes, frame_scores)):\n            pad_bboxes = np.zeros((max_len - len(scores), 4))\n            pad_scores = np.zeros((max_len - len(scores)))\n            frame_bboxes[idx] = np.concatenate([bboxes, pad_bboxes])\n            frame_scores[idx] = np.concatenate([scores, pad_scores])\n            \n        return frame_bboxes, frame_scores\n    \n    def _drop_zero(self, bboxes, scores):\n        \"\"\"\n        drop padded zero prediction\n        \"\"\"\n        return bboxes[scores > 0], scores[scores > 0]\n \n    def apply(self, bboxes, scores, linkage_threshold, nms_threshold, score_metric):\n        bboxes, scores = bboxes.copy(), scores.copy()\n        if self.verbose:\n            original_scores = scores.copy()\n        self.queue.append((bboxes, scores))\n        frame_bboxes, frame_scores = zip(*self.queue)\n        frame_bboxes, frame_scores = list(frame_bboxes), list(frame_scores)\n        frame_bboxes, frame_scores = self._pad_sequence(frame_bboxes, frame_scores)\n        frame_bboxes = np.stack(frame_bboxes)\n        frame_scores = np.stack(frame_scores)\n        seq_nms(\n            frame_bboxes,\n            frame_scores,\n            labels=[],\n            linkage_threshold=linkage_threshold,\n            nms_threshold=nms_threshold,\n            score_metric=score_metric,\n        )\n        # decay\n        for i in range(len(self.queue)):\n            bb, ss = self.queue[i]\n            self.queue[i] = (bb, ss * self.decay)\n\n        bboxes, scores = frame_bboxes[-1, :], frame_scores[-1, :]\n        bboxes, scores = self._drop_zero(bboxes, scores)\n        if self.verbose:\n            newly_detected = (original_scores < CONF_THRE) & (scores >= CONF_THRE)\n            newly_suppressed = (original_scores >= CONF_THRE) & (scores < CONF_THRE)\n            if self.verbose >= 2:\n                print(\"Newly Detected Labels:\")\n                print(f\"  - Original Score: {original_scores[newly_detected]}\")\n                print(f\"  - Updated Score: {scores[newly_detected]}\")\n                print(f\"  - Coordinate: {bboxes[newly_detected]}\")\n                print(\"Newly Suppressed Labels:\")\n                print(f\"  - Original Score: {original_scores[newly_suppressed]}\")\n                print(f\"  - Updated Score: {scores[newly_suppressed]}\")\n                print(f\"  - Coordinate: {bboxes[newly_suppressed]}\")\n        \n        if self.verbose:\n            return bboxes, scores, (voc2coco(bboxes[newly_detected]), scores[newly_detected]), (voc2coco(bboxes[newly_suppressed]), scores[newly_suppressed])\n        else:\n            return bboxes, scores, (np.zeros((0, 4)), np.zeros((0))), (np.zeros((0, 4)), np.zeros((0)))","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:14.516683Z","iopub.execute_input":"2022-02-16T02:14:14.517013Z","iopub.status.idle":"2022-02-16T02:14:14.562025Z","shell.execute_reply.started":"2022-02-16T02:14:14.516971Z","shell.execute_reply":"2022-02-16T02:14:14.561195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_boxes_around_edges(bboxes, scores, width, height, pixels=40, score_thr=0.2):\n    \"\"\"\n    remove low confidence boxes around the edges\n    coodinate should be in voc (x1, y1, x2, y2)\n    \"\"\"\n    filter_idx = (\n        (bboxes[:, 2] < pixels) | (bboxes[:, 0] >= width - pixels) | \n        ((bboxes[:, 3] < pixels)) | (bboxes[:, 1] >= height - pixels)\n    ) & (scores < score_thr)\n    \n    return bboxes[~filter_idx], scores[~filter_idx]","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:14.562998Z","iopub.execute_input":"2022-02-16T02:14:14.563205Z","iopub.status.idle":"2022-02-16T02:14:14.597274Z","shell.execute_reply.started":"2022-02-16T02:14:14.563176Z","shell.execute_reply":"2022-02-16T02:14:14.596447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def detect_pipeline(img, model, post_processor, verbose=0):\n    height, width = img.shape[:2]\n    \n    # predict -> voc(x1, y1, x2, y2)\n    bboxes, scores = predict(model, img, size=IMG_SIZE, augment=AUGMENT, verbose=verbose)\n    bboxes = normalize_voc(bboxes, height, width)\n\n    # renormalize box coordinate\n    bboxes = renormalize_voc(bboxes, height, width)\n    \n    # remove low confidence boxes around the edges\n    bboxes, scores = remove_boxes_around_edges(bboxes, scores, width, height, pixels=EDGE_PIXELS, score_thr=EDGE_SCORE_THR)\n    \n    if post_processor.num_frames >= 2:\n        # Seq-NMS: https://arxiv.org/abs/1602.08465\n        bboxes, scores, newly_detected, newly_suppressed = post_processor.apply(\n            bboxes,\n            scores,\n            linkage_threshold=SEQ_NMS_LK_THR,\n            nms_threshold=SEQ_NMS_NMS_THR,\n            score_metric=SEQ_NMS_SCORE_METRIC,\n        )\n    else:\n        newly_detected, newly_suppressed = (np.zeros((0, 4)), np.zeros((0))), (np.zeros((0, 4)), np.zeros((0)))\n    return bboxes, scores, newly_detected, newly_suppressed","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:14.600038Z","iopub.execute_input":"2022-02-16T02:14:14.600251Z","iopub.status.idle":"2022-02-16T02:14:14.634563Z","shell.execute_reply.started":"2022-02-16T02:14:14.600226Z","shell.execute_reply":"2022-02-16T02:14:14.633779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run Inference on Train","metadata":{}},{"cell_type":"code","source":"eval_df = df[(df.num_bbox >= 1) & (df.fold_id == 8)]\nlen(eval_df)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:14:14.639026Z","iopub.execute_input":"2022-02-16T02:14:14.639404Z","iopub.status.idle":"2022-02-16T02:14:14.679942Z","shell.execute_reply.started":"2022-02-16T02:14:14.639369Z","shell.execute_reply":"2022-02-16T02:14:14.679212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_frame = 370\nduration = 20\n\nmodel = load_model(CKPT_PATH, conf=CONF, iou=IOU)\npost_processor = SeqNmsPostProcessor(num_frames=SEQ_NMS_NUM_FRAMES, decay=SEQ_NMS_DECAY, verbose=2)\n\nfor idx, item in enumerate(eval_df.head(start_frame + duration).tail(duration).itertuples()):\n    path = item.image_path\n    image_id = item.image_id\n    gt_boxes = item.annotations\n    gt_boxes = [[b['x'], b['y'], b['width'], b['height']] for b in gt_boxes]\n    gt_boxes = np.array(gt_boxes)\n\n    img = cv2.imread(path)[...,::-1]\n    bboxes, scores, newly_detected, newly_suppressed = detect_pipeline(img, model, post_processor, verbose=2)\n\n    # convert coordinate\n    bboxes = voc2coco(bboxes)\n    \n    # filter by conf threshold\n    bboxes, scores = filter_by_threshold(bboxes, scores, CONF_THRE, verbose=2)\n    \n    # display image with boxes\n    display(show_img(img, bboxes, scores, gt_boxes, image_id, newly_detected, newly_suppressed, bbox_format='coco', img_size=(960, 540)))\n","metadata":{"execution":{"iopub.status.busy":"2022-02-16T02:20:09.164038Z","iopub.execute_input":"2022-02-16T02:20:09.164313Z","iopub.status.idle":"2022-02-16T02:20:21.773716Z","shell.execute_reply.started":"2022-02-16T02:20:09.164283Z","shell.execute_reply":"2022-02-16T02:20:21.771571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}