{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Trained Model\n\n* fold5\n    * CV: 0.7727\n    * Link to W&B: https://wandb.ai/bilzard/starfish_f2/runs/4971zk4i\n* fold6\n    * CV: 0.7107\n    * Link to W&B: https://wandb.ai/bilzard/starfish_f2/runs/m19svzdf\n* fold7\n    * CV: 0.68\n    * Link to W&B: https://wandb.ai/bilzard/starfish_f2/runs/24dkcfjs\n* fold8\n    * CV: 0.569\n    * Link to W&B: https://wandb.ai/bilzard/starfish_f2/runs/110613o7\n* fold9\n    * CV: 0.7105\n    * Link to W&B: https://wandb.ai/bilzard/starfish_negative_sample/runs/37zffvxx","metadata":{}},{"cell_type":"markdown","source":"# Environment Setup","metadata":{}},{"cell_type":"code","source":"! cp -r ../input/starfish-clone-sourse-repository/great-barrier-reef/. /tmp/great-barrier-reef\n! cd /tmp/great-barrier-reef && git log | head","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-15T00:59:56.370437Z","iopub.execute_input":"2022-02-15T00:59:56.370833Z","iopub.status.idle":"2022-02-15T00:59:59.957133Z","shell.execute_reply.started":"2022-02-15T00:59:56.370723Z","shell.execute_reply":"2022-02-15T00:59:59.956312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\ncat << 'EOF' > /tmp/run.bash\nPIP_DEP_PATH='/kaggle/input/starfish-build-yolov5-runtime-environment/pip_deps'\necho ${PIP_DEP_PATH}\npip install ${PIP_DEP_PATH}/* -f ./ --no-index --no-deps --find-links=\"${PIP_DEP_PATH}\"\n\nEOF\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:59:59.95963Z","iopub.execute_input":"2022-02-15T00:59:59.959957Z","iopub.status.idle":"2022-02-15T00:59:59.98835Z","shell.execute_reply.started":"2022-02-15T00:59:59.959914Z","shell.execute_reply":"2022-02-15T00:59:59.987395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!time /tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-15T00:59:59.990386Z","iopub.execute_input":"2022-02-15T00:59:59.990988Z","iopub.status.idle":"2022-02-15T01:00:25.70763Z","shell.execute_reply.started":"2022-02-15T00:59:59.990941Z","shell.execute_reply":"2022-02-15T01:00:25.70686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp ../input/yolov5-runtime-environment/arial.ttf /root/.config/Ultralytics/Arial.ttf","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:00:25.710902Z","iopub.execute_input":"2022-02-15T01:00:25.711116Z","iopub.status.idle":"2022-02-15T01:00:27.040109Z","shell.execute_reply.started":"2022-02-15T01:00:25.711089Z","shell.execute_reply":"2022-02-15T01:00:27.039184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# setup seq_nms","metadata":{}},{"cell_type":"code","source":"%%bash\n\ncat << 'EOF' > /tmp/run.bash\n\ncd /tmp/great-barrier-reef/seq_nms\npython setup.py build_ext --inplace\n\nEOF\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:00:27.042885Z","iopub.execute_input":"2022-02-15T01:00:27.043636Z","iopub.status.idle":"2022-02-15T01:00:27.062256Z","shell.execute_reply.started":"2022-02-15T01:00:27.043594Z","shell.execute_reply":"2022-02-15T01:00:27.061313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!/tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:00:27.065751Z","iopub.execute_input":"2022-02-15T01:00:27.065968Z","iopub.status.idle":"2022-02-15T01:00:31.163108Z","shell.execute_reply.started":"2022-02-15T01:00:27.065942Z","shell.execute_reply":"2022-02-15T01:00:31.162152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare Inference Utilities","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nfrom collections import deque\nfrom importlib import reload\n\nsys.path.append('/tmp/great-barrier-reef')\nsys.path.append('/tmp/great-barrier-reef/seq_nms')\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom ensemble_boxes import weighted_boxes_fusion\nfrom omegaconf import OmegaConf\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\n\nfrom seq_nms import seq_nms\n\n%load_ext autoreload\n%autoreload 2","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:00:31.165003Z","iopub.execute_input":"2022-02-15T01:00:31.165296Z","iopub.status.idle":"2022-02-15T01:00:33.536304Z","shell.execute_reply.started":"2022-02-15T01:00:31.16526Z","shell.execute_reply":"2022-02-15T01:00:33.535526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR  = '/kaggle/input/tensorflow-great-barrier-reef/'\nCKPT_PATHS = [\n    '../input/starfishyolov5models/01_split_by_partition_400_10_fold_img_640_l_heavyx2_g0_pw15_iou20_5x3_neg03fold_5_best.pt',\n    '../input/starfishyolov5models/01_split_by_partition_400_10_fold_img_640_l_heavyx2_g0_pw15_iou20_5x3_neg03fold_6_best.pt',\n    '../input/starfishyolov5models/01_split_by_partition_400_10_fold_img_640_l_heavyx2_g0_pw15_iou20_5x3_neg03fold_7_best.pt',\n    '../input/starfishyolov5models/01_split_by_partition_400_10_fold_img_640_l_heavyx2_g0_pw15_iou20_5x3_neg03fold_8_best.pt',\n    '../input/starfishyolov5models/01_split_by_partition_400_10_fold_img_640_l_heavyx2_g0_pw15_5x3_neg03fold_9_best.pt',\n]\nFOLD_ID = 9\nCONF = 0.001\nMAX_DET = 200\nIOU = 0.30\nCONF_THRE = 0.5\nP_KEEP = 1.0\nRANDOM_SEED = 123\n\n# ===================\n# WBF\n# ===================\nSKIP_BOX_THR = 0.0001\nIOU_THR = IOU\nWEIGHTS = [1, 1, 1, 1, 1]\n\n# ===========\n# TTA\n# ===========\nAUGMENT = False\nIMG_SIZE  = 2560\nTTA_SCALES = (0.7, 1.0, 1.3)\n\n# ===========\n# around-edge, low-confidence boxes removal\n# ===========\nEDGE_PIXELS = 60\nEDGE_SCORE_THR = 0.2\n\n# ===========\n# Seq-NMS\n# ===========\nSEQ_NMS_NMS_THR = 0.3\nSEQ_NMS_NUM_FRAMES = 20\nSEQ_NMS_LK_THR = 0.3\nSEQ_NMS_SCORE_METRIC = 'max'\nSEQ_NMS_DECAY = 0.9","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:07:19.807785Z","iopub.execute_input":"2022-02-15T01:07:19.808064Z","iopub.status.idle":"2022-02-15T01:07:19.860435Z","shell.execute_reply.started":"2022-02-15T01:07:19.808032Z","shell.execute_reply":"2022-02-15T01:07:19.859559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Data\ndf = pd.read_csv('../input/gbr-add-pseudo-label-to-fold-csv/pseudo_labels.csv')\ndf['image_path'] = (\n    f'{ROOT_DIR}/train_images/video_' + \n    df['video_id'].astype(str) + \n    '/' + \n    df['video_frame'].astype(str) +\n    '.jpg'\n)\ndf['annotations'] = df['annotations'].apply(lambda x: eval(x))\ndisplay(df.head(2))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:07:19.96698Z","iopub.execute_input":"2022-02-15T01:07:19.967296Z","iopub.status.idle":"2022-02-15T01:07:20.362126Z","shell.execute_reply.started":"2022-02-15T01:07:19.967265Z","shell.execute_reply":"2022-02-15T01:07:20.361402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['num_bbox'] = df['annotations'].apply(lambda x: len(x))\ndata = (df.num_bbox>0).value_counts()/len(df)*100\nprint(f\"No BBox: {data[0]:0.2f}% | With BBox: {data[1]:0.2f}%\")","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:07:20.363995Z","iopub.execute_input":"2022-02-15T01:07:20.364431Z","iopub.status.idle":"2022-02-15T01:07:20.421683Z","shell.execute_reply.started":"2022-02-15T01:07:20.364391Z","shell.execute_reply":"2022-02-15T01:07:20.420931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize_voc(bboxes, image_height=720, image_width=1280):\n    '''\n    voc => [x1, y1, x2, y2]\n    normalized voc => [sx1, sy1, sx2, sy2]\n    '''\n    bboxes[..., 0::2] /= image_width\n    bboxes[..., 1::2] /= image_height\n    return bboxes\n\n    \ndef renormalize_voc(bboxes, image_height=720, image_width=1280):\n    '''\n    normalized voc => [sx1, sy1, sx2, sy2]\n    voc => [x1, y1, x2, y2]\n    '''\n    bboxes[..., 0::2] *= image_width\n    bboxes[..., 1::2] *= image_height\n    bboxes = bboxes.astype(int)\n    return bboxes\n\n\ndef voc2coco(bboxes, image_height=720, image_width=1280):\n    '''\n    voc  => [x1, y1, x2, y2]\n    coco => [xmin, ymin, w, h]\n    '''\n    bboxes[..., 2:] = bboxes[..., 2:] - bboxes[..., :2]\n    return bboxes\n\n\ndef load_image(image_path):\n    return cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n\n\ndef plot_one_box(x, img, color=None, label=None, line_thickness=None):\n    # Plots one bounding box on image img\n    tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1  # line/font thickness\n    color = color or [random.randint(0, 255) for _ in range(3)]\n    c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\n    cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)\n    if label:\n        tf = max(tl - 1, 1)  # font thickness\n        t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\n        c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\n        cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA)  # filled\n        cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)\n\ndef draw_bboxes(img, bboxes, classes, class_ids, colors=None, show_classes=None, bbox_format='yolo', class_name=False, line_thickness=2):  \n     \n    image = img.copy()\n    show_classes = classes if show_classes is None else show_classes\n    colors = [(0, 255 ,0)] if colors is None else colors\n\n    if bbox_format == 'yolo':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes:\n            \n                x1 = round(float(bbox[0])*image.shape[1])\n                y1 = round(float(bbox[1])*image.shape[0])\n                w  = round(float(bbox[2])*image.shape[1]/2) #w/2 \n                h  = round(float(bbox[3])*image.shape[0]/2)\n\n                voc_bbox = (x1-w, y1-h, x1+w, y1+h)\n                plot_one_box(voc_bbox, \n                             image,\n                             color = color,\n                             label = cls if class_name else str(get_label(cls)),\n                             line_thickness = line_thickness)\n            \n    elif bbox_format == 'coco':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes:            \n                x1 = int(round(bbox[0]))\n                y1 = int(round(bbox[1]))\n                w  = int(round(bbox[2]))\n                h  = int(round(bbox[3]))\n\n                voc_bbox = (x1, y1, x1+w, y1+h)\n                plot_one_box(voc_bbox, \n                             image,\n                             color = color,\n                             label = cls if class_name else str(cls_id),\n                             line_thickness = line_thickness)\n\n    elif bbox_format == 'voc_pascal':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes: \n                x1 = int(round(bbox[0]))\n                y1 = int(round(bbox[1]))\n                x2 = int(round(bbox[2]))\n                y2 = int(round(bbox[3]))\n                voc_bbox = (x1, y1, x2, y2)\n                plot_one_box(voc_bbox, \n                             image,\n                             color = color,\n                             label = cls if class_name else str(cls_id),\n                             line_thickness = line_thickness)\n    else:\n        raise ValueError('wrong bbox format')\n\n    return image\n\ndef get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndef get_imgsize(row):\n    row['width'], row['height'] = imagesize.get(row['image_path'])\n    return row\n\nnp.random.seed(32)\ncolors = [\n    (255, 0, 0),  # red\n    (0, 255, 0),  # green\n    (255, 255, 0), # yellow\n    (0, 127, 255), # cyan\n]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-15T01:07:20.423161Z","iopub.execute_input":"2022-02-15T01:07:20.423636Z","iopub.status.idle":"2022-02-15T01:07:20.487811Z","shell.execute_reply.started":"2022-02-15T01:07:20.423597Z","shell.execute_reply":"2022-02-15T01:07:20.487096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(ckpt_path, conf, iou):\n    model = torch.hub.load('/tmp/great-barrier-reef/yolov5',\n                           'custom',\n                           path=ckpt_path,\n                           source='local',\n                           force_reload=True,\n                           tile_helper=None,\n                          )  # local repo\n    model.conf = conf  # NMS confidence threshold\n    model.iou  = iou  # NMS IoU threshold\n    model.classes = None   # (optional list) filter by class, i.e. = [0, 15, 16] for persons, cats and dogs\n    model.multi_label = False  # NMS multiple labels per box\n    model.max_det = MAX_DET  # maximum number of detections per image\n    model.model.model.tta_scales = TTA_SCALES\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:07:20.53091Z","iopub.execute_input":"2022-02-15T01:07:20.53111Z","iopub.status.idle":"2022-02-15T01:07:20.572368Z","shell.execute_reply.started":"2022-02-15T01:07:20.531086Z","shell.execute_reply":"2022-02-15T01:07:20.571594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drop_pred(bboxes, scores, p_keep):\n    '''\n    randomly drop prediction for the probability (1 - p_keep).\n    '''\n    if p_keep == 1:\n        return bboxes, scores\n    bboxes = bboxes.copy()\n    scores = scores.copy()\n    assert len(bboxes) == len(scores)\n    pp = np.random.uniform(size=len(bboxes))\n    bboxes = bboxes[pp <= p_keep]\n    scores = scores[pp <= p_keep]\n    return bboxes, scores\n\n\ndef filter_by_threshold(bboxes, scores, conf_thre, verbose=0):\n    bboxes = bboxes.copy()\n    scores = scores.copy()\n    \n    filter_by_score = scores >= conf_thre\n    bboxes, scores = bboxes[filter_by_score], scores[filter_by_score]\n    assert len(bboxes) == len(scores)\n    if verbose >= 2:\n        print('final predictions:')\n        print(bboxes, scores)\n    return bboxes, scores\n    \n\ndef predict(model, img, size=768, augment=False, verbose=0):\n    results = model(img, size=size, augment=augment)  # custom inference size\n    preds = results.pandas().xyxy[0]\n    bboxes = preds[['xmin','ymin','xmax','ymax']].values\n    if len(bboxes) == 0:\n        bboxes, scores = np.zeros((0, 4), dtype=int), np.zeros((0), dtype=int)\n\n    scores = preds.confidence.values\n\n    return bboxes, scores\n    \ndef format_prediction(bboxes, scores):\n    annot = ''\n    if len(bboxes) > 0:\n        for idx in range(len(bboxes)):\n            xmin, ymin, w, h = bboxes[idx]\n            conf = scores[idx]\n            annot += f'{conf:.8f} {xmin} {ymin} {w} {h}'\n            annot +=' '\n        annot = annot.strip(' ')\n    return annot\n\ndef show_img(img, bboxes, scores, gt_boxes=[], image_id=\"\", newly_detected=([], []), newly_suppressed=([], []), bbox_format='yolo', img_size=(960, 540)):\n    labels = [0] * len(bboxes)\n    img = draw_bboxes(img=img,\n                      bboxes=gt_boxes,\n                      classes=[\"GT\" for _ in range(len(gt_boxes))],\n                      class_ids=[1] * len(gt_boxes),\n                      class_name=True,\n                      colors=colors, \n                      bbox_format=bbox_format,\n                      line_thickness=2)\n    img = draw_bboxes(img=img,\n                      bboxes=bboxes,\n                      classes=[f\"Pred:{score:.2f}\" for score in scores],\n                      class_ids=labels,\n                      class_name=True, \n                      colors=colors, \n                      bbox_format=bbox_format,\n                      line_thickness=2)\n    img = draw_bboxes(img=img,\n                      bboxes=newly_detected[0],\n                      classes=[f\"Det:{score:.2f}\" for score in newly_detected[1]],\n                      class_ids=[2] * len(newly_detected[0]),\n                      class_name=True, \n                      colors=colors, \n                      bbox_format=bbox_format,\n                      line_thickness=2)\n    img = draw_bboxes(img=img,\n                      bboxes=newly_suppressed[0],\n                      classes=[f\"Sup:{score:.2f}\" for score in newly_suppressed[1]],\n                      class_ids=[3] * len(newly_suppressed[0]),\n                      class_name=True, \n                      colors=colors, \n                      bbox_format=bbox_format,\n                      line_thickness=2)\n    cv2.putText(img, image_id, (0, 25), 0, 1, [255, 255, 255], thickness=1, lineType=cv2.LINE_AA)\n    return Image.fromarray(img).resize(img_size)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:07:20.706331Z","iopub.execute_input":"2022-02-15T01:07:20.70652Z","iopub.status.idle":"2022-02-15T01:07:20.760166Z","shell.execute_reply.started":"2022-02-15T01:07:20.706497Z","shell.execute_reply":"2022-02-15T01:07:20.759456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SeqNmsPostProcessor:\n    def __init__(self, num_frames=5, decay=0.9, verbose=0):\n        self.queue = deque(maxlen=num_frames)\n        self.verbose = verbose\n        self.num_frames = num_frames\n        self.decay = decay\n        if self.verbose:\n            print(f\"SeqNmsPostProcessor(num_frames={num_frames} decay={decay})\")\n        \n    def _pad_sequence(self, frame_bboxes, frame_scores):\n        \"\"\"\n        pad bboxes and frame_scores in order to shape (f, b, 4) and (f, b) respectively\n        \"\"\"\n        max_len = max([len(item) for item in frame_scores])\n        for idx, (bboxes, scores) in enumerate(zip(frame_bboxes, frame_scores)):\n            pad_bboxes = np.zeros((max_len - len(scores), 4))\n            pad_scores = np.zeros((max_len - len(scores)))\n            frame_bboxes[idx] = np.concatenate([bboxes, pad_bboxes])\n            frame_scores[idx] = np.concatenate([scores, pad_scores])\n            \n        return frame_bboxes, frame_scores\n    \n    def _drop_zero(self, bboxes, scores):\n        \"\"\"\n        drop padded zero prediction\n        \"\"\"\n        return bboxes[scores > 0], scores[scores > 0]\n \n    def apply(self, bboxes, scores, linkage_threshold, nms_threshold, score_metric):\n        bboxes, scores = bboxes.copy(), scores.copy()\n        if self.verbose:\n            original_scores = scores.copy()\n        self.queue.append((bboxes, scores))\n        frame_bboxes, frame_scores = zip(*self.queue)\n        frame_bboxes, frame_scores = list(frame_bboxes), list(frame_scores)\n        frame_bboxes, frame_scores = self._pad_sequence(frame_bboxes, frame_scores)\n        frame_bboxes = np.stack(frame_bboxes)\n        frame_scores = np.stack(frame_scores)\n        seq_nms(\n            frame_bboxes,\n            frame_scores,\n            labels=[],\n            linkage_threshold=linkage_threshold,\n            nms_threshold=nms_threshold,\n            score_metric=score_metric,\n        )\n        # decay\n        for i in range(len(self.queue)):\n            bb, ss = self.queue[i]\n            self.queue[i] = (bb, ss * self.decay)\n\n        bboxes, scores = frame_bboxes[-1, :], frame_scores[-1, :]\n        bboxes, scores = self._drop_zero(bboxes, scores)\n        if self.verbose:\n            newly_detected = (original_scores < CONF_THRE) & (scores >= CONF_THRE)\n            newly_suppressed = (original_scores >= CONF_THRE) & (scores < CONF_THRE)\n            if self.verbose >= 2:\n                print(\"Newly Detected Labels:\")\n                print(f\"  - Original Score: {original_scores[newly_detected]}\")\n                print(f\"  - Updated Score: {scores[newly_detected]}\")\n                print(f\"  - Coordinate: {bboxes[newly_detected]}\")\n                print(\"Newly Suppressed Labels:\")\n                print(f\"  - Original Score: {original_scores[newly_suppressed]}\")\n                print(f\"  - Updated Score: {scores[newly_suppressed]}\")\n                print(f\"  - Coordinate: {bboxes[newly_suppressed]}\")\n        \n        if self.verbose:\n            return bboxes, scores, (voc2coco(bboxes[newly_detected]), scores[newly_detected]), (voc2coco(bboxes[newly_suppressed]), scores[newly_suppressed])\n        else:\n            return bboxes, scores, (np.zeros((0, 4)), np.zeros((0))), (np.zeros((0, 4)), np.zeros((0)))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:07:20.888824Z","iopub.execute_input":"2022-02-15T01:07:20.889165Z","iopub.status.idle":"2022-02-15T01:07:20.942922Z","shell.execute_reply.started":"2022-02-15T01:07:20.889133Z","shell.execute_reply":"2022-02-15T01:07:20.941981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_boxes_around_edges(bboxes, scores, width, height, pixels=40, score_thr=0.2):\n    \"\"\"\n    remove low confidence boxes around the edges\n    coodinate should be in voc (x1, y1, x2, y2)\n    \"\"\"\n    filter_idx = (\n        (bboxes[:, 2] < pixels) | (bboxes[:, 0] >= width - pixels) | \n        ((bboxes[:, 3] < pixels)) | (bboxes[:, 1] >= height - pixels)\n    ) & (scores < score_thr)\n    \n    return bboxes[~filter_idx], scores[~filter_idx]","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:07:21.061189Z","iopub.execute_input":"2022-02-15T01:07:21.061397Z","iopub.status.idle":"2022-02-15T01:07:21.103002Z","shell.execute_reply.started":"2022-02-15T01:07:21.061372Z","shell.execute_reply":"2022-02-15T01:07:21.102127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def detect_pipeline(img, models, post_processor, verbose=0):\n    height, width = img.shape[:2]\n    \n    # predict -> voc(x1, y1, x2, y2)\n    boxes_list, scores_list, labels_list = [], [], []\n    for model in models:\n        bboxes, scores = predict(model, img, size=IMG_SIZE, augment=AUGMENT, verbose=verbose)\n        bboxes = normalize_voc(bboxes, height, width)\n        boxes_list.append(bboxes)\n        scores_list.append(scores)\n        labels_list.append([0 for _ in range(len(scores))])\n    \n    # WBF -> voc(x1, y1, x2, y2)\n    bboxes, scores, labels = weighted_boxes_fusion(boxes_list, scores_list, labels_list, weights=WEIGHTS, iou_thr=IOU_THR, skip_box_thr=SKIP_BOX_THR)\n\n    # renormalize box coordinate\n    bboxes = renormalize_voc(bboxes, height, width)\n    \n    # remove low confidence boxes around the edges\n    bboxes, scores = remove_boxes_around_edges(bboxes, scores, width, height, pixels=EDGE_PIXELS, score_thr=EDGE_SCORE_THR)\n    \n    if post_processor.num_frames >= 2:\n        # Seq-NMS: https://arxiv.org/abs/1602.08465\n        bboxes, scores, newly_detected, newly_suppressed = post_processor.apply(\n            bboxes,\n            scores,\n            linkage_threshold=SEQ_NMS_LK_THR,\n            nms_threshold=SEQ_NMS_NMS_THR,\n            score_metric=SEQ_NMS_SCORE_METRIC,\n        )\n    else:\n        newly_detected, newly_suppressed = (np.zeros((0, 4)), np.zeros((0))), (np.zeros((0, 4)), np.zeros((0)))\n    return bboxes, scores, newly_detected, newly_suppressed","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:07:21.242944Z","iopub.execute_input":"2022-02-15T01:07:21.243179Z","iopub.status.idle":"2022-02-15T01:07:21.288417Z","shell.execute_reply.started":"2022-02-15T01:07:21.243151Z","shell.execute_reply":"2022-02-15T01:07:21.287674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run Inference on Train","metadata":{}},{"cell_type":"code","source":"eval_df = df[(df.num_bbox >= 1) & (df.fold_id == FOLD_ID)]\nlen(eval_df)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:07:21.620266Z","iopub.execute_input":"2022-02-15T01:07:21.620538Z","iopub.status.idle":"2022-02-15T01:07:21.67028Z","shell.execute_reply.started":"2022-02-15T01:07:21.62051Z","shell.execute_reply":"2022-02-15T01:07:21.669594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_frame = 300\nduration = 3\n\nmodels = [\n    load_model(ckpt_path, conf=CONF, iou=IOU)\n    for ckpt_path in CKPT_PATHS\n]\npost_processor = SeqNmsPostProcessor(num_frames=SEQ_NMS_NUM_FRAMES, decay=SEQ_NMS_DECAY, verbose=2)\n\nfor idx, item in enumerate(eval_df.head(start_frame + duration).tail(duration).itertuples()):\n    path = item.image_path\n    image_id = item.image_id\n    gt_boxes = item.annotations\n    gt_boxes = [[b['x'], b['y'], b['width'], b['height']] for b in gt_boxes]\n    gt_boxes = np.array(gt_boxes)\n\n    img = cv2.imread(path)[...,::-1]\n    bboxes, scores, newly_detected, newly_suppressed = detect_pipeline(img, models, post_processor, verbose=2)\n\n    # convert coordinate\n    bboxes = voc2coco(bboxes)\n    \n    # filter by conf threshold\n    bboxes, scores = filter_by_threshold(bboxes, scores, CONF_THRE, verbose=2)\n    \n    # LB Probing\n    bboxes, scores = drop_pred(bboxes, scores, p_keep=P_KEEP)\n    \n    # display image with boxes\n    display(show_img(img, bboxes, scores, gt_boxes, image_id, newly_detected, newly_suppressed, bbox_format='coco'))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:07:21.806379Z","iopub.execute_input":"2022-02-15T01:07:21.806586Z","iopub.status.idle":"2022-02-15T01:07:29.452169Z","shell.execute_reply.started":"2022-02-15T01:07:21.806562Z","shell.execute_reply":"2022-02-15T01:07:29.449802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Init Env","metadata":{}},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()# initialize the environment\niter_test = env.iter_test()      # an iterator which loops over the test set and sample submission","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:06:31.359954Z","iopub.execute_input":"2022-02-15T01:06:31.360243Z","iopub.status.idle":"2022-02-15T01:06:31.432307Z","shell.execute_reply.started":"2022-02-15T01:06:31.360209Z","shell.execute_reply":"2022-02-15T01:06:31.431551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run Inference on Test","metadata":{}},{"cell_type":"code","source":"models = [\n    load_model(ckpt_path, conf=CONF, iou=IOU)\n    for ckpt_path in CKPT_PATHS\n]\npost_processor = SeqNmsPostProcessor(num_frames=SEQ_NMS_NUM_FRAMES, decay=SEQ_NMS_DECAY, verbose=0)\n\nnp.random.seed(RANDOM_SEED)\n\nfor idx, (img, pred_df) in enumerate(tqdm(iter_test)):\n    bboxes, scores, newly_detected, newly_suppressed = detect_pipeline(img, models, post_processor, verbose=0)\n    \n    # convert coordinate\n    bboxes = voc2coco(bboxes)\n    \n    # filter by conf threshold\n    bboxes, scores = filter_by_threshold(bboxes, scores, CONF_THRE, verbose=0)    \n    \n    # LB Probing\n    bboxes, scores = drop_pred(bboxes, scores, p_keep=P_KEEP)\n    \n    annot = format_prediction(bboxes, scores)    \n    pred_df['annotations'] = annot\n    env.predict(pred_df)\n    if idx < 3:\n        display(show_img(img, bboxes, scores, bbox_format='coco'))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T01:06:31.757932Z","iopub.execute_input":"2022-02-15T01:06:31.758545Z","iopub.status.idle":"2022-02-15T01:06:42.614158Z","shell.execute_reply.started":"2022-02-15T01:06:31.758509Z","shell.execute_reply":"2022-02-15T01:06:42.613413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check Submission","metadata":{}},{"cell_type":"code","source":"sub_df = pd.read_csv('submission.csv')\nsub_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}