{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# norfair dependencies\n%cd /kaggle/input/norfair031py3/\n!pip install commonmark-0.9.1-py2.py3-none-any.whl -f ./ --no-index\n!pip install rich-9.13.0-py3-none-any.whl\n\n!mkdir /kaggle/working/tmp\n!cp -r /kaggle/input/norfair031py3/filterpy-1.4.5/filterpy-1.4.5/ /kaggle/working/tmp/\n%cd /kaggle/working/tmp/filterpy-1.4.5/\n!pip install .\n!rm -rf /kaggle/working/tmp\n\n# norfair\n%cd /kaggle/input/norfair031py3/\n!pip install norfair-0.3.1-py3-none-any.whl -f ./ --no-index\n%cd ..","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:04:06.583511Z","iopub.execute_input":"2022-02-09T14:04:06.584077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kernel_mode = True\n# kernel_mode = False\nval_mode = False\n# val_mode = True\n\nsave_video = False\n# save_video = True\ntracking = False\n# tracking = True\n\nimport sys\n\nif kernel_mode:\n    sys.path.insert(0, '../input/weightedboxesfusion')\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\nimport numpy as np\nfrom tqdm.notebook import tqdm\n\ntqdm.pandas()\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nimport glob\nimport shutil\n\nimport torch\nfrom PIL import Image\nimport ast\nimport gc\n\nimport subprocess\n\nfrom ensemble_boxes import *\n\nfrom norfair import Detection, Tracker\n\ngc.enable()","metadata":{"papermill":{"duration":2.531336,"end_time":"2022-02-05T07:29:07.144746","exception":false,"start_time":"2022-02-05T07:29:04.61341","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"papermill":{"duration":1.387204,"end_time":"2022-02-05T07:29:08.544459","exception":false,"start_time":"2022-02-05T07:29:07.157255","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if kernel_mode:\n    lib_path = '/kaggle/input/yolov5-lib-ds'\nelse:\n    lib_path = '/workspace/Github/yolov5'\n\n# IMG_SIZE = 1280\nIMG_SIZE = 2560\nCONF = 0.4\n# CONF = 0.15\nIOU = 0.2\n# CONF = 0.05\n# CONF = 0.5\n# IOU = 0.55\n# IOU = 0.45\nAUGMENT = True\nTRACKING  = True\n\nmodel_outputs = {\n    \"YOLOv5m6-f10-v8-2560-rand-crop-fold0\": {\n        \"config_name\":\n        \"yolov5m\",\n        \"model_name\":\n        \"yolov5m6\",\n        \"model_file\":\n        \"../input/yolov5m6-f10-v8-2560-rand-crop-fold0/best.pt\"\n        if kernel_mode else\n        \"/workspace/Github/yolov5/starfish/YOLOv5m6-f10-v8-2560-rand-crop-fold0/weights/best.pt\",\n        \"CONF\":\n        CONF,\n        \"IOU\":\n        IOU,\n        \"IMG_SIZE\":\n        IMG_SIZE,\n        \"weight\":\n        1.0,\n    },\n    \"YOLOv5m6-f10-v8-2560-rand-crop-fold1\": {\n        \"config_name\":\n        \"yolov5m\",\n        \"model_name\":\n        \"yolov5m6\",\n        \"model_file\":\n        \"../input/yolov5m6-f10-v8-2560-rand-crop-fold1/best.pt\"\n        if kernel_mode else\n        \"/workspace/Github/yolov5/starfish/YOLOv5m6-f10-v8-2560-rand-crop-fold1/weights/best.pt\",\n        \"CONF\":\n        CONF,\n        \"IOU\":\n        IOU,\n        \"IMG_SIZE\":\n        IMG_SIZE,\n        \"weight\":\n        1.0,\n    },\n    \"YOLOv5m6-f10-v8-2560-rand-crop-fold2\": {\n        \"config_name\":\n        \"yolov5m\",\n        \"model_name\":\n        \"yolov5m6\",\n        \"model_file\":\n        \"../input/yolov5m6-f10-v8-2560-rand-crop-fold2/best.pt\"\n        if kernel_mode else\n        \"/workspace/Github/yolov5/starfish/YOLOv5m6-f10-v8-2560-rand-crop-fold2/weights/best.pt\",\n        \"CONF\":\n        CONF,\n        \"IOU\":\n        IOU,\n        \"IMG_SIZE\":\n        IMG_SIZE,\n        \"weight\":\n        1.0,\n    },\n    \"YOLOv5m6-f10-v8-2560-rand-crop-fold3\": {\n        \"config_name\":\n        \"yolov5m\",\n        \"model_name\":\n        \"yolov5m6\",\n        \"model_file\":\n        \"../input/yolov5m6-f10-v8-2560-rand-crop-fold3/best.pt\"\n        if kernel_mode else\n        \"/workspace/Github/yolov5/starfish/YOLOv5m6-f10-v8-2560-rand-crop-fold3/weights/best.pt\",\n        \"CONF\":\n        CONF,\n        \"IOU\":\n        IOU,\n        \"IMG_SIZE\":\n        IMG_SIZE,\n        \"weight\":\n        1.0,\n    },\n    \"YOLOv5m6-f10-v8-2560-rand-crop-fold4\": {\n        \"config_name\":\n        \"yolov5m\",\n        \"model_name\":\n        \"yolov5m6\",\n        \"model_file\":\n        \"../input/yolov5m6-f10-v8-2560-rand-crop-fold4/best.pt\"\n        if kernel_mode else\n        \"/workspace/Github/yolov5/starfish/YOLOv5m6-f10-v8-2560-rand-crop-fold4/weights/best.pt\",\n        \"CONF\":\n        CONF,\n        \"IOU\":\n        IOU,\n        \"IMG_SIZE\":\n        IMG_SIZE,\n        \"weight\":\n        1.0,\n    },\n    \"YOLOv5m6-f10-v8-2560-rand-crop-fold5\": {\n        \"config_name\":\n        \"yolov5m\",\n        \"model_name\":\n        \"yolov5m6\",\n        \"model_file\":\n        \"../input/yolov5m6-f10-v8-2560-rand-crop-fold5/best.pt\"\n        if kernel_mode else\n        \"/workspace/Github/yolov5/starfish/YOLOv5m6-f10-v8-2560-rand-crop-fold5/weights/best.pt\",\n        \"CONF\":\n        CONF,\n        \"IOU\":\n        IOU,\n        \"IMG_SIZE\":\n        IMG_SIZE,\n        \"weight\":\n        1.0,\n    },\n    \"YOLOv5m6-f10-v8-2560-rand-crop-fold6\": {\n        \"config_name\":\n        \"yolov5m\",\n        \"model_name\":\n        \"yolov5m6\",\n        \"model_file\":\n        \"../input/yolov5m6-f10-v8-2560-rand-crop-fold6/best.pt\"\n        if kernel_mode else\n        \"/workspace/Github/yolov5/starfish/YOLOv5m6-f10-v8-2560-rand-crop-fold6/weights/best.pt\",\n        \"CONF\":\n        CONF,\n        \"IOU\":\n        IOU,\n        \"IMG_SIZE\":\n        IMG_SIZE,\n        \"weight\":\n        1.0,\n    },\n    \"YOLOv5m6-f10-v8-2560-rand-crop-fold7\": {\n        \"config_name\":\n        \"yolov5m\",\n        \"model_name\":\n        \"yolov5m6\",\n        \"model_file\":\n        \"../input/yolov5m6-f10-v8-2560-rand-crop-fold7/best.pt\"\n        if kernel_mode else\n        \"/workspace/Github/yolov5/starfish/YOLOv5m6-f10-v8-2560-rand-crop-fold7/weights/best.pt\",\n        \"CONF\":\n        CONF,\n        \"IOU\":\n        IOU,\n        \"IMG_SIZE\":\n        IMG_SIZE,\n        \"weight\":\n        1.0,\n    },\n    \"YOLOv5m6-f10-v8-2560-rand-crop-fold8\": {\n        \"config_name\":\n        \"yolov5m\",\n        \"model_name\":\n        \"yolov5m6\",\n        \"model_file\":\n        \"../input/yolov5m6-f10-v8-2560-rand-crop-fold8/best.pt\"\n        if kernel_mode else\n        \"/workspace/Github/yolov5/starfish/YOLOv5m6-f10-v8-2560-rand-crop-fold8/weights/best.pt\",\n        \"CONF\":\n        CONF,\n        \"IOU\":\n        IOU,\n        \"IMG_SIZE\":\n        IMG_SIZE,\n        \"weight\":\n        1.0,\n    },\n    \"YOLOv5m6-f10-v8-2560-rand-crop-fold9\": {\n        \"config_name\":\n        \"yolov5m\",\n        \"model_name\":\n        \"yolov5m6\",\n        \"model_file\":\n        \"../input/yolov5m6-f10-v8-2560-rand-crop-fold9/best.pt\"\n        if kernel_mode else\n        \"/workspace/Github/yolov5/starfish/YOLOv5m6-f10-v8-2560-rand-crop-fold9/weights/best.pt\",\n        \"CONF\":\n        CONF,\n        \"IOU\":\n        IOU,\n        \"IMG_SIZE\":\n        IMG_SIZE,\n        \"weight\":\n        1.0,\n    },\n}\n\n# colors = [(0, 0, 130), (0, 130, 0), (255, 0, 0), (255, 0, 255), (0, 0, 255)]\nnp.random.seed(32)\ncolors = [(np.random.randint(255), np.random.randint(255), np.random.randint(255)) \\\n          for idx in range(len(model_outputs))]\n\niou_thr = 0.55\n# skip_box_thr = 0.05\n# skip_box_thr = 0.01\n# skip_box_thr = 0.0005\nskip_box_thr = 0.0001\nsigma = 0.01\nweights = np.ones(len(model_outputs))","metadata":{"papermill":{"duration":0.024751,"end_time":"2022-02-05T07:29:08.580788","exception":false,"start_time":"2022-02-05T07:29:08.556037","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{"papermill":{"duration":0.011093,"end_time":"2022-02-05T07:29:08.603461","exception":false,"start_time":"2022-02-05T07:29:08.592368","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def voc2yolo(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    voc  => [x1, y1, x2, y1]\n    yolo => [xmid, ymid, w, h] (normalized)\n    \"\"\"\n    bboxes = bboxes.copy().astype(\n        float)  # otherwise all value will be 0 as voc_pascal dtype is np.int\n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]] / image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]] / image_height\n    w = bboxes[..., 2] - bboxes[..., 0]\n    h = bboxes[..., 3] - bboxes[..., 1]\n    bboxes[..., 0] = bboxes[..., 0] + w / 2\n    bboxes[..., 1] = bboxes[..., 1] + h / 2\n    bboxes[..., 2] = w\n    bboxes[..., 3] = h\n    return bboxes\n\n\ndef yolo2voc(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    voc  => [x1, y1, x2, y1]\n\n    \"\"\"\n    bboxes = bboxes.copy().astype(\n        float)  # otherwise all value will be 0 as voc_pascal dtype is np.int\n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]] * image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]] * image_height\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]] / 2\n    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]\n    return bboxes\n\n\ndef coco2yolo(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    coco => [xmin, ymin, w, h]\n    yolo => [xmid, ymid, w, h] (normalized)\n    \"\"\"\n    bboxes = bboxes.copy().astype(\n        float)  # otherwise all value will be 0 as voc_pascal dtype is np.int\n    # normolizinig\n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]] / image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]] / image_height\n    # converstion (xmin, ymin) => (xmid, ymid)\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]] / 2\n    return bboxes\n\n\ndef yolo2coco(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    coco => [xmin, ymin, w, h]\n    \"\"\"\n    bboxes = bboxes.copy().astype(\n        float)  # otherwise all value will be 0 as voc_pascal dtype is np.int\n    # denormalizing\n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]] * image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]] * image_height\n    # converstion (xmid, ymid) => (xmin, ymin)\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]] / 2\n    return bboxes\n\n\ndef voc2coco(bboxes, image_height=720, image_width=1280):\n    bboxes = voc2yolo(bboxes, image_height, image_width)\n    bboxes = yolo2coco(bboxes, image_height, image_width)\n    return bboxes\n\n\ndef load_image(image_path):\n    return cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n\n\ndef plot_one_box(x, img, color=None, label=None, line_thickness=None):\n    # Plots one bounding box on image img\n    tl = line_thickness or round(\n        0.002 * (img.shape[0] + img.shape[1]) / 2) + 1  # line/font thickness\n    color = color or [random.randint(0, 255) for _ in range(3)]\n    c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\n    cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)\n    if label:\n        tf = max(tl - 1, 1)  # font thickness\n        t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\n        c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\n        cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA)  # filled\n        cv2.putText(img,\n                    label, (c1[0], c1[1] - 2),\n                    0,\n                    tl / 3, [225, 255, 255],\n                    thickness=tf,\n                    lineType=cv2.LINE_AA)\n\n\ndef draw_bboxes(img,\n                bboxes,\n                classes,\n                class_ids,\n                colors=None,\n                show_classes=None,\n                bbox_format='yolo',\n                class_name=False,\n                line_thickness=2):\n\n    image = img.copy()\n    show_classes = classes if show_classes is None else show_classes\n    colors = (0, 255, 0) if colors is None else colors\n\n    if bbox_format == 'yolo':\n\n        for idx in range(len(bboxes)):\n\n            bbox = bboxes[idx]\n            cls = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n\n            if cls in show_classes:\n\n                x1 = round(float(bbox[0]) * image.shape[1])\n                y1 = round(float(bbox[1]) * image.shape[0])\n                w = round(float(bbox[2]) * image.shape[1] / 2)  #w/2\n                h = round(float(bbox[3]) * image.shape[0] / 2)\n\n                voc_bbox = (x1 - w, y1 - h, x1 + w, y1 + h)\n                plot_one_box(voc_bbox,\n                             image,\n                             color=color,\n                             label=cls if class_name else str(get_label(cls)),\n                             line_thickness=line_thickness)\n\n    elif bbox_format == 'coco':\n\n        for idx in range(len(bboxes)):\n\n            bbox = bboxes[idx]\n            cls = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n\n            if cls in show_classes:\n                x1 = int(round(bbox[0]))\n                y1 = int(round(bbox[1]))\n                w = int(round(bbox[2]))\n                h = int(round(bbox[3]))\n\n                voc_bbox = (x1, y1, x1 + w, y1 + h)\n                plot_one_box(voc_bbox,\n                             image,\n                             color=color,\n                             label=cls if class_name else str(cls_id),\n                             line_thickness=line_thickness)\n\n    elif bbox_format == 'voc_pascal':\n\n        for idx in range(len(bboxes)):\n\n            bbox = bboxes[idx]\n            cls = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n\n            if cls in show_classes:\n                x1 = int(round(bbox[0]))\n                y1 = int(round(bbox[1]))\n                x2 = int(round(bbox[2]))\n                y2 = int(round(bbox[3]))\n                voc_bbox = (x1, y1, x2, y2)\n                plot_one_box(voc_bbox,\n                             image,\n                             color=color,\n                             label=cls if class_name else str(cls_id),\n                             line_thickness=line_thickness)\n    else:\n        raise ValueError('wrong bbox format')\n\n    return image\n\n\ndef show_img(img, bboxes, bbox_format='yolo', bbox_colors=None):\n    names = ['starfish'] * len(bboxes)\n    labels = [0] * len(bboxes)\n    img = draw_bboxes(img=img,\n                      bboxes=bboxes,\n                      classes=names,\n                      class_ids=labels,\n                      class_name=True,\n                      colors=colors if bbox_colors is None else bbox_colors,\n                      bbox_format=bbox_format,\n                      line_thickness=2)\n    return Image.fromarray(img).resize((800, 400))\n\n\ndef get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\n\ndef get_imgsize(row):\n    row['width'], row['height'] = imagesize.get(row['image_path'])\n    return row\n\n\n# Helper to convert bbox in format [x_min, y_min, x_max, y_max, score] to norfair.Detection class\ndef to_norfair(detects, frame_id):\n    result = []\n    for x_min, y_min, x_max, y_max, score in detects:\n        xc, yc = (x_min + x_max) / 2, (y_min + y_max) / 2\n        w, h = x_max - x_min, y_max - y_min\n        result.append(\n            Detection(points=np.array([xc, yc]),\n                      scores=np.array([score]),\n                      data=np.array([w, h, frame_id])))\n\n    return result\n\n\n# Euclidean distance function to match detections on this frame with tracked_objects from previous frames\ndef euclidean_distance(detection, tracked_object):\n    return np.linalg.norm(detection.points - tracked_object.estimate)\n\n\nnp.random.seed(32)\n# colors = [(np.random.randint(255), np.random.randint(255), np.random.randint(255))\\\n#           for idx in range(1)]","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.055459,"end_time":"2022-02-05T07:29:08.670119","exception":false,"start_time":"2022-02-05T07:29:08.61466","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(ckpt_path,\n               lib_path='/kaggle/input/yolov5-lib-ds',\n               conf=0.25,\n               iou=0.50):\n    model = torch.hub.load(lib_path,\n                           'custom',\n                           path=ckpt_path,\n                           source='local',\n                           force_reload=True)  # local repo\n    model.conf = conf  # NMS confidence threshold\n    model.iou = iou  # NMS IoU threshold\n    model.multi_label = False  # NMS multiple labels per box\n    model.max_det = 30  # maximum number of detections per image\n    return model\n\n\ndef predict(model, img, size=768, augment=False):\n    height, width = img.shape[:2]\n    results = model(img, size=size, augment=augment)  # custom inference size\n    preds = results.pandas().xyxy[0]\n    bboxes = preds[['xmin', 'ymin', 'xmax', 'ymax']].values\n    if len(bboxes):\n        bboxes = voc2coco(bboxes, height, width).astype(int)\n        confs = preds.confidence.values\n        classes = preds.name.values\n        return bboxes, confs, classes\n    else:\n        return [], [], []\n\n\ndef format_prediction(bboxes, confs):\n    annot = ''\n    if len(bboxes) > 0:\n        for idx in range(len(bboxes)):\n            xmin, ymin, w, h = bboxes[idx]\n            conf = confs[idx]\n            annot += f'{conf} {xmin} {ymin} {w} {h}'\n            annot += ' '\n        annot = annot.strip(' ')\n    return annot","metadata":{"_kg_hide-input":true,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.023287,"end_time":"2022-02-05T07:29:08.704884","exception":false,"start_time":"2022-02-05T07:29:08.681597","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"collapsed":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_wbf(bboxes,\n            confs,\n            classs,\n            image_size,\n            iou_thr=0.50,\n            skip_box_thr=0.0001,\n            weights=None):\n    \n    boxes = [bbox / (image_size - 1) for bbox in bboxes]\n    scores = [conf for conf in confs]\n    labels = [np.ones(conf.shape[0]) for conf in confs]\n    boxes, scores, labels = weighted_boxes_fusion(boxes,\n                                                  scores,\n                                                  labels,\n                                                  weights=weights,\n                                                  iou_thr=iou_thr,\n                                                  skip_box_thr=skip_box_thr)\n    boxes = boxes * (image_size - 1)\n    return boxes, scores, labels","metadata":{"papermill":{"duration":0.021255,"end_time":"2022-02-05T07:29:08.73748","exception":false,"start_time":"2022-02-05T07:29:08.716225","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tracking_function(tracker, frame_id, bboxes, scores):\n\n    detects = []\n    scores = []\n    predictions = []\n\n    if len(scores) > 0:\n        for i in range(len(bboxes)):\n            box = bboxes[i]\n            score = scores[i]\n            x_min = int(box[0])\n            y_min = int(box[1])\n            bbox_width = int(box[2])\n            bbox_height = int(box[3])\n            detects.append(\n                [x_min, y_min, x_min + bbox_width, y_min + bbox_height, score])\n            predictions.append('{:.2f} {} {} {} {}'.format(\n                score, x_min, y_min, bbox_width, bbox_height))\n            scores.append(score)\n\n            # print(predictions[:-1])\n\n    # Update tracks using detects from current frame\n    tracked_objects = tracker.update(detections=to_norfair(detects, frame_id))\n    for tobj in tracked_objects:\n        bbox_width, bbox_height, last_detected_frame_id = tobj.last_detection.data\n\n        if last_detected_frame_id == frame_id:  # Skip objects that were detected on current frame\n            continue\n\n        # Add objects that have no detections on current frame to predictions\n        xc, yc = tobj.estimate[0]\n        x_min, y_min = int(round(xc - bbox_width / 2)), int(\n            round(yc - bbox_height / 2))\n        score = tobj.last_detection.scores[0]\n\n        predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min,\n                                                       bbox_width,\n                                                       bbox_height))\n        scores.append(score)\n\n    return predictions, scores","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble Validation Test","metadata":{"heading_collapsed":true,"papermill":{"duration":0.011096,"end_time":"2022-02-05T07:29:08.759797","exception":false,"start_time":"2022-02-05T07:29:08.748701","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if val_mode:\n    if kernel_mode:\n        TEST_IMAGE_PATH = \"/kaggle/input/tensorflow-great-barrier-reef/train_images/video_2/5747.jpg\"\n        # TEST_IMAGE_PATH = \"/kaggle/input/tensorflow-great-barrier-reef/train_images/video_1/5659.jpg\"\n    else:\n        TEST_IMAGE_PATH = \"/workspace/Kaggle/Starfish/train_images/video_2/5747.jpg\"\n        # TEST_IMAGE_PATH = \"/workspace/Kaggle/Starfish/train_images/video_1/5659.jpg\"\n\n    img = cv2.imread(TEST_IMAGE_PATH)\n\n    out_image = None\n\n    boxes_list = []\n    scores_list = []\n    labels_list = []\n    weight_list = []\n    for i, (model_name, metadata) in enumerate(model_outputs.items()):\n        print(model_name)\n        ckpt_path = metadata[\"model_file\"]\n        weight = metadata[\"weight\"]\n\n        model = load_model(ckpt_path, lib_path=lib_path, conf=0.4, iou=0.2)\n\n        bboxes, scores, bbclasses = predict(model,\n                                            img,\n                                            size=2560,\n                                            augment=AUGMENT)\n\n        if out_image is not None:\n            out_image = draw_bboxes(img=out_image,\n                                    bboxes=bboxes,\n                                    classes=[f\"model{i+1}\"] * len(bboxes),\n                                    class_ids=[0] * len(bboxes),\n                                    class_name=True,\n                                    colors=colors[i],\n                                    bbox_format='coco',\n                                    line_thickness=2)\n        else:\n            out_image = draw_bboxes(img=img,\n                                    bboxes=bboxes,\n                                    classes=[f\"model{i+1}\"] * len(bboxes),\n                                    class_ids=[0] * len(bboxes),\n                                    class_name=True,\n                                    colors=colors[i],\n                                    bbox_format='coco',\n                                    line_thickness=2)\n\n        if len(bboxes) > 0:\n            # Convert (h, w) to (x2, y2)\n            bboxes[:, 2] = bboxes[:, 2] + bboxes[:, 0]\n            bboxes[:, 3] = bboxes[:, 3] + bboxes[:, 1]\n\n            boxes_list.append(bboxes)\n            scores_list.append(scores)\n            labels_list.append(bbclasses)\n            weight_list.append(weight)\n\n        del model\n        torch.cuda.empty_cache()\n        gc.collect()\n\n    out_image = cv2.cvtColor(out_image, cv2.COLOR_BGR2RGB)\n    display(Image.fromarray(out_image))","metadata":{"hidden":true,"papermill":{"duration":0.027534,"end_time":"2022-02-05T07:29:08.821481","exception":false,"start_time":"2022-02-05T07:29:08.793947","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if val_mode:\n    ## WBF ##\n    boxes, scores, labels = run_wbf(boxes_list,\n                                    scores_list,\n                                    labels_list,\n                                    IMG_SIZE,\n                                    iou_thr,\n                                    skip_box_thr,\n                                    weights=weight_list)\n    # Convert (x2, y2) back to (h, w)\n    boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n    boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n    out_image_esemble = draw_bboxes(img=img,\n                                    bboxes=boxes,\n                                    classes=[\"WBF\"] * len(boxes),\n                                    class_ids=[0] * len(boxes),\n                                    class_name=True,\n                                    colors=(50, 50, 50),\n                                    bbox_format='coco',\n                                    line_thickness=2)\n    out_image_esemble = cv2.cvtColor(out_image_esemble, cv2.COLOR_BGR2RGB)\n    print(\n        \"------------------------------  WBF   ---------------------------------------\"\n    )\n    display(Image.fromarray(out_image_esemble))","metadata":{"hidden":true,"papermill":{"duration":0.02089,"end_time":"2022-02-05T07:29:08.853935","exception":false,"start_time":"2022-02-05T07:29:08.833045","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble Video","metadata":{}},{"cell_type":"code","source":"if save_video:\n    TRAIN_PATH = '/workspace/Kaggle/Starfish'\n    df = pd.read_csv(f\"{TRAIN_PATH}/train.csv\")\n\n    def get_path(row):\n        row['image_path'] = f'{TRAIN_PATH}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n        return row\n\n    df['annotations'] = df['annotations'].apply(eval)\n    df['n_annotations'] = df['annotations'].str.len()\n    df['has_annotations'] = df['annotations'].str.len() > 0\n    df['has_2_or_more_annotations'] = df['annotations'].str.len() >= 2\n    df['doesnt_have_annotations'] = df['annotations'].str.len() == 0\n    df['start_cut_here'] = df['has_annotations'] & df[\n        'doesnt_have_annotations'].shift(\n            1) & df['doesnt_have_annotations'].shift(2)\n    df['end_cut_here'] = df['doesnt_have_annotations'] & df[\n        'has_annotations'].shift(1) & df['has_annotations'].shift(2)\n    df['sequence_change'] = df['sequence'] != df['sequence'].shift(1)\n    df['last_row'] = df.index == len(df) - 1\n    df['cut_here'] = df['start_cut_here'] | df['end_cut_here'] | df[\n        'sequence_change'] | df['last_row']\n    start_idx = 0\n    for subsequence_id, end_idx in enumerate(df[df['cut_here']].index):\n        df.loc[start_idx:end_idx, 'subsequence_id'] = subsequence_id\n        start_idx = end_idx\n    df['subsequence_id'] = df['subsequence_id'].astype(int)\n    drop_cols = [\n        'start_cut_here', 'end_cut_here', 'sequence_change', 'last_row',\n        'cut_here', 'has_2_or_more_annotations', 'doesnt_have_annotations'\n    ]\n    df = df.drop(drop_cols, axis=1)\n\n    # Path of images\n    df = df.progress_apply(get_path, axis=1)\n\n    df_manybbox = df[df[\"n_annotations\"] > 3]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if tracking:\n#     tracker = Tracker(\n#         distance_function=euclidean_distance,\n#         distance_threshold=30,\n#         hit_inertia_min=3,\n#         hit_inertia_max=6,\n#         initialization_delay=1,\n#     )\n\n# model = load_model(CKPT_PATH, conf=CONF, iou=IOU)\n# image_paths = df[df.num_bbox > 1].sample(100).image_path.tolist()\n# frame_id = 0\n# for idx, path in enumerate(image_paths):\n#     img = cv2.imread(path)[..., ::-1]\n#     if FDA_aug:\n#         img = FDA_trans(image=img)['image']\n#     bboxes, confis = predict(model, img, size=IMG_SIZE, augment=AUGMENT)\n#     predict_box = tracking_function(tracker, frame_id, bboxes, confis)\n\n#     if len(predict_box) > 0:\n#         box = [list(map(int, box.split(' ')[1:])) for box in predict_box]\n#     else:\n#         box = []\n#     display(show_img(img, box, bbox_format='coco'))\n#     if idx > 5:\n#         break\n        \n#     frame_id += 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if save_video:\n    # slice_limit = 50\n    slice_limit = 300\n    loaded_models = {}\n\n    selected_slices = {\n        0: 5250,\n        1: 4500,\n        2: 5650,\n        # 2: 5741,\n        # 2: 4750,\n    }\n    selected_fps = {\n        0: 10,\n        1: 5,\n        2: 10,\n    }\n    for video_id in [2]:\n        #     for video_id in df[\"video_id\"].unique():\n        # partly borrowed from https://github.com/RobMulla/helmet-assignment/blob/main/helmet_assignment/video.py\n        #         fps = 30\n        #         width = 1280\n        #         height = 720\n\n        if tracking:\n            save_path = f'video_{video_id}_track.mp4'\n            tmp_path = f'tmp_video_{video_id}_track.avi'\n        else:\n            save_path = f'video_{video_id}.mp4'\n            tmp_path = f'tmp_video_{video_id}.avi'\n            #         output_video = cv2.VideoWriter(tmp_path,\n            #                                        cv2.VideoWriter_fourcc(*\"MP4V\"), fps,\n            #                                        (width, height))\n\n        video_size = (1280, 720)\n\n        output_video = cv2.VideoWriter(tmp_path,\n                                       cv2.VideoWriter_fourcc(*'DIVX'),\n                                       selected_fps[video_id], video_size)\n\n        video_df = df[df[\"video_id\"] == video_id]\n\n        # slice_start = np.random.randint(df_manybbox.shape[0] - slice_limit)\n        # slice_start = np.random.randint(df_manybbox.shape[0] - slice_limit)\n        # slice_start = np.random.choice(df_manybbox.index, size=1)[0]\n        # slice_start = 5464\n        slice_start = selected_slices[video_id]\n        print(f\"Slice start: {slice_start}\")\n\n        slice_df = video_df[video_df[\"video_frame\"] >= slice_start].head(\n            slice_limit)\n        # slice_df = video_df.iloc[slice_start:slice_start + slice_limit, :]\n\n        frame_id = 0\n        tracker = Tracker(\n            distance_function=euclidean_distance,\n            distance_threshold=30,\n            hit_inertia_min=3,\n            hit_inertia_max=6,\n            initialization_delay=1,\n        )\n\n        for index, row in tqdm(slice_df.iterrows(), total=slice_df.shape[0]):\n\n            img_path = row[\"image_path\"]\n\n            img = cv2.imread(img_path)\n\n            colors = [(0, 0, 130), (0, 130, 0), (255, 0, 0), (255, 0, 255),\n                      (0, 0, 255)]\n\n            boxes_list = []\n            scores_list = []\n            labels_list = []\n            weight_list = []\n            for i, (model_name, metadata) in enumerate(model_outputs.items()):\n                # print(model_name)\n                ckpt_path = metadata[\"model_file\"]\n\n                conf = metadata[\"CONF\"]\n                iou = metadata[\"IOU\"]\n                img_size = metadata[\"IMG_SIZE\"]\n\n                weight = metadata[\"weight\"]\n\n                if model_name not in loaded_models:\n                    model = load_model(ckpt_path,\n                                       lib_path=lib_path,\n                                       conf=conf,\n                                       iou=iou)\n                    loaded_models[model_name] = model\n                else:\n                    model = loaded_models[model_name]\n\n                bboxes, scores, bbclasses = predict(model,\n                                                    img,\n                                                    size=img_size,\n                                                    augment=AUGMENT)\n\n                if tracking:\n                    predict_boxes, predict_scores = tracking_function(\n                        tracker, frame_id, bboxes, scores)\n\n                    if len(predict_boxes) > 0:\n                        bboxes = [\n                            list(map(int,\n                                     box.split(' ')[1:]))\n                            for box in predict_boxes\n                        ]\n                        bboxes = np.array(bboxes)\n                        scores = np.array(predict_scores)\n\n                if len(bboxes) > 0:\n                    # Convert (h, w) to (x2, y2)\n                    bboxes[:, 2] = bboxes[:, 2] + bboxes[:, 0]\n                    bboxes[:, 3] = bboxes[:, 3] + bboxes[:, 1]\n\n                    boxes_list.append(bboxes)\n                    scores_list.append(scores)\n                    labels_list.append(bbclasses)\n                    weight_list.append(weight)\n\n            ## WBF ##\n            if len(boxes_list) > 0:\n                ensembled_boxes, ensembled_scores, ensembled_labels = run_wbf(\n                    boxes_list,\n                    scores_list,\n                    labels_list,\n                    IMG_SIZE,\n                    iou_thr,\n                    skip_box_thr,\n                    weights=weight_list)\n                # Convert (x2, y2) back to (h, w)\n                ensembled_boxes[:,\n                                2] = ensembled_boxes[:, 2] - ensembled_boxes[:,\n                                                                             0]\n                ensembled_boxes[:,\n                                3] = ensembled_boxes[:, 3] - ensembled_boxes[:,\n                                                                             1]\n                out_image_esemble = draw_bboxes(\n                    img=img,\n                    bboxes=ensembled_boxes,\n                    classes=[\n                        f\"Starfish ({ensembled_scores[k]:.2f})\"\n                        for k in range(len(ensembled_boxes))\n                    ],\n                    class_ids=[0] * len(ensembled_boxes),\n                    class_name=True,\n                    colors=colors[1],\n                    bbox_format='coco',\n                    line_thickness=2)\n            else:\n                out_image_esemble = img\n\n            # display(Image.fromarray(out_image_esemble))\n            output_video.write(out_image_esemble)\n\n            frame_id += 1\n\n        output_video.release()\n\n        # Not all browsers support the codec, we will re-load the file at tmp_output_path\n        # and convert to a codec that is more broadly readable using ffmpeg\n        if os.path.exists(save_path):\n            os.remove(save_path)\n\n        AVI2MP4 = \"-ac 2 -b:v 2000k -c:a aac -c:v libx264 -b:a 160k -vprofile high -bf 0 -strict experimental -f mp4\"\n\n        # command = f\"ffmpeg -i {tmp_path} {AVI2MP4} {save_path}\"\n        command = f\"ffmpeg -i {tmp_path} -vf scale=1280:720 -preset slow {save_path}\"\n        subprocess.call(command, shell=True)\n        #         subprocess.run([\n        #             \"ffmpeg\", \"-i\", tmp_path, \"-crf\", \"18\", \"-preset\", \"veryfast\",\n        #             \"-vcodec\", \"libx264\", save_path\n        #         ])\n        os.remove(tmp_path)\n\n        break","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Slice start: 5464","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\nfrom base64 import b64encode\n\n\ndef play(filename):\n    html = ''\n    video = open(filename, 'rb').read()\n    src = 'data:video/mp4;base64,' + b64encode(video).decode()\n    html += '<video width=800 controls autoplay loop><source src=\"%s\" type=\"video/mp4\"></video>' % src\n    return HTML(html)\n\n\n# play('video_0.mp4')\n# play('video_1.mp4')\n# play('video_2.mp4')\n# play('video_2_track.mp4')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.011195,"end_time":"2022-02-05T07:29:08.876635","exception":false,"start_time":"2022-02-05T07:29:08.86544","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble Test Set\n","metadata":{"papermill":{"duration":0.011353,"end_time":"2022-02-05T07:29:08.899552","exception":false,"start_time":"2022-02-05T07:29:08.888199","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%cd /kaggle/working/","metadata":{"papermill":{"duration":0.020583,"end_time":"2022-02-05T07:29:08.93147","exception":false,"start_time":"2022-02-05T07:29:08.910887","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if kernel_mode:\n    import greatbarrierreef\n    env = greatbarrierreef.make_env()  # initialize the environment\n    iter_test = env.iter_test()\n\n    loaded_models = {}\n\n    frame_id = 0\n    tracker = Tracker(\n        distance_function=euclidean_distance,\n        distance_threshold=30,\n        hit_inertia_min=3,\n        hit_inertia_max=6,\n        initialization_delay=1,\n    )\n    for idx, (img, pred_df) in enumerate(tqdm(iter_test)):\n        boxes_list = []\n        scores_list = []\n        labels_list = []\n        weight_list = []\n        for i, (model_name, metadata) in enumerate(model_outputs.items()):\n            print(model_name)\n            ckpt_path = metadata[\"model_file\"]\n\n            conf = metadata[\"CONF\"]\n            iou = metadata[\"IOU\"]\n            img_size = metadata[\"IMG_SIZE\"]\n\n            weight = metadata[\"weight\"]\n\n            if model_name not in loaded_models:\n                model = load_model(ckpt_path,\n                                   lib_path=lib_path,\n                                   conf=conf,\n                                   iou=iou)\n                loaded_models[model_name] = model\n            else:\n                model = loaded_models[model_name]\n\n            bboxes, scores, bbclasses = predict(model,\n                                                img,\n                                                size=img_size,\n                                                augment=AUGMENT)\n\n            try:\n                if tracking:\n                    predict_boxes, predict_scores = tracking_function(\n                        tracker, frame_id, bboxes, scores)\n\n                    if len(predict_boxes) > 0:\n                        bboxes = [\n                            list(map(int,\n                                     box.split(' ')[1:]))\n                            for box in predict_boxes\n                        ]\n                        bboxes = np.array(bboxes)\n                        scores = np.array(predict_scores)\n                \n                if len(bboxes) > 0:\n                    # Convert (h, w) to (x2, y2)\n                    bboxes[:, 2] = bboxes[:, 2] + bboxes[:, 0]\n                    bboxes[:, 3] = bboxes[:, 3] + bboxes[:, 1]\n\n                    boxes_list.append(bboxes)\n                    scores_list.append(scores)\n                    labels_list.append(bbclasses)\n                    weight_list.append(weight)\n\n                if idx != 0 and idx % 1000 == 0:\n                    # del model\n                    # del loaded_models[model_name]\n                    torch.cuda.empty_cache()\n                    gc.collect()\n            except:\n                pass\n\n        if len(boxes_list) > 0:\n            try:\n                ensembled_boxes, ensembled_scores, ensembled_labels = run_wbf(\n                    boxes_list,\n                    scores_list,\n                    labels_list,\n                    IMG_SIZE,\n                    iou_thr,\n                    skip_box_thr,\n                    weights=weight_list)\n\n                # Convert (x2, y2) back to (h, w)\n                ensembled_boxes[:, 2] = ensembled_boxes[:, 2] - ensembled_boxes[:, 0]\n                ensembled_boxes[:, 3] = ensembled_boxes[:, 3] - ensembled_boxes[:, 1]\n\n                annotations = format_prediction(ensembled_boxes,\n                                                ensembled_scores)\n                pred_df['annotations'] = annotations\n                env.predict(pred_df)\n                \n                if idx<3:\n                    display(show_img(img, ensembled_boxes, bbox_format='coco'))\n            except:\n                pred_df['annotations'] = ''\n                env.predict(pred_df)\n        else:\n            ensembled_boxes = []\n            ensembled_scores = None\n            pred_df['annotations'] = ''\n            env.predict(pred_df)\n            \n        frame_id += 1","metadata":{"papermill":{"duration":21.495098,"end_time":"2022-02-05T07:29:30.438437","exception":false,"start_time":"2022-02-05T07:29:08.943339","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv('submission.csv')\nsub_df.head()","metadata":{"papermill":{"duration":0.03734,"end_time":"2022-02-05T07:29:30.495672","exception":false,"start_time":"2022-02-05T07:29:30.458332","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}