{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 📚 Import Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\nimport glob\nimport shutil\nimport albumentations as A\nimport sys\nsys.path.append('../input/tensorflow-great-barrier-reef')\nimport torch\nfrom PIL import Image\nimport ast","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-13T18:47:35.002738Z","iopub.execute_input":"2022-02-13T18:47:35.003658Z","iopub.status.idle":"2022-02-13T18:47:38.208748Z","shell.execute_reply.started":"2022-02-13T18:47:35.003607Z","shell.execute_reply":"2022-02-13T18:47:38.207929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nimport numpy as np\nfrom numba import jit\nimport time\n\n@jit(nopython=True)\ndef bb_intersection_over_union(A, B) -> float:\n    xA = max(A[0], B[0])\n    yA = max(A[1], B[1])\n    xB = min(A[2], B[2])\n    yB = min(A[3], B[3])\n\n    # compute the area of intersection rectangle\n    interArea = max(0, xB - xA) * max(0, yB - yA)\n\n    if interArea == 0:\n        return 0.0\n\n    # compute the area of both the prediction and ground-truth rectangles\n    boxAArea = (A[2] - A[0]) * (A[3] - A[1])\n    boxBArea = (B[2] - B[0]) * (B[3] - B[1])\n\n    iou = interArea / float(boxAArea + boxBArea - interArea)\n    return iou\n\n\ndef prefilter_boxes(boxes, scores, labels, weights, thr):\n    # Create dict with boxes stored by its label\n    new_boxes = dict()\n\n    for t in range(len(boxes)):\n\n        if len(boxes[t]) != len(scores[t]):\n            print('Error. Length of boxes arrays not equal to length of scores array: {} != {}'.format(len(boxes[t]), len(scores[t])))\n            exit()\n\n        if len(boxes[t]) != len(labels[t]):\n            print('Error. Length of boxes arrays not equal to length of labels array: {} != {}'.format(len(boxes[t]), len(labels[t])))\n            exit()\n\n        for j in range(len(boxes[t])):\n            score = scores[t][j]\n            if score < thr:\n                continue\n            label = int(labels[t][j])\n            box_part = boxes[t][j]\n            x1 = float(box_part[0])\n            y1 = float(box_part[1])\n            x2 = float(box_part[2])\n            y2 = float(box_part[3])\n\n            # Box data checks\n            if x2 < x1:\n                warnings.warn('X2 < X1 value in box. Swap them.')\n                x1, x2 = x2, x1\n            if y2 < y1:\n                warnings.warn('Y2 < Y1 value in box. Swap them.')\n                y1, y2 = y2, y1\n            if x1 < 0:\n                warnings.warn('X1 < 0 in box. Set it to 0.')\n                x1 = 0\n            if x1 > 1:\n                warnings.warn('X1 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.')\n                x1 = 1\n            if x2 < 0:\n                warnings.warn('X2 < 0 in box. Set it to 0.')\n                x2 = 0\n            if x2 > 1:\n                warnings.warn('X2 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.')\n                x2 = 1\n            if y1 < 0:\n                warnings.warn('Y1 < 0 in box. Set it to 0.')\n                y1 = 0\n            if y1 > 1:\n                warnings.warn('Y1 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.')\n                y1 = 1\n            if y2 < 0:\n                warnings.warn('Y2 < 0 in box. Set it to 0.')\n                y2 = 0\n            if y2 > 1:\n                warnings.warn('Y2 > 1 in box. Set it to 1. Check that you normalize boxes in [0, 1] range.')\n                y2 = 1\n            if (x2 - x1) * (y2 - y1) == 0.0:\n                warnings.warn(\"Zero area box skipped: {}.\".format(box_part))\n                continue\n\n            # [label, score, weight, model index, x1, y1, x2, y2]\n            b = [int(label), float(score) * weights[t], weights[t], t, x1, y1, x2, y2]\n            if label not in new_boxes:\n                new_boxes[label] = []\n            new_boxes[label].append(b)\n\n    # Sort each list in dict by score and transform it to numpy array\n    for k in new_boxes:\n        current_boxes = np.array(new_boxes[k])\n        new_boxes[k] = current_boxes[current_boxes[:, 1].argsort()[::-1]]\n\n    return new_boxes\n\n\ndef get_weighted_box(boxes, conf_type='avg'):\n    \"\"\"\n    Create weighted box for set of boxes\n    :param boxes: set of boxes to fuse\n    :param conf_type: type of confidence one of 'avg' or 'max'\n    :return: weighted box (label, score, weight, x1, y1, x2, y2)\n    \"\"\"\n\n    box = np.zeros(8, dtype=np.float32)\n    conf = 0\n    conf_list = []\n    w = 0\n    for b in boxes:\n        box[4:] += (b[1] * b[4:])\n        conf += b[1]\n        conf_list.append(b[1])\n        w += b[2]\n    box[0] = boxes[0][0]\n    if conf_type == 'avg':\n        box[1] = conf / len(boxes)\n    elif conf_type == 'max':\n        box[1] = np.array(conf_list).max()\n    elif conf_type in ['box_and_model_avg', 'absent_model_aware_avg']:\n        box[1] = conf / len(boxes)\n    box[2] = w\n    box[3] = -1 # model index field is retained for consistensy but is not used.\n    box[4:] /= conf\n    return box\n\n\ndef find_matching_box(boxes_list, new_box, match_iou):\n    best_iou = match_iou\n    best_index = -1\n    for i in range(len(boxes_list)):\n        box = boxes_list[i]\n        if box[0] != new_box[0]:\n            continue\n        iou = bb_intersection_over_union(box[4:], new_box[4:])\n        if iou > best_iou:\n            best_index = i\n            best_iou = iou\n\n    return best_index, best_iou\n\n\ndef find_matching_box_quickly(boxes_list, new_box, match_iou):\n    \"\"\" Reimplementation of find_matching_box with numpy instead of loops. Gives significant speed up for larger arrays\n        (~100x). This was previously the bottleneck since the function is called for every entry in the array.\n    \"\"\"\n    def bb_iou_array(boxes, new_box):\n        # bb interesection over union\n        xA = np.maximum(boxes[:, 0], new_box[0])\n        yA = np.maximum(boxes[:, 1], new_box[1])\n        xB = np.minimum(boxes[:, 2], new_box[2])\n        yB = np.minimum(boxes[:, 3], new_box[3])\n\n        interArea = np.maximum(xB - xA, 0) * np.maximum(yB - yA, 0)\n\n        # compute the area of both the prediction and ground-truth rectangles\n        boxAArea = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])\n        boxBArea = (new_box[2] - new_box[0]) * (new_box[3] - new_box[1])\n\n        iou = interArea / (boxAArea + boxBArea - interArea)\n\n        return iou\n\n    if boxes_list.shape[0] == 0:\n        return -1, match_iou\n\n    # boxes = np.array(boxes_list)\n    boxes = boxes_list\n\n    ious = bb_iou_array(boxes[:, 4:], new_box[4:])\n\n    ious[boxes[:, 0] != new_box[0]] = -1\n\n    best_idx = np.argmax(ious)\n    best_iou = ious[best_idx]\n\n    if best_iou <= match_iou:\n        best_iou = match_iou\n        best_idx = -1\n\n    return best_idx, best_iou\n\n\ndef weighted_boxes_fusion(boxes_list, scores_list, labels_list, weights=None, iou_thr=0.55, skip_box_thr=0.0, conf_type='avg', allows_overflow=False):\n    '''\n    :param boxes_list: list of boxes predictions from each model, each box is 4 numbers.\n    It has 3 dimensions (models_number, model_preds, 4)\n    Order of boxes: x1, y1, x2, y2. We expect float normalized coordinates [0; 1]\n    :param scores_list: list of scores for each model\n    :param labels_list: list of labels for each model\n    :param weights: list of weights for each model. Default: None, which means weight == 1 for each model\n    :param iou_thr: IoU value for boxes to be a match\n    :param skip_box_thr: exclude boxes with score lower than this variable\n    :param conf_type: how to calculate confidence in weighted boxes. 'avg': average value, 'max': maximum value, 'box_and_model_avg': box and model wise hybrid weighted average, 'absent_model_aware_avg': weighted average that takes into account the absent model.\n    :param allows_overflow: false if we want confidence score not exceed 1.0\n    :return: boxes: boxes coordinates (Order of boxes: x1, y1, x2, y2).\n    :return: scores: confidence scores\n    :return: labels: boxes labels\n    '''\n\n    if weights is None:\n        weights = np.ones(len(boxes_list))\n    if len(weights) != len(boxes_list):\n        print('Warning: incorrect number of weights {}. Must be: {}. Set weights equal to 1.'.format(len(weights), len(boxes_list)))\n        weights = np.ones(len(boxes_list))\n    weights = np.array(weights)\n\n    if conf_type not in ['avg', 'max', 'box_and_model_avg', 'absent_model_aware_avg']:\n        print('Unknown conf_type: {}. Must be \"avg\", \"max\" or \"box_and_model_avg\", or \"absent_model_aware_avg\"'.format(conf_type))\n        exit()\n\n    filtered_boxes = prefilter_boxes(boxes_list, scores_list, labels_list, weights, skip_box_thr)\n    if len(filtered_boxes) == 0:\n        return np.zeros((0, 4)), np.zeros((0,)), np.zeros((0,))\n\n    overall_boxes = []\n    for label in filtered_boxes:\n        boxes = filtered_boxes[label]\n        new_boxes = []\n        weighted_boxes = np.empty((0,8))\n        # Clusterize boxes\n        for j in range(0, len(boxes)):\n            index, best_iou = find_matching_box_quickly(weighted_boxes, boxes[j], iou_thr)\n\n            if index != -1:\n                new_boxes[index].append(boxes[j])\n                weighted_boxes[index] = get_weighted_box(new_boxes[index], conf_type)\n            else:\n                new_boxes.append([boxes[j].copy()])\n                weighted_boxes = np.vstack((weighted_boxes, boxes[j].copy()))\n        # Rescale confidence based on number of models and boxes\n        for i in range(len(new_boxes)):\n            clustered_boxes = np.array(new_boxes[i])\n            if conf_type == 'box_and_model_avg':\n                # weighted average for boxes\n                weighted_boxes[i, 1] = weighted_boxes[i, 1] * len(clustered_boxes) / weighted_boxes[i, 2]\n                # identify unique model index by model index column\n                _, idx = np.unique(clustered_boxes[:, 3], return_index=True)\n                # rescale by unique model weights\n                weighted_boxes[i, 1] = weighted_boxes[i, 1] *  clustered_boxes[idx, 2].sum() / weights.sum()\n            elif conf_type == 'absent_model_aware_avg':\n                # get unique model index in the cluster\n                models = np.unique(clustered_boxes[:, 3]).astype(int)\n                # create a mask to get unused model weights\n                mask = np.ones(len(weights), dtype=bool)\n                mask[models] = False\n                # absent model aware weighted average\n                weighted_boxes[i, 1] = weighted_boxes[i, 1] * len(clustered_boxes) / (weighted_boxes[i, 2] + weights[mask].sum())\n            elif conf_type == 'max':\n                weighted_boxes[i, 1] = weighted_boxes[i, 1] / weights.max()\n            elif not allows_overflow:\n                weighted_boxes[i, 1] = weighted_boxes[i, 1] * min(len(weights), len(clustered_boxes)) / weights.sum()\n            else:\n                weighted_boxes[i, 1] = weighted_boxes[i, 1] * len(clustered_boxes) / weights.sum()\n        overall_boxes.append(weighted_boxes)\n    overall_boxes = np.concatenate(overall_boxes, axis=0)\n    overall_boxes = overall_boxes[overall_boxes[:, 1].argsort()[::-1]]\n    boxes = overall_boxes[:, 4:]\n    scores = overall_boxes[:, 1]\n    labels = overall_boxes[:, 0]\n    return boxes, scores, labels","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:47:38.211035Z","iopub.execute_input":"2022-02-13T18:47:38.211318Z","iopub.status.idle":"2022-02-13T18:47:38.933000Z","shell.execute_reply.started":"2022-02-13T18:47:38.211279Z","shell.execute_reply":"2022-02-13T18:47:38.932306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_format = \"yolo\"\n\n#1: UD\ntransform_1 = A.Compose([\n                        A.VerticalFlip(p=1.0),\n                        ])\ninverse_transform_1 = A.Compose([\n                        A.VerticalFlip(p=1.0),\n                        ], bbox_params=A.BboxParams(format=transform_format, label_fields=[\"bbox_classes\"]))\n\n#2: LR\ntransform_2 = A.Compose([\n                        A.HorizontalFlip(p=1.0),\n                        ]) \ninverse_transform_2 = A.Compose([\n                        A.HorizontalFlip(p=1.0),\n                        ], bbox_params=A.BboxParams(format=transform_format, label_fields=[\"bbox_classes\"])) \n\n#-------------------------------------------------------------------------------------------------\nTTA_transform_list = [transform_1, transform_2] \ninverse_TTA_transform_list = [inverse_transform_1, inverse_transform_2]","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:47:38.935457Z","iopub.execute_input":"2022-02-13T18:47:38.935949Z","iopub.status.idle":"2022-02-13T18:47:38.942643Z","shell.execute_reply.started":"2022-02-13T18:47:38.935912Z","shell.execute_reply":"2022-02-13T18:47:38.941963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def voc2yolo(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    voc  => [x1, y1, x2, y2]\n    yolo => [xmid, ymid, w, h] (normalized)\n    \"\"\"\n    \n    bboxes = np.array(bboxes.copy()).astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]/ image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]/ image_height\n    \n    w = bboxes[..., 2] - bboxes[..., 0]\n    h = bboxes[..., 3] - bboxes[..., 1]\n    \n    bboxes[..., 0] = bboxes[..., 0] + w/2\n    bboxes[..., 1] = bboxes[..., 1] + h/2\n    bboxes[..., 2] = w\n    bboxes[..., 3] = h\n    \n    bboxes[..., 0::2] = np.clip(bboxes[..., 0::2], 0.0001, 0.9999)\n    bboxes[..., 1::2] = np.clip(bboxes[..., 1::2], 0.0001, 0.9999)\n    \n    return bboxes\n\ndef yolo2voc(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    voc  => [x1, y1, x2, y2]\n    \n    \"\"\" \n    bboxes = np.array(bboxes.copy()).astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height\n    \n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]\n    \n    return bboxes\n\ndef coco2yolo(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    coco => [xmin, ymin, w, h]\n    yolo => [xmid, ymid, w, h] (normalized)\n    \"\"\"\n    \n    bboxes = np.array(bboxes.copy()).astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    # normolizinig\n    bboxes[..., [0, 2]]= bboxes[..., [0, 2]]/ image_width\n    bboxes[..., [1, 3]]= bboxes[..., [1, 3]]/ image_height\n    \n    # converstion (xmin, ymin) => (xmid, ymid)\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]/2\n    \n    bboxes[..., 0::2] = np.clip(bboxes[..., 0::2], 0.0001, 0.9999)\n    bboxes[..., 1::2] = np.clip(bboxes[..., 1::2], 0.0001, 0.9999)\n    \n    return bboxes\n\ndef yolo2coco(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    coco => [xmin, ymin, w, h]\n    \n    \"\"\" \n    bboxes = np.array(bboxes.copy()).astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    # denormalizing\n    bboxes[..., [0, 2]]= bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]]= bboxes[..., [1, 3]]* image_height\n    \n    # converstion (xmid, ymid) => (xmin, ymin) \n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    \n    return bboxes\n\ndef voc2coco(bboxes, image_height=720, image_width=1280):\n    bboxes = np.array(bboxes).astype(np.float32).copy()\n    bboxes  = voc2yolo(bboxes, image_height, image_width)\n    bboxes  = yolo2coco(bboxes, image_height, image_width)\n    return bboxes\n\n\ndef yolo2albumentations(bboxes, image_height, image_width):\n    bboxes = np.array(bboxes).astype(np.float32).copy()\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    albumentations => [xmin, ymin, xmax, ymax] (normalized)\n    \"\"\"\n    # converstion (xmid, ymid) => (xmin, ymin)\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    # converstion (w, h) => (xmax, ymax)\n    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]\n    return bboxes\n\ndef albumentations2yolo(bboxes, image_height, image_width):\n    bboxes = np.array(bboxes).astype(np.float32).copy()\n    \"\"\"\n    albumentations => [xmin, ymin, xmax, ymax] (normalized)\n    yolo => [xmid, ymid, w, h] (normalized)\n    \"\"\"    \n    # converstion (xmax, ymax) => (w, h)\n    bboxes[..., [2, 3]] = bboxes[..., [2, 3]] - bboxes[..., [0, 1]]\n    # converstion (xmin, ymin) => (xmid, ymid)\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]/2\n    return bboxes\n\ndef voc2albumentations(bboxes, image_height, image_width):\n    bboxes = voc2yolo(bboxes, image_height, image_width)\n    bboxes = yolo2albumentations(bboxes, image_height, image_width)\n    return bboxes\n\ndef coco2albumentations(bboxes, image_height, image_width):\n    bboxes = coco2yolo(bboxes, image_height, image_width)\n    bboxes = yolo2albumentations(bboxes, image_height, image_width)\n    return bboxes\n\ndef albumentations2coco(bboxes, image_height, image_width):\n    bboxes = albumentations2yolo(bboxes, image_height, image_width)\n    bboxes = yolo2coco(bboxes, image_height, image_width)\n    return bboxes\n\ndef albumentations2xyxy(bboxes, image_height, image_width):\n    bboxes = np.array(bboxes).astype(np.float32).copy()\n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]] * image_width\n    bboxes[..., [1, 3]]= bboxes[..., [1, 3]]* image_height\n    return bboxes\n\ndef xyxy2albumentations(bboxes, image_height, image_width):\n    bboxes = np.array(bboxes).astype(np.float32).copy()\n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]] / image_width\n    bboxes[..., [1, 3]]= bboxes[..., [1, 3]] / image_height\n    \n    bboxes[..., 0::2] = np.clip(bboxes[..., 0::2], 0.0001, 0.9999)\n    bboxes[..., 1::2] = np.clip(bboxes[..., 1::2], 0.0001, 0.9999)\n    return bboxes\n\ndef xyxy2coco(bboxes, image_height, image_width):\n    bboxes = xyxy2albumentations(bboxes, image_height, image_width)\n    bboxes = albumentations2coco(bboxes, image_height, image_width)\n    return bboxes\n\n\ndef load_image(image_path):\n    return cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n\n\ndef plot_one_box(x, img, color=None, label=None, line_thickness=None):\n    # Plots one bounding box on image img\n    tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1  # line/font thickness\n    color = color or [random.randint(0, 255) for _ in range(3)]\n    c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\n    cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)\n    if label:\n        tf = max(tl - 1, 1)  # font thickness\n        t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\n        c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\n        cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA)  # filled\n        cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)\n\ndef draw_bboxes(img, bboxes, gt_bboxes, classes, class_ids, colors = None, show_classes = None, bbox_format = 'yolo', class_name = False, line_thickness = 2):  \n     \n    image = img.copy()\n    show_classes = classes if show_classes is None else show_classes\n    colors = (0, 255 ,0) if colors is None else colors\n    \n    if bbox_format == 'yolo':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes:\n            \n                x1 = round(float(bbox[0])*image.shape[1])\n                y1 = round(float(bbox[1])*image.shape[0])\n                w  = round(float(bbox[2])*image.shape[1]/2) #w/2 \n                h  = round(float(bbox[3])*image.shape[0]/2)\n\n                voc_bbox = (x1-w, y1-h, x1+w, y1+h)\n                plot_one_box(voc_bbox, \n                             image,\n                             color = color,\n                             label = cls if class_name else str(get_label(cls)),\n                             line_thickness = line_thickness)\n            \n    elif bbox_format == 'coco':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes:            \n                x1 = int(round(bbox[0]))\n                y1 = int(round(bbox[1]))\n                w  = int(round(bbox[2]))\n                h  = int(round(bbox[3]))\n\n                voc_bbox = (x1, y1, x1+w, y1+h)\n                plot_one_box(voc_bbox, \n                             image,\n                             color = color,\n                             label = cls if class_name else str(cls_id),\n                             line_thickness = line_thickness)\n\n    elif bbox_format == 'voc_pascal':\n        \n        for idx in range(len(bboxes)):  \n            \n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n            \n            if cls in show_classes: \n                x1 = int(round(bbox[0]))\n                y1 = int(round(bbox[1]))\n                x2 = int(round(bbox[2]))\n                y2 = int(round(bbox[3]))\n                voc_bbox = (x1, y1, x2, y2)\n                plot_one_box(voc_bbox, \n                             image,\n                             color = color,\n                             label = cls if class_name else str(cls_id),\n                             line_thickness = line_thickness)\n    else:\n        raise ValueError('wrong bbox format')\n        \n    if gt_bboxes is not None:\n        for idx in range(len(gt_bboxes)):\n\n            gt_bbox  = gt_bboxes[idx]\n            color = (0, 255, 0)\n          \n            x1 = int(round(gt_bbox[0]))\n            y1 = int(round(gt_bbox[1]))\n            w  = int(round(gt_bbox[2]))\n            h  = int(round(gt_bbox[3]))\n\n            voc_bbox = (x1, y1, x1+w, y1+h)\n            plot_one_box(voc_bbox, \n                         image,\n                         color = color,\n                         label = 'cots',\n                         line_thickness = line_thickness)\n\n    return image\n\ndef get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndef get_imgsize(row):\n    row['width'], row['height'] = imagesize.get(row['image_path'])\n    return row\n\nnp.random.seed(32)\ncolors = [(np.random.randint(255), np.random.randint(255), np.random.randint(255))\\\n          for idx in range(1)]","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:47:38.946251Z","iopub.execute_input":"2022-02-13T18:47:38.947557Z","iopub.status.idle":"2022-02-13T18:47:39.002778Z","shell.execute_reply.started":"2022-02-13T18:47:38.947516Z","shell.execute_reply":"2022-02-13T18:47:39.002071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r ../input/d/awsaf49/yolov5-lib-ds ./\nsys.path.append('./yolov5-lib-ds')","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:47:39.003812Z","iopub.execute_input":"2022-02-13T18:47:39.004135Z","iopub.status.idle":"2022-02-13T18:47:39.991355Z","shell.execute_reply.started":"2022-02-13T18:47:39.004099Z","shell.execute_reply":"2022-02-13T18:47:39.990376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile ./yolov5-lib-ds/models/yolo.py\n\n# YOLOv5 🚀 by Ultralytics, GPL-3.0 license\n\"\"\"\nYOLO-specific modules\nUsage:\n    $ python path/to/models/yolo.py --cfg yolov5s.yaml\n\"\"\"\n\nimport argparse\nimport sys\nfrom copy import deepcopy\nfrom pathlib import Path\n\nFILE = Path(__file__).resolve()\nROOT = FILE.parents[1]  # YOLOv5 root directory\nif str(ROOT) not in sys.path:\n    sys.path.append(str(ROOT))  # add ROOT to PATH\n# ROOT = ROOT.relative_to(Path.cwd())  # relative\n\nfrom models.common import *\nfrom models.experimental import *\nfrom utils.autoanchor import check_anchor_order\nfrom utils.general import LOGGER, check_version, check_yaml, make_divisible, print_args\nfrom utils.plots import feature_visualization\nfrom utils.torch_utils import fuse_conv_and_bn, initialize_weights, model_info, scale_img, select_device, time_sync\n\ntry:\n    import thop  # for FLOPs computation\nexcept ImportError:\n    thop = None\n\n\nclass Detect(nn.Module):\n    stride = None  # strides computed during build\n    onnx_dynamic = False  # ONNX export parameter\n\n    def __init__(self, nc=80, anchors=(), ch=(), inplace=True):  # detection layer\n        super().__init__()\n        self.nc = nc  # number of classes\n        self.no = nc + 5  # number of outputs per anchor\n        self.nl = len(anchors)  # number of detection layers\n        self.na = len(anchors[0]) // 2  # number of anchors\n        self.grid = [torch.zeros(1)] * self.nl  # init grid\n        self.anchor_grid = [torch.zeros(1)] * self.nl  # init anchor grid\n        self.register_buffer('anchors', torch.tensor(anchors).float().view(self.nl, -1, 2))  # shape(nl,na,2)\n        self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch)  # output conv\n        self.inplace = inplace  # use in-place ops (e.g. slice assignment)\n\n    def forward(self, x):\n        z = []  # inference output\n        for i in range(self.nl):\n            x[i] = self.m[i](x[i])  # conv\n            bs, _, ny, nx = x[i].shape  # x(bs,255,20,20) to x(bs,3,20,20,85)\n            x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous()\n\n            if not self.training:  # inference\n                if self.onnx_dynamic or self.grid[i].shape[2:4] != x[i].shape[2:4]:\n                    self.grid[i], self.anchor_grid[i] = self._make_grid(nx, ny, i)\n\n                y = x[i].sigmoid()\n                if self.inplace:\n                    y[..., 0:2] = (y[..., 0:2] * 2 - 0.5 + self.grid[i]) * self.stride[i]  # xy\n                    y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i]  # wh\n                else:  # for YOLOv5 on AWS Inferentia https://github.com/ultralytics/yolov5/pull/2953\n                    xy = (y[..., 0:2] * 2 - 0.5 + self.grid[i]) * self.stride[i]  # xy\n                    wh = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i]  # wh\n                    y = torch.cat((xy, wh, y[..., 4:]), -1)\n                z.append(y.view(bs, -1, self.no))\n\n        return x if self.training else (torch.cat(z, 1), x)\n\n    def _make_grid(self, nx=20, ny=20, i=0):\n        d = self.anchors[i].device\n        if check_version(torch.__version__, '1.10.0'):  # torch>=1.10.0 meshgrid workaround for torch>=0.7 compatibility\n            yv, xv = torch.meshgrid([torch.arange(ny, device=d), torch.arange(nx, device=d)], indexing='ij')\n        else:\n            yv, xv = torch.meshgrid([torch.arange(ny, device=d), torch.arange(nx, device=d)])\n        grid = torch.stack((xv, yv), 2).expand((1, self.na, ny, nx, 2)).float()\n        anchor_grid = (self.anchors[i].clone() * self.stride[i]) \\\n            .view((1, self.na, 1, 1, 2)).expand((1, self.na, ny, nx, 2)).float()\n        return grid, anchor_grid\n\n\nclass Model(nn.Module):\n    def __init__(self, cfg='yolov5s.yaml', ch=3, nc=None, anchors=None):  # model, input channels, number of classes\n        super().__init__()\n        if isinstance(cfg, dict):\n            self.yaml = cfg  # model dict\n        else:  # is *.yaml\n            import yaml  # for torch hub\n            self.yaml_file = Path(cfg).name\n            with open(cfg, encoding='ascii', errors='ignore') as f:\n                self.yaml = yaml.safe_load(f)  # model dict\n\n        # Define model\n        ch = self.yaml['ch'] = self.yaml.get('ch', ch)  # input channels\n        if nc and nc != self.yaml['nc']:\n            LOGGER.info(f\"Overriding model.yaml nc={self.yaml['nc']} with nc={nc}\")\n            self.yaml['nc'] = nc  # override yaml value\n        if anchors:\n            LOGGER.info(f'Overriding model.yaml anchors with anchors={anchors}')\n            self.yaml['anchors'] = round(anchors)  # override yaml value\n        self.model, self.save = parse_model(deepcopy(self.yaml), ch=[ch])  # model, savelist\n        self.names = [str(i) for i in range(self.yaml['nc'])]  # default names\n        self.inplace = self.yaml.get('inplace', True)\n\n        # Build strides, anchors\n        m = self.model[-1]  # Detect()\n        if isinstance(m, Detect):\n            s = 256  # 2x min stride\n            m.inplace = self.inplace\n            m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))])  # forward\n            m.anchors /= m.stride.view(-1, 1, 1)\n            check_anchor_order(m)\n            self.stride = m.stride\n            self._initialize_biases()  # only run once\n\n        # Init weights, biases\n        initialize_weights(self)\n        self.info()\n        LOGGER.info('')\n\n    def forward(self, x, augment=False, profile=False, visualize=False):\n        if augment:\n            return self._forward_augment(x)  # augmented inference, None\n        return self._forward_once(x, profile, visualize)  # single-scale inference, train\n\n    def _forward_augment(self, x):\n        img_size = x.shape[-2:]  # height, width\n        s = [1, 1, 0.83, 0.83]  # scales\n        f = [None, 3, None, 3, None]  # flips (2-ud, 3-lr)\n        y = []  # outputs\n        for si, fi in zip(s, f):\n            xi = scale_img(x.flip(fi) if fi else x, si, gs=int(self.stride.max()))\n            yi = self._forward_once(xi)[0]  # forward\n            # cv2.imwrite(f'img_{si}.jpg', 255 * xi[0].cpu().numpy().transpose((1, 2, 0))[:, :, ::-1])  # save\n            yi = self._descale_pred(yi, fi, si, img_size)\n            y.append(yi)\n        y = self._clip_augmented(y)  # clip augmented tails\n        return torch.cat(y, 1), None  # augmented inference, train\n\n    def _forward_once(self, x, profile=False, visualize=False):\n        y, dt = [], []  # outputs\n        for m in self.model:\n            if m.f != -1:  # if not from previous layer\n                x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f]  # from earlier layers\n            if profile:\n                self._profile_one_layer(m, x, dt)\n            x = m(x)  # run\n            y.append(x if m.i in self.save else None)  # save output\n            if visualize:\n                feature_visualization(x, m.type, m.i, save_dir=visualize)\n        return x\n\n    def _descale_pred(self, p, flips, scale, img_size):\n        # de-scale predictions following augmented inference (inverse operation)\n        if self.inplace:\n            p[..., :4] /= scale  # de-scale\n            if flips == 2:\n                p[..., 1] = img_size[0] - p[..., 1]  # de-flip ud\n            elif flips == 3:\n                p[..., 0] = img_size[1] - p[..., 0]  # de-flip lr\n        else:\n            x, y, wh = p[..., 0:1] / scale, p[..., 1:2] / scale, p[..., 2:4] / scale  # de-scale\n            if flips == 2:\n                y = img_size[0] - y  # de-flip ud\n            elif flips == 3:\n                x = img_size[1] - x  # de-flip lr\n            p = torch.cat((x, y, wh, p[..., 4:]), -1)\n        return p\n\n    def _clip_augmented(self, y):\n        # Clip YOLOv5 augmented inference tails\n        nl = self.model[-1].nl  # number of detection layers (P3-P5)\n        g = sum(4 ** x for x in range(nl))  # grid points\n        e = 1  # exclude layer count\n        i = (y[0].shape[1] // g) * sum(4 ** x for x in range(e))  # indices\n        y[0] = y[0][:, :-i]  # large\n        i = (y[-1].shape[1] // g) * sum(4 ** (nl - 1 - x) for x in range(e))  # indices\n        y[-1] = y[-1][:, i:]  # small\n        return y\n\n    def _profile_one_layer(self, m, x, dt):\n        c = isinstance(m, Detect)  # is final layer, copy input as inplace fix\n        o = thop.profile(m, inputs=(x.copy() if c else x,), verbose=False)[0] / 1E9 * 2 if thop else 0  # FLOPs\n        t = time_sync()\n        for _ in range(10):\n            m(x.copy() if c else x)\n        dt.append((time_sync() - t) * 100)\n        if m == self.model[0]:\n            LOGGER.info(f\"{'time (ms)':>10s} {'GFLOPs':>10s} {'params':>10s}  {'module'}\")\n        LOGGER.info(f'{dt[-1]:10.2f} {o:10.2f} {m.np:10.0f}  {m.type}')\n        if c:\n            LOGGER.info(f\"{sum(dt):10.2f} {'-':>10s} {'-':>10s}  Total\")\n\n    def _initialize_biases(self, cf=None):  # initialize biases into Detect(), cf is class frequency\n        # https://arxiv.org/abs/1708.02002 section 3.3\n        # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1.\n        m = self.model[-1]  # Detect() module\n        for mi, s in zip(m.m, m.stride):  # from\n            b = mi.bias.view(m.na, -1)  # conv.bias(255) to (3,85)\n            b.data[:, 4] += math.log(8 / (640 / s) ** 2)  # obj (8 objects per 640 image)\n            b.data[:, 5:] += math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum())  # cls\n            mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\n\n    def _print_biases(self):\n        m = self.model[-1]  # Detect() module\n        for mi in m.m:  # from\n            b = mi.bias.detach().view(m.na, -1).T  # conv.bias(255) to (3,85)\n            LOGGER.info(\n                ('%6g Conv2d.bias:' + '%10.3g' * 6) % (mi.weight.shape[1], *b[:5].mean(1).tolist(), b[5:].mean()))\n\n    # def _print_weights(self):\n    #     for m in self.model.modules():\n    #         if type(m) is Bottleneck:\n    #             LOGGER.info('%10.3g' % (m.w.detach().sigmoid() * 2))  # shortcut weights\n\n    def fuse(self):  # fuse model Conv2d() + BatchNorm2d() layers\n        LOGGER.info('Fusing layers... ')\n        for m in self.model.modules():\n            if isinstance(m, (Conv, DWConv)) and hasattr(m, 'bn'):\n                m.conv = fuse_conv_and_bn(m.conv, m.bn)  # update conv\n                delattr(m, 'bn')  # remove batchnorm\n                m.forward = m.forward_fuse  # update forward\n        self.info()\n        return self\n\n    def info(self, verbose=False, img_size=640):  # print model information\n        model_info(self, verbose, img_size)\n\n    def _apply(self, fn):\n        # Apply to(), cpu(), cuda(), half() to model tensors that are not parameters or registered buffers\n        self = super()._apply(fn)\n        m = self.model[-1]  # Detect()\n        if isinstance(m, Detect):\n            m.stride = fn(m.stride)\n            m.grid = list(map(fn, m.grid))\n            if isinstance(m.anchor_grid, list):\n                m.anchor_grid = list(map(fn, m.anchor_grid))\n        return self\n\n\ndef parse_model(d, ch):  # model_dict, input_channels(3)\n    LOGGER.info(f\"\\n{'':>3}{'from':>18}{'n':>3}{'params':>10}  {'module':<40}{'arguments':<30}\")\n    anchors, nc, gd, gw = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple']\n    na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors  # number of anchors\n    no = na * (nc + 5)  # number of outputs = anchors * (classes + 5)\n\n    layers, save, c2 = [], [], ch[-1]  # layers, savelist, ch out\n    for i, (f, n, m, args) in enumerate(d['backbone'] + d['head']):  # from, number, module, args\n        m = eval(m) if isinstance(m, str) else m  # eval strings\n        for j, a in enumerate(args):\n            try:\n                args[j] = eval(a) if isinstance(a, str) else a  # eval strings\n            except NameError:\n                pass\n\n        n = n_ = max(round(n * gd), 1) if n > 1 else n  # depth gain\n        if m in [Conv, GhostConv, Bottleneck, GhostBottleneck, SPP, SPPF, DWConv, MixConv2d, Focus, CrossConv,\n                 BottleneckCSP, C3, C3TR, C3SPP, C3Ghost]:\n            c1, c2 = ch[f], args[0]\n            if c2 != no:  # if not output\n                c2 = make_divisible(c2 * gw, 8)\n\n            args = [c1, c2, *args[1:]]\n            if m in [BottleneckCSP, C3, C3TR, C3Ghost]:\n                args.insert(2, n)  # number of repeats\n                n = 1\n        elif m is nn.BatchNorm2d:\n            args = [ch[f]]\n        elif m is Concat:\n            c2 = sum(ch[x] for x in f)\n        elif m is Detect:\n            args.append([ch[x] for x in f])\n            if isinstance(args[1], int):  # number of anchors\n                args[1] = [list(range(args[1] * 2))] * len(f)\n        elif m is Contract:\n            c2 = ch[f] * args[0] ** 2\n        elif m is Expand:\n            c2 = ch[f] // args[0] ** 2\n        else:\n            c2 = ch[f]\n\n        m_ = nn.Sequential(*(m(*args) for _ in range(n))) if n > 1 else m(*args)  # module\n        t = str(m)[8:-2].replace('__main__.', '')  # module type\n        np = sum(x.numel() for x in m_.parameters())  # number params\n        m_.i, m_.f, m_.type, m_.np = i, f, t, np  # attach index, 'from' index, type, number params\n        LOGGER.info(f'{i:>3}{str(f):>18}{n_:>3}{np:10.0f}  {t:<40}{str(args):<30}')  # print\n        save.extend(x % i for x in ([f] if isinstance(f, int) else f) if x != -1)  # append to savelist\n        layers.append(m_)\n        if i == 0:\n            ch = []\n        ch.append(c2)\n    return nn.Sequential(*layers), sorted(save)\n\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser()\n    parser.add_argument('--cfg', type=str, default='yolov5s.yaml', help='model.yaml')\n    parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')\n    parser.add_argument('--profile', action='store_true', help='profile model speed')\n    parser.add_argument('--test', action='store_true', help='test all yolo*.yaml')\n    opt = parser.parse_args()\n    opt.cfg = check_yaml(opt.cfg)  # check YAML\n    print_args(FILE.stem, opt)\n    device = select_device(opt.device)\n\n    # Create model\n    model = Model(opt.cfg).to(device)\n    model.train()\n\n    # Profile\n    if opt.profile:\n        img = torch.rand(8 if torch.cuda.is_available() else 1, 3, 640, 640).to(device)\n        y = model(img, profile=True)\n\n    # Test all models\n    if opt.test:\n        for cfg in Path(ROOT / 'models').rglob('yolo*.yaml'):\n            try:\n                _ = Model(cfg)\n            except Exception as e:\n                print(f'Error in {cfg}: {e}')\n\n    # Tensorboard (not working https://github.com/ultralytics/yolov5/issues/2898)\n    # from torch.utils.tensorboard import SummaryWriter\n    # tb_writer = SummaryWriter('.')\n    # LOGGER.info(\"Run 'tensorboard --logdir=models' to view tensorboard at http://localhost:6006/\")\n    # tb_writer.add_graph(torch.jit.trace(model, img, strict=False), [])  # add model graph","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:47:39.993408Z","iopub.execute_input":"2022-02-13T18:47:39.993742Z","iopub.status.idle":"2022-02-13T18:47:40.011961Z","shell.execute_reply.started":"2022-02-13T18:47:39.993700Z","shell.execute_reply":"2022-02-13T18:47:40.011117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tracker","metadata":{}},{"cell_type":"code","source":"# Dependencies\n%cd /kaggle/input/norfair031py3/\n!pip install commonmark-0.9.1-py2.py3-none-any.whl -f ./ --no-index\n!pip install rich-9.13.0-py3-none-any.whl\n\n!mkdir /kaggle/working/tmp\n!cp -r /kaggle/input/norfair031py3/filterpy-1.4.5/filterpy-1.4.5/ /kaggle/working/tmp/\n%cd /kaggle/working/tmp/filterpy-1.4.5/\n!pip install .\n!rm -rf /kaggle/working/tmp\n\n%cd /kaggle/input/norfair031py3/\n!pip install norfair-0.3.1-py3-none-any.whl -f ./ --no-index\n%cd /kaggle/working/\n\nfrom norfair import Detection, Tracker\n\ndef to_norfair(detects, frame_id):\n    result = []\n    for x_min, y_min, x_max, y_max, score in detects:\n        xc, yc = (x_min + x_max) / 2, (y_min + y_max) / 2\n        w, h = x_max - x_min, y_max - y_min\n        result.append(Detection(points=np.array([xc, yc]), scores=np.array([score]), data=np.array([w, h, frame_id])))\n\n    return result\n\n# Euclidean distance function to match detections on this frame with tracked_objects from previous frames\ndef euclidean_distance(detection, tracked_object):\n    return np.linalg.norm(detection.points - tracked_object.estimate)\n\ntracker = Tracker(\n    distance_function=euclidean_distance, \n    distance_threshold=30,\n    hit_inertia_min=3,\n    hit_inertia_max=6,\n    initialization_delay=1,\n)\nframe_id = 0","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:47:40.013378Z","iopub.execute_input":"2022-02-13T18:47:40.013755Z","iopub.status.idle":"2022-02-13T18:48:54.248176Z","shell.execute_reply.started":"2022-02-13T18:47:40.013719Z","shell.execute_reply":"2022-02-13T18:48:54.247401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_path(row):\n    row['image_path'] = f'{ROOT_DIR}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    return row","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:48:54.249564Z","iopub.execute_input":"2022-02-13T18:48:54.250614Z","iopub.status.idle":"2022-02-13T18:48:54.255045Z","shell.execute_reply.started":"2022-02-13T18:48:54.250568Z","shell.execute_reply":"2022-02-13T18:48:54.254396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Data\nROOT_DIR  = '/kaggle/input/tensorflow-great-barrier-reef/'\ndf = pd.read_csv(f'{ROOT_DIR}/train.csv')\ndf = df.progress_apply(get_path, axis=1)\ndf['annotations'] = df['annotations'].progress_apply(lambda x: ast.literal_eval(x))\ndisplay(df.head(2))","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:48:54.256083Z","iopub.execute_input":"2022-02-13T18:48:54.257587Z","iopub.status.idle":"2022-02-13T18:49:10.105222Z","shell.execute_reply.started":"2022-02-13T18:48:54.257547Z","shell.execute_reply":"2022-02-13T18:49:10.104427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Number of BBoxes","metadata":{}},{"cell_type":"code","source":"df['num_bbox'] = df['annotations'].progress_apply(lambda x: len(x))\ndata = (df.num_bbox>0).value_counts()/len(df)*100\nprint(f\"No BBox: {data[0]:0.2f}% | With BBox: {data[1]:0.2f}%\")","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:49:10.108743Z","iopub.execute_input":"2022-02-13T18:49:10.108938Z","iopub.status.idle":"2022-02-13T18:49:10.204811Z","shell.execute_reply.started":"2022-02-13T18:49:10.108914Z","shell.execute_reply":"2022-02-13T18:49:10.204132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-13T18:49:10.205856Z","iopub.execute_input":"2022-02-13T18:49:10.206543Z","iopub.status.idle":"2022-02-13T18:49:11.573236Z","shell.execute_reply.started":"2022-02-13T18:49:10.206506Z","shell.execute_reply":"2022-02-13T18:49:11.572250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(ckpt_path, conf=0.25, iou=0.50):\n    model = torch.hub.load('./yolov5-lib-ds',\n                           'custom',\n                           path=ckpt_path,\n                           source='local',\n                           force_reload=True)  # local repo\n    model.conf = conf  # NMS confidence threshold\n    model.iou  = iou   # NMS IoU threshold\n    model.classes = None   # (optional list) filter by class, i.e. = [0, 15, 16] for persons, cats and dogs\n    model.multi_label = False  # NMS multiple labels per box\n    model.max_det = 1000  # maximum number of detections per image\n    return model","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-13T18:49:11.574808Z","iopub.execute_input":"2022-02-13T18:49:11.575095Z","iopub.status.idle":"2022-02-13T18:49:11.580560Z","shell.execute_reply.started":"2022-02-13T18:49:11.575054Z","shell.execute_reply":"2022-02-13T18:49:11.579869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔭 Inference","metadata":{}},{"cell_type":"markdown","source":"## Helper","metadata":{}},{"cell_type":"code","source":"def predict(model, img, size=768, augment=False):\n    height, width = img.shape[:2]\n    results = model(img, size=size, augment=augment)  # custom inference size\n    preds   = results.pandas().xyxy[0]\n    bboxes  = preds[['xmin','ymin','xmax','ymax']].values\n    if len(bboxes):\n        bboxes  = voc2coco(bboxes,height,width).astype(int)\n        confs   = preds.confidence.values\n        return bboxes, confs\n    else:\n        return [],[]\n    \ndef format_prediction(bboxes, confs):\n    annot = ''\n    if len(bboxes)>0:\n        for idx in range(len(bboxes)):\n            xmin, ymin, w, h = bboxes[idx]\n            conf             = confs[idx]\n            annot += f'{conf} {xmin} {ymin} {w} {h}'\n            annot +=' '\n        annot = annot.strip(' ')\n    return annot\n\ndef show_img(img, bboxes, gt_bboxes=None, visual_size=(800, 400), colors=colors, bbox_format='coco'):\n    names  = ['starfish']*len(bboxes)\n    labels = [0]*len(bboxes)\n    img    = draw_bboxes(img = img,\n                           bboxes = bboxes, \n                           gt_bboxes = gt_bboxes,\n                           classes = names,\n                           class_ids = labels,\n                           class_name = True, \n                           colors = colors, \n                           bbox_format = bbox_format,\n                           line_thickness = 2)\n    return Image.fromarray(img).resize(visual_size)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-13T18:49:11.581943Z","iopub.execute_input":"2022-02-13T18:49:11.582365Z","iopub.status.idle":"2022-02-13T18:49:11.596203Z","shell.execute_reply.started":"2022-02-13T18:49:11.582328Z","shell.execute_reply":"2022-02-13T18:49:11.595463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tracking_function(tracker, frame_id, bboxes, scores):\n    \n    detects = []\n    predictions = []\n    \n    if len(scores)>0:\n        for i in range(len(bboxes)):\n            box = bboxes[i]\n            score = scores[i]\n            x_min = int(box[0])\n            y_min = int(box[1])\n            bbox_width = int(box[2])\n            bbox_height = int(box[3])\n            detects.append([x_min, y_min, x_min+bbox_width, y_min+bbox_height, score])\n            predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n#             print(predictions[:-1])\n    # Update tracks using detects from current frame\n    tracked_objects = tracker.update(detections=to_norfair(detects, frame_id))\n    for tobj in tracked_objects:\n        bbox_width, bbox_height, last_detected_frame_id = tobj.last_detection.data\n        if last_detected_frame_id == frame_id:  # Skip objects that were detected on current frame\n            continue\n        # Add objects that have no detections on current frame to predictions\n        xc, yc = tobj.estimate[0]\n        x_min, y_min = int(round(xc - bbox_width / 2)), int(round(yc - bbox_height / 2))\n        score = tobj.last_detection.scores[0]\n\n        predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n        \n    return predictions","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:49:11.597667Z","iopub.execute_input":"2022-02-13T18:49:11.598221Z","iopub.status.idle":"2022-02-13T18:49:11.609957Z","shell.execute_reply.started":"2022-02-13T18:49:11.598183Z","shell.execute_reply":"2022-02-13T18:49:11.609050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_bbox(anno):\n    for idx, a in enumerate(anno):\n        try:\n            anno[idx] = [a['x'], a['y'], a['width'], a['height']]\n        except:\n            break\n    return anno","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:49:11.611324Z","iopub.execute_input":"2022-02-13T18:49:11.611844Z","iopub.status.idle":"2022-02-13T18:49:11.622071Z","shell.execute_reply.started":"2022-02-13T18:49:11.611805Z","shell.execute_reply":"2022-02-13T18:49:11.621260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_output_from_prediction(model_info, img):\n    model = model_info['model']\n    s = model_info['s']\n    f = model_info['f']\n    assert len(s) == len(f)\n    use_wbf = True\n    if len(s) == 1:\n        use_wbf = False\n        \n    image_size = model_info['image_size']\n    conf_wbf = model_info.get('conf_wbf', 0.5)\n    iou_wbf = model_info.get('iou_wbf', 0.5)\n    \n    for si, fi in zip (s, f):\n        imgi = img.copy()\n        if fi is not None:\n            imgi = TTA_transform_list[fi](image=imgi)['image']\n        bboxes, confis = predict(model, imgi, size=int(image_size * si), augment=False)\n        if fi is not None:\n            if len(bboxes):\n                bboxes = coco2yolo(bboxes, ORI_SIZE[1], ORI_SIZE[0])\n            new = inverse_TTA_transform_list[fi](image=img, bboxes=bboxes, bbox_classes=confis)\n            bboxes = new['bboxes']\n            if len(bboxes):\n                bboxes = yolo2coco(new['bboxes'], ORI_SIZE[1], ORI_SIZE[0])\n            confis = new['bbox_classes']\n        if not use_wbf:\n            return bboxes, confis\n        \n        if len(bboxes):\n            bboxes = coco2albumentations(bboxes, ORI_SIZE[1], ORI_SIZE[0])\n        else:\n            bboxes = []\n        \n        bboxes = list(map(list, bboxes))\n        boxes_list.append(bboxes)\n        scores_list.append(confis)\n        labels_list.append(np.zeros_like(confis, dtype=np.uint8).tolist())\n\n    bboxes, confis, _ = weighted_boxes_fusion(boxes_list, scores_list, labels_list, \n                                              iou_thr=iou_wbf, skip_box_thr=conf_wbf)\n    if len(bboxes):\n        bboxes = albumentations2coco(bboxes, ORI_SIZE[1], ORI_SIZE[0]).astype(int)\n    else:\n        bboxes = []\n    return bboxes, confis","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:49:11.625393Z","iopub.execute_input":"2022-02-13T18:49:11.626612Z","iopub.status.idle":"2022-02-13T18:49:11.640362Z","shell.execute_reply.started":"2022-02-13T18:49:11.626514Z","shell.execute_reply":"2022-02-13T18:49:11.639604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Run Inference on **Train**","metadata":{}},{"cell_type":"code","source":"!cp ../input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:49:11.641554Z","iopub.execute_input":"2022-02-13T18:49:11.641801Z","iopub.status.idle":"2022-02-13T18:49:12.307007Z","shell.execute_reply.started":"2022-02-13T18:49:11.641767Z","shell.execute_reply":"2022-02-13T18:49:12.305845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ORI_SIZE  = (1280, 720)\n\nFDA_aug = False\n\nCONF_WBF_ENSEMBLE = 0.28 # 0.18\nIOU_WBF_ENSEMBLE = 0.49","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:49:12.308613Z","iopub.execute_input":"2022-02-13T18:49:12.308911Z","iopub.status.idle":"2022-02-13T18:49:12.314521Z","shell.execute_reply.started":"2022-02-13T18:49:12.308871Z","shell.execute_reply":"2022-02-13T18:49:12.313362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1 = {'model': load_model('/kaggle/input/yolov5-1920-4/best.pt', conf=0.1, iou=0.6),\n          's': [1, 0.83, 0.67],\n          'f': [None, 1, None],\n          'image_size': 7749,\n          'conf_wbf': 0.38,\n          'iou_wbf': 0.49}","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:49:12.315806Z","iopub.execute_input":"2022-02-13T18:49:12.316232Z","iopub.status.idle":"2022-02-13T18:49:20.654487Z","shell.execute_reply.started":"2022-02-13T18:49:12.316196Z","shell.execute_reply":"2022-02-13T18:49:20.653722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model2 = {'model': load_model('/kaggle/input/reef-baseline-fold12/l6_3600_uflip_vm5_f12_up/f1/best.pt', conf=0.1, iou=0.6),\n          's': [1, 0.83, 0.67],\n          'f': [None, 1, None],\n          'image_size': 7000,\n          'conf_wbf': 0.38,\n          'iou_wbf': 0.49}","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:49:20.656189Z","iopub.execute_input":"2022-02-13T18:49:20.656450Z","iopub.status.idle":"2022-02-13T18:49:21.216254Z","shell.execute_reply.started":"2022-02-13T18:49:20.656412Z","shell.execute_reply":"2022-02-13T18:49:21.215507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model3 = {'model': load_model('/kaggle/input/reef-baseline-fold12/l6_3600_uflip_vm5_f12_up/f2/best.pt', conf=0.1, iou=0.6),\n          's': [1, 0.83],\n          'f': [None, 1],\n          'image_size': 7000,\n          'conf_wbf': 0.38,\n          'iou_wbf': 0.49}","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:49:21.217817Z","iopub.execute_input":"2022-02-13T18:49:21.218391Z","iopub.status.idle":"2022-02-13T18:49:21.760828Z","shell.execute_reply.started":"2022-02-13T18:49:21.218345Z","shell.execute_reply":"2022-02-13T18:49:21.760045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model4 = {'model': load_model('/kaggle/input/yolov5s6/f2_sub2.pt', conf=0.3, iou=0.5),\n          's': [1, 0.83],\n          'f': [None, 1],\n          'image_size': 5678,\n          'conf_wbf': 0.38,\n          'iou_wbf': 0.49}","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:49:21.762507Z","iopub.execute_input":"2022-02-13T18:49:21.762722Z","iopub.status.idle":"2022-02-13T18:49:22.367142Z","shell.execute_reply.started":"2022-02-13T18:49:21.762694Z","shell.execute_reply":"2022-02-13T18:49:22.366421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model5 = {'model': load_model('/kaggle/input/yolov5s6/yolov5s6_10_2_lossfn.pt', conf=0.3, iou=0.5),\n          's': [1, 0.83],\n          'f': [None, 1],\n          'image_size': 5678,\n          'conf_wbf': 0.38,\n          'iou_wbf': 0.49}","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:49:22.368614Z","iopub.execute_input":"2022-02-13T18:49:22.369135Z","iopub.status.idle":"2022-02-13T18:49:23.002237Z","shell.execute_reply.started":"2022-02-13T18:49:22.369090Z","shell.execute_reply":"2022-02-13T18:49:23.001494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Use TRACKING...\")    \ntracker = Tracker(\ndistance_function=euclidean_distance, \ndistance_threshold=30,\nhit_inertia_min=3,\nhit_inertia_max=6,\ninitialization_delay=1,\n)\n\ndf_samples = df[df.num_bbox>1].sample(100)\ndf_samples = df[9302:]\nimage_paths = df_samples.image_path.tolist()\nimage_anns = df_samples.annotations.tolist()\n\nmodel_list = [model4, model1]\n\nuse_wbf_ensemble = True\nif len(model_list) == 1:\n    use_wbf_ensemble = False\n\nframe_id = 0\nfor idx, (path, anns) in enumerate(zip(image_paths, image_anns)):\n    img = cv2.imread(path)[...,::-1]\n    if FDA_aug:\n        img = FDA_trans(image=img)['image']\n        \n    img_list, boxes_list, scores_list, labels_list = [], [], [], []\n    gt_bboxes = np.array([[ann['x'], ann['y'], ann['width'], ann['height']] for ann in anns])\n    \n    for model_info in model_list:\n        bboxes, confis = get_output_from_prediction(model_info, img)\n        bboxes = coco2albumentations(bboxes, ORI_SIZE[1], ORI_SIZE[0])\n        \n        bboxes = list(map(list, bboxes))\n        boxes_list.append(bboxes)\n        scores_list.append(confis)\n        labels_list.append(np.zeros_like(confis, dtype=np.uint8).tolist())\n    \n    if use_wbf_ensemble:\n        bboxes, confis, _ = weighted_boxes_fusion(boxes_list, scores_list, labels_list, \n                                                  iou_thr=IOU_WBF_ENSEMBLE, skip_box_thr=CONF_WBF_ENSEMBLE)\n    else:\n        bboxes = boxes_list[0]\n        confis = scores_list[0]\n        \n    bboxes = albumentations2coco(bboxes, ORI_SIZE[1], ORI_SIZE[0]).astype(int)\n#     display(show_img(img, bboxes=bboxes, bbox_format='coco'))\n\n    predict_box = tracking_function(tracker, frame_id, bboxes, confis)\n\n    if len(predict_box)>0:\n        box = [list(map(int,box.split(' ')[1:])) for box in predict_box]\n    else:\n        box = []\n    print(confis)\n\n    display(show_img(img, box, bbox_format='coco'))\n    display(show_img(img, bboxes, bbox_format='coco'))  # Predict\n    display(show_img(img, gt_bboxes, bbox_format='coco'))\n    print('\\n\\n')\n    if idx>1:\n        break\n    frame_id += 1","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-13T18:49:23.004062Z","iopub.execute_input":"2022-02-13T18:49:23.004434Z","iopub.status.idle":"2022-02-13T18:49:36.385497Z","shell.execute_reply.started":"2022-02-13T18:49:23.004387Z","shell.execute_reply":"2022-02-13T18:49:36.384848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Init `Env`","metadata":{}},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()# initialize the environment\niter_test = env.iter_test()      # an iterator which loops over the test set and sample submission","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-13T18:49:36.386635Z","iopub.execute_input":"2022-02-13T18:49:36.387492Z","iopub.status.idle":"2022-02-13T18:49:36.410151Z","shell.execute_reply.started":"2022-02-13T18:49:36.387421Z","shell.execute_reply":"2022-02-13T18:49:36.409497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Run Inference on **Test**","metadata":{}},{"cell_type":"code","source":"!ls /kaggle/working","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:49:36.411575Z","iopub.execute_input":"2022-02-13T18:49:36.411837Z","iopub.status.idle":"2022-02-13T18:49:37.106182Z","shell.execute_reply.started":"2022-02-13T18:49:36.411802Z","shell.execute_reply":"2022-02-13T18:49:37.105324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRACKING = True","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:49:37.109373Z","iopub.execute_input":"2022-02-13T18:49:37.109613Z","iopub.status.idle":"2022-02-13T18:49:37.114264Z","shell.execute_reply.started":"2022-02-13T18:49:37.109585Z","shell.execute_reply":"2022-02-13T18:49:37.113485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t1 = time.time()\n\nif TRACKING:\n    tracker = Tracker(\n    distance_function=euclidean_distance, \n    distance_threshold=30,\n    hit_inertia_min=3,\n    hit_inertia_max=6,\n    initialization_delay=1,\n)\n    model_list = [model4, model1]\n\n    use_wbf_ensemble = True\n    if len(model_list) == 1:\n        use_wbf_ensemble = False\n\n    frame_id =0\n    for idx, (img, pred_df) in enumerate(tqdm(iter_test)):\n        if FDA_aug:\n            img = FDA_trans(image=img)['image']\n        img_list, boxes_list, scores_list, labels_list = [], [], [], []\n\n        for model_info in model_list:\n            bboxes, confis = get_output_from_prediction(model_info, img)\n            if len(bboxes):\n                bboxes = coco2albumentations(bboxes, ORI_SIZE[1], ORI_SIZE[0])\n            else:\n                bboxes = []\n\n            bboxes = list(map(list, bboxes))\n            boxes_list.append(bboxes)\n            scores_list.append(confis)\n            labels_list.append(np.zeros_like(confis, dtype=np.uint8).tolist())\n\n        if use_wbf_ensemble:\n            bboxes, confis, _ = weighted_boxes_fusion(boxes_list, scores_list, labels_list, \n                                                      iou_thr=IOU_WBF_ENSEMBLE, skip_box_thr=CONF_WBF_ENSEMBLE)\n        else:\n            bboxes = boxes_list[0]\n            confis = scores_list[0]\n\n        bboxes, confis, _ = weighted_boxes_fusion(boxes_list, scores_list, labels_list, \n                                                  iou_thr=IOU_WBF_ENSEMBLE, skip_box_thr=CONF_WBF_ENSEMBLE)\n        if len(bboxes):\n            bboxes = albumentations2coco(bboxes, ORI_SIZE[1], ORI_SIZE[0]).astype(int)\n        else:\n            bboxes = []\n\n        predictions = tracking_function(tracker, frame_id, bboxes, confis)\n\n        prediction_str = ' '.join(predictions)\n        pred_df['annotations'] = prediction_str\n        env.predict(pred_df)\n        if frame_id < 3:\n            if len(predictions)>0:\n                box = [list(map(int,box.split(' ')[1:])) for box in predictions]\n            else:\n                box = []\n            display(show_img(img, box, bbox_format='coco'))\n    #     print('Prediction:', pred_df)\n        frame_id += 1\n\nelse:\n    model = load_model(CKPT_PATH, conf=CONF_MODEL, iou=IOU_MODEL)\n    for idx, (img, pred_df) in enumerate(tqdm(iter_test)):\n        bboxes, confs  = predict(model, img, size=IMG_SIZE, augment=AUGMENT)\n        annot          = format_prediction(bboxes, confs)\n        pred_df['annotations'] = annot\n        env.predict(pred_df)\n        if idx<3:\n            display(show_img(img, bboxes, bbox_format='coco'))\n            \nprint(f'{time.time() - t1}s')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-13T18:49:37.119187Z","iopub.execute_input":"2022-02-13T18:49:37.119697Z","iopub.status.idle":"2022-02-13T18:49:44.670067Z","shell.execute_reply.started":"2022-02-13T18:49:37.119663Z","shell.execute_reply":"2022-02-13T18:49:44.669440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 👀 Check Submission","metadata":{}},{"cell_type":"code","source":"# sub_df = pd.read_csv('submission.csv')\n# sub_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-13T18:49:44.671172Z","iopub.execute_input":"2022-02-13T18:49:44.672757Z","iopub.status.idle":"2022-02-13T18:49:44.676568Z","shell.execute_reply.started":"2022-02-13T18:49:44.672718Z","shell.execute_reply":"2022-02-13T18:49:44.675615Z"},"trusted":true},"execution_count":null,"outputs":[]}]}