{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Reference - https://www.kaggle.com/yamqwe/great-barrier-reef-yolox-yolov5-ensemble","metadata":{}},{"cell_type":"code","source":"import warnings; warnings.filterwarnings(\"ignore\")\n\nimport os\nimport cv2\nimport ast\nimport sys\nimport glob\nimport torch\nimport shutil\nimport importlib\nimport traceback\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nsys.path.append('../input/tensorflow-great-barrier-reef')\ntqdm.pandas()\n\nfrom PIL import Image\nfrom IPython.display import display\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\n","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:12:16.162245Z","iopub.execute_input":"2022-02-10T20:12:16.162595Z","iopub.status.idle":"2022-02-10T20:12:17.867456Z","shell.execute_reply.started":"2022-02-10T20:12:16.162500Z","shell.execute_reply":"2022-02-10T20:12:17.866744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Install YoloX","metadata":{}},{"cell_type":"code","source":"%cp -r ../input/yolox-cots-models /kaggle/working/\n%cd /kaggle/working/yolox-cots-models/yolox-dep","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:12:17.869887Z","iopub.execute_input":"2022-02-10T20:12:17.870341Z","iopub.status.idle":"2022-02-10T20:12:35.423586Z","shell.execute_reply.started":"2022-02-10T20:12:17.870302Z","shell.execute_reply":"2022-02-10T20:12:35.422796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install YOLOX required modules\n!pip install pip-21.3.1-py3-none-any.whl -f ./ --no-index\n!pip install loguru-0.5.3-py3-none-any.whl -f ./ --no-index\n!pip install ninja-1.10.2.3-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl -f ./ --no-index\n!pip install onnx-1.8.1-cp37-cp37m-manylinux2010_x86_64.whl -f ./ --no-index\n!pip install onnxruntime-1.8.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -f ./ --no-index\n!pip install onnxoptimizer-0.2.6-cp37-cp37m-manylinux2014_x86_64.whl -f ./ --no-index\n!pip install thop-0.0.31.post2005241907-py3-none-any.whl -f ./ --no-index\n!pip install tabulate-0.8.9-py3-none-any.whl -f ./ --no-index","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:12:35.425252Z","iopub.execute_input":"2022-02-10T20:12:35.425797Z","iopub.status.idle":"2022-02-10T20:13:40.046461Z","shell.execute_reply.started":"2022-02-10T20:12:35.425758Z","shell.execute_reply":"2022-02-10T20:13:40.045460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install YOLOX\n%cd /kaggle/working/yolox-cots-models/YOLOX\n!pip install -r requirements.txt\n!pip install -v -e . ","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T20:13:40.049338Z","iopub.execute_input":"2022-02-10T20:13:40.049668Z","iopub.status.idle":"2022-02-10T20:14:57.858997Z","shell.execute_reply.started":"2022-02-10T20:13:40.049625Z","shell.execute_reply":"2022-02-10T20:14:57.857967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install CocoAPI tool\n%cd /kaggle/working/yolox-cots-models/yolox-dep/cocoapi/PythonAPI\n!make\n!make install\n!python setup.py install","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T20:14:57.860730Z","iopub.execute_input":"2022-02-10T20:14:57.860998Z","iopub.status.idle":"2022-02-10T20:15:21.889262Z","shell.execute_reply.started":"2022-02-10T20:14:57.860965Z","shell.execute_reply":"2022-02-10T20:15:21.888331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pycocotools","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:15:21.892423Z","iopub.execute_input":"2022-02-10T20:15:21.892645Z","iopub.status.idle":"2022-02-10T20:15:21.897948Z","shell.execute_reply.started":"2022-02-10T20:15:21.892615Z","shell.execute_reply":"2022-02-10T20:15:21.897159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# norfair dependencies\n%cd /kaggle/input/norfair031py3/\n!pip install commonmark-0.9.1-py2.py3-none-any.whl -f ./ --no-index\n!pip install rich-9.13.0-py3-none-any.whl\n\n!mkdir /kaggle/working/tmp\n!cp -r /kaggle/input/norfair031py3/filterpy-1.4.5/filterpy-1.4.5/ /kaggle/working/tmp/\n%cd /kaggle/working/tmp/filterpy-1.4.5/\n!pip install .\n!rm -rf /kaggle/working/tmp\n\n# norfair\n%cd /kaggle/input/norfair031py3/\n!pip install norfair-0.3.1-py3-none-any.whl -f ./ --no-index","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:15:21.899437Z","iopub.execute_input":"2022-02-10T20:15:21.899931Z","iopub.status.idle":"2022-02-10T20:16:36.409310Z","shell.execute_reply.started":"2022-02-10T20:15:21.899896Z","shell.execute_reply":"2022-02-10T20:16:36.408299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## YoloX","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working/yolox-cots-models/YOLOX\n\n# CHECKPOINT_FILE = '/kaggle/input/cotsyoloxse40ap2534stratifiedk/YOLOX_outputs/cots_config/best_ckpt.pth'\nCHECKPOINT_FILE = '/kaggle/working/yolox-cots-models/yx_l_003.pth'","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:36.411911Z","iopub.execute_input":"2022-02-10T20:16:36.412689Z","iopub.status.idle":"2022-02-10T20:16:36.419460Z","shell.execute_reply.started":"2022-02-10T20:16:36.412641Z","shell.execute_reply":"2022-02-10T20:16:36.418741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config_file_template = '''\n\n#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n    def __init__(self):\n        super(Exp, self).__init__()\n        self.depth = 1\n        self.width = 1\n        self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n        self.num_classes = 1\n\n'''\n\nwith open('cots_config.py', 'w') as f:\n    f.write(config_file_template)","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:36.420506Z","iopub.execute_input":"2022-02-10T20:16:36.421440Z","iopub.status.idle":"2022-02-10T20:16:36.428361Z","shell.execute_reply.started":"2022-02-10T20:16:36.421400Z","shell.execute_reply":"2022-02-10T20:16:36.427626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from yolox.utils import postprocess\nfrom yolox.data.data_augment import ValTransform\n\nCOCO_CLASSES = (\n  \"starfish\",\n)\n\n# get YOLOX experiment\ncurrent_exp = importlib.import_module('cots_config')\nexp = current_exp.Exp()\n\n# set inference parameters\ntest_size = (800, 1280)\nnum_classes = 1\nconfthre = 0.2\nnmsthre = 0.2\n\n\n# get YOLOX model\nyolox_model = exp.get_model()\nyolox_model.cuda()\nyolox_model.eval()\n\n# get custom trained checkpoint\nckpt_file = CHECKPOINT_FILE\nckpt = torch.load(ckpt_file, map_location=\"cpu\")\nyolox_model.load_state_dict(ckpt[\"model\"])","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-02-10T20:16:36.432186Z","iopub.execute_input":"2022-02-10T20:16:36.432769Z","iopub.status.idle":"2022-02-10T20:16:40.580781Z","shell.execute_reply.started":"2022-02-10T20:16:36.432730Z","shell.execute_reply":"2022-02-10T20:16:40.578749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def yolox_inference(img, model, test_size): \n    bboxes = []\n    bbclasses = []\n    scores = []\n    \n    preproc = ValTransform(legacy = False)\n\n    tensor_img, _ = preproc(img, None, test_size)\n    tensor_img = torch.from_numpy(tensor_img).unsqueeze(0)\n    tensor_img = tensor_img.float()\n    tensor_img = tensor_img.cuda()\n\n    with torch.no_grad():\n        outputs = model(tensor_img)\n        outputs = postprocess(\n                    outputs, num_classes, confthre,\n                    nmsthre, class_agnostic=True\n                )\n\n    if outputs[0] is None:\n        return [], [], []\n    \n    outputs = outputs[0].cpu()\n    bboxes = outputs[:, 0:4]\n\n    bboxes /= min(test_size[0] / img.shape[0], test_size[1] / img.shape[1])\n    bbclasses = outputs[:, 6]\n    scores = outputs[:, 4] * outputs[:, 5]\n    \n    return bboxes, bbclasses, scores","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:40.585132Z","iopub.execute_input":"2022-02-10T20:16:40.587034Z","iopub.status.idle":"2022-02-10T20:16:40.598783Z","shell.execute_reply.started":"2022-02-10T20:16:40.586993Z","shell.execute_reply":"2022-02-10T20:16:40.597813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_yolox_predictions(img, bboxes, scores, bbclasses, confthre, classes_dict):\n    for i in range(len(bboxes)):\n            box = bboxes[i]\n            cls_id = int(bbclasses[i])\n            score = scores[i]\n            if score < confthre:\n                continue\n            x0 = int(box[0])\n            y0 = int(box[1])\n            x1 = int(box[2])\n            y1 = int(box[3])\n\n            cv2.rectangle(img, (x0, y0), (x1, y1), (0, 255, 0), 2)\n            cv2.putText(img, '{}:{:.1f}%'.format(classes_dict[cls_id], score * 100), (x0, y0 - 3), cv2.FONT_HERSHEY_PLAIN, 0.8, (0,255,0), thickness = 1)\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:40.602631Z","iopub.execute_input":"2022-02-10T20:16:40.604519Z","iopub.status.idle":"2022-02-10T20:16:40.615755Z","shell.execute_reply.started":"2022-02-10T20:16:40.604479Z","shell.execute_reply":"2022-02-10T20:16:40.614748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## YoloV5","metadata":{}},{"cell_type":"code","source":"ROOT_DIR  = '/kaggle/input/tensorflow-great-barrier-reef/'\nCKPT_PATH = '/kaggle/input/yolov5s6/f2_sub2.pt'\nIMG_SIZE  = 6400\nCONF      = 0.4\nIOU       = 0.5\nAUGMENT   = False","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:40.617513Z","iopub.execute_input":"2022-02-10T20:16:40.618076Z","iopub.status.idle":"2022-02-10T20:16:40.626656Z","shell.execute_reply.started":"2022-02-10T20:16:40.618024Z","shell.execute_reply":"2022-02-10T20:16:40.625751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_path(row):\n    row['image_path'] = f'{ROOT_DIR}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    return row","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:40.627995Z","iopub.execute_input":"2022-02-10T20:16:40.628240Z","iopub.status.idle":"2022-02-10T20:16:40.638088Z","shell.execute_reply.started":"2022-02-10T20:16:40.628207Z","shell.execute_reply":"2022-02-10T20:16:40.637275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Data\ndf = pd.read_csv(f'{ROOT_DIR}/train.csv')\ndf = df.progress_apply(get_path, axis=1)\ndf['annotations'] = df['annotations'].progress_apply(lambda x: ast.literal_eval(x))\ndf['num_bbox'] = df['annotations'].progress_apply(lambda x: len(x))\ndata = (df.num_bbox>0).value_counts()/len(df)*100","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:40.639485Z","iopub.execute_input":"2022-02-10T20:16:40.640325Z","iopub.status.idle":"2022-02-10T20:16:56.514370Z","shell.execute_reply.started":"2022-02-10T20:16:40.640283Z","shell.execute_reply":"2022-02-10T20:16:56.513553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def voc2yolo(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    voc  => [x1, y1, x2, y1]\n    yolo => [xmid, ymid, w, h] (normalized)\n    \"\"\"\n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]/ image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]/ image_height\n    w = bboxes[..., 2] - bboxes[..., 0]\n    h = bboxes[..., 3] - bboxes[..., 1]\n    bboxes[..., 0] = bboxes[..., 0] + w/2\n    bboxes[..., 1] = bboxes[..., 1] + h/2\n    bboxes[..., 2] = w\n    bboxes[..., 3] = h\n    return bboxes\n\ndef yolo2voc(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    voc  => [x1, y1, x2, y1]\n\n    \"\"\"\n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]\n    return bboxes\n\ndef coco2yolo(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    coco => [xmin, ymin, w, h]\n    yolo => [xmid, ymid, w, h] (normalized)\n    \"\"\"\n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    # normolizinig\n    bboxes[..., [0, 2]]= bboxes[..., [0, 2]]/ image_width\n    bboxes[..., [1, 3]]= bboxes[..., [1, 3]]/ image_height\n    # converstion (xmin, ymin) => (xmid, ymid)\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]/2\n    return bboxes\n\ndef yolo2coco(bboxes, image_height=720, image_width=1280):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    coco => [xmin, ymin, w, h]\n    \"\"\"\n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    # denormalizing\n    bboxes[..., [0, 2]]= bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]]= bboxes[..., [1, 3]]* image_height\n    # converstion (xmid, ymid) => (xmin, ymin)\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    return bboxes\n\ndef voc2coco(bboxes, image_height=720, image_width=1280):\n    bboxes  = voc2yolo(bboxes, image_height, image_width)\n    bboxes  = yolo2coco(bboxes, image_height, image_width)\n    return bboxes\n\ndef load_image(image_path):\n    return cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n\ndef plot_one_box(x, img, color=None, label=None, line_thickness=None):\n    # Plots one bounding box on image img\n    tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1  # line/font thickness\n    color = color or [random.randint(0, 255) for _ in range(3)]\n    c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\n    cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)\n    if label:\n        tf = max(tl - 1, 1)  # font thickness\n        t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\n        c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\n        cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA)  # filled\n        cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)\n\ndef draw_bboxes(img, bboxes, classes, class_ids, colors = None, show_classes = None, bbox_format = 'yolo', class_name = False, line_thickness = 2):\n\n    image = img.copy()\n    show_classes = classes if show_classes is None else show_classes\n    colors = (0, 255 ,0) if colors is None else colors\n\n    if bbox_format == 'yolo':\n\n        for idx in range(len(bboxes)):\n\n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n\n            if cls in show_classes:\n\n                x1 = round(float(bbox[0])*image.shape[1])\n                y1 = round(float(bbox[1])*image.shape[0])\n                w  = round(float(bbox[2])*image.shape[1]/2) #w/2\n                h  = round(float(bbox[3])*image.shape[0]/2)\n\n                voc_bbox = (x1-w, y1-h, x1+w, y1+h)\n                plot_one_box(voc_bbox,\n                             image,\n                             color = color,\n                             label = cls if class_name else str(get_label(cls)),\n                             line_thickness = line_thickness)\n\n    elif bbox_format == 'coco':\n\n        for idx in range(len(bboxes)):\n\n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n\n            if cls in show_classes:\n                x1 = int(round(bbox[0]))\n                y1 = int(round(bbox[1]))\n                w  = int(round(bbox[2]))\n                h  = int(round(bbox[3]))\n\n                voc_bbox = (x1, y1, x1+w, y1+h)\n                plot_one_box(voc_bbox,\n                             image,\n                             color = color,\n                             label = cls if class_name else str(cls_id),\n                             line_thickness = line_thickness)\n\n    elif bbox_format == 'voc_pascal':\n\n        for idx in range(len(bboxes)):\n\n            bbox  = bboxes[idx]\n            cls   = classes[idx]\n            cls_id = class_ids[idx]\n            color = colors[cls_id] if type(colors) is list else colors\n\n            if cls in show_classes:\n                x1 = int(round(bbox[0]))\n                y1 = int(round(bbox[1]))\n                x2 = int(round(bbox[2]))\n                y2 = int(round(bbox[3]))\n                voc_bbox = (x1, y1, x2, y2)\n                plot_one_box(voc_bbox,\n                             image,\n                             color = color,\n                             label = cls if class_name else str(cls_id),\n                             line_thickness = line_thickness)\n    else:\n        raise ValueError('wrong bbox format')\n\n    return image\n\ndef get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndef get_imgsize(row):\n    row['width'], row['height'] = imagesize.get(row['image_path'])\n    return row\n\nnp.random.seed(32)\ncolors = [(np.random.randint(255), np.random.randint(255), np.random.randint(255))\\\n          for idx in range(1)]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T20:16:56.515842Z","iopub.execute_input":"2022-02-10T20:16:56.516271Z","iopub.status.idle":"2022-02-10T20:16:56.554146Z","shell.execute_reply.started":"2022-02-10T20:16:56.516229Z","shell.execute_reply":"2022-02-10T20:16:56.553275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##############################################################\n#                      Tracking helpers                      #\n##############################################################\n\nimport numpy as np\nfrom norfair import Detection, Tracker\n\n# Helper to convert bbox in format [x_min, y_min, x_max, y_max, score] to norfair.Detection class\ndef to_norfair(detects, frame_id):\n    result = []\n    for x_min, y_min, x_max, y_max, score in detects:\n        xc, yc = (x_min + x_max) / 2, (y_min + y_max) / 2\n        w, h = x_max - x_min, y_max - y_min\n        result.append(Detection(points=np.array([xc, yc]), scores=np.array([score]), data=np.array([w, h, frame_id])))\n        \n    return result\n\n# Euclidean distance function to match detections on this frame with tracked_objects from previous frames\ndef euclidean_distance(detection, tracked_object):\n    return np.linalg.norm(detection.points - tracked_object.estimate)","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:56.555447Z","iopub.execute_input":"2022-02-10T20:16:56.555726Z","iopub.status.idle":"2022-02-10T20:16:57.238666Z","shell.execute_reply.started":"2022-02-10T20:16:56.555678Z","shell.execute_reply":"2022-02-10T20:16:57.237875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:57.240102Z","iopub.execute_input":"2022-02-10T20:16:57.240348Z","iopub.status.idle":"2022-02-10T20:16:57.573520Z","shell.execute_reply.started":"2022-02-10T20:16:57.240313Z","shell.execute_reply":"2022-02-10T20:16:57.572582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(ckpt_path, conf=0.15, iou=0.20):\n    model = torch.hub.load('/kaggle/input/yolov5-lib-ds',\n                           'custom',\n                           path=ckpt_path,\n                           source='local',\n                           force_reload=True)  # local repo\n    model.conf = conf  # NMS confidence threshold\n    model.iou  = iou  # NMS IoU threshold\n    model.classes = None   # (optional list) filter by class, i.e. = [0, 15, 16] for persons, cats and dogs\n    model.multi_label = False  # NMS multiple labels per box\n    model.max_det = 1000  # maximum number of detections per image\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:57.575159Z","iopub.execute_input":"2022-02-10T20:16:57.575435Z","iopub.status.idle":"2022-02-10T20:16:57.582182Z","shell.execute_reply.started":"2022-02-10T20:16:57.575397Z","shell.execute_reply":"2022-02-10T20:16:57.581080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, img, size=768, augment=False):\n    height, width = img.shape[:2]\n    results = model(img, size=size, augment=augment)  # custom inference size\n    preds   = results.pandas().xyxy[0]\n    bboxes  = preds[['xmin','ymin','xmax','ymax']].values\n    if len(bboxes):\n        bboxes  = voc2coco(bboxes,height,width).astype(int)\n        confs   = preds.confidence.values\n        return bboxes, confs\n    else:\n        return [],[]\n\ndef format_prediction(bboxes, confs):\n    annot = ''\n    if len(bboxes)>0:\n        for idx in range(len(bboxes)):\n            xmin, ymin, w, h = bboxes[idx]\n            conf             = confs[idx]\n            annot += f'{conf} {xmin} {ymin} {w} {h}'\n            annot +=' '\n        annot = annot.strip(' ')\n    return annot\n\ndef show_img(img, bboxes, bbox_format='yolo', bbox_colors = None):\n    names  = ['starfish']*len(bboxes)\n    labels = [0]*len(bboxes)\n    img    = draw_bboxes(img = img,\n                           bboxes = bboxes,\n                           classes = names,\n                           class_ids = labels,\n                           class_name = True,\n                           colors = colors if bbox_colors is None else bbox_colors,\n                           bbox_format = bbox_format,\n                           line_thickness = 2)\n    return Image.fromarray(img).resize((800, 400))","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:57.583469Z","iopub.execute_input":"2022-02-10T20:16:57.584343Z","iopub.status.idle":"2022-02-10T20:16:57.597280Z","shell.execute_reply.started":"2022-02-10T20:16:57.584305Z","shell.execute_reply":"2022-02-10T20:16:57.596555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tracking_function(tracker, frame_id, bboxes, scores):\n    \n    detects = []\n    predictions = []\n    \n    if len(scores)>0:\n        for i in range(len(bboxes)):\n            box = bboxes[i]\n            score = scores[i]\n            x_min = int(box[0])\n            y_min = int(box[1])\n            bbox_width = int(box[2])\n            bbox_height = int(box[3])\n            detects.append([x_min, y_min, x_min+bbox_width, y_min+bbox_height, score])\n            predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n#             print(predictions[:-1])\n    # Update tracks using detects from current frame\n    tracked_objects = tracker.update(detections=to_norfair(detects, frame_id))\n    for tobj in tracked_objects:\n        bbox_width, bbox_height, last_detected_frame_id = tobj.last_detection.data\n        if last_detected_frame_id == frame_id:  # Skip objects that were detected on current frame\n            continue\n#         if last_detected_frame_id + 1 != frame_id:  # Track only one frame forward\n#             continue\n        # Add objects that have no detections on current frame to predictions\n        xc, yc = tobj.estimate[0]\n        x_min, y_min = int(round(xc - bbox_width / 2)), int(round(yc - bbox_height / 2))\n        score = tobj.last_detection.scores[0]\n\n        predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n        \n    return predictions\n\n","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:57.598538Z","iopub.execute_input":"2022-02-10T20:16:57.599266Z","iopub.status.idle":"2022-02-10T20:16:57.610059Z","shell.execute_reply.started":"2022-02-10T20:16:57.599209Z","shell.execute_reply":"2022-02-10T20:16:57.609016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def CLAHE(image):\n    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n    equalized = clahe.apply(gray)\n    return equalized\ndef Gamma_enhancement(image):\n    gamma = 1/0.6\n    R = 255.0\n    return (R * np.power(image.astype(np.uint32)/R, gamma)).astype(np.uint8)","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:57.611464Z","iopub.execute_input":"2022-02-10T20:16:57.612016Z","iopub.status.idle":"2022-02-10T20:16:57.620965Z","shell.execute_reply.started":"2022-02-10T20:16:57.611978Z","shell.execute_reply":"2022-02-10T20:16:57.620163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensembling + Tracking on Train","metadata":{}},{"cell_type":"code","source":"import sys; sys.path.append('/kaggle/input/weightedboxesfusion/')","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:57.622747Z","iopub.execute_input":"2022-02-10T20:16:57.623007Z","iopub.status.idle":"2022-02-10T20:16:57.629214Z","shell.execute_reply.started":"2022-02-10T20:16:57.622973Z","shell.execute_reply":"2022-02-10T20:16:57.628425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = load_model(CKPT_PATH, conf=CONF, iou=IOU)","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:16:57.630560Z","iopub.execute_input":"2022-02-10T20:16:57.631327Z","iopub.status.idle":"2022-02-10T20:17:04.451674Z","shell.execute_reply.started":"2022-02-10T20:16:57.631291Z","shell.execute_reply":"2022-02-10T20:17:04.450886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_wbf(bboxes, confs, image_size=512, iou_thr=0.2, skip_box_thr=0.001, weights=None):\n    boxes =  [bbox/(image_size-1) for bbox in bboxes]\n    scores = [conf for conf in confs]    \n    labels = [np.ones(conf.shape[0]) for conf in confs]\n    boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    boxes = boxes*(image_size-1)\n    return boxes, scores, labels\n\nfrom ensemble_boxes import *","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:17:04.453443Z","iopub.execute_input":"2022-02-10T20:17:04.453754Z","iopub.status.idle":"2022-02-10T20:17:04.485855Z","shell.execute_reply.started":"2022-02-10T20:17:04.453710Z","shell.execute_reply":"2022-02-10T20:17:04.485197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tracker1 = Tracker(\n    distance_function=euclidean_distance, \n    distance_threshold=30,\n    hit_inertia_min=3,\n    hit_inertia_max=6,\n    initialization_delay=1,\n)\ntracker2 = Tracker(\n    distance_function=euclidean_distance, \n    distance_threshold=30,\n    hit_inertia_min=3,\n    hit_inertia_max=6,\n    initialization_delay=1,\n)\ntracker3 = Tracker(\n    distance_function=euclidean_distance, \n    distance_threshold=30,\n    hit_inertia_min=3,\n    hit_inertia_max=6,\n    initialization_delay=1,\n)","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:17:04.487076Z","iopub.execute_input":"2022-02-10T20:17:04.487480Z","iopub.status.idle":"2022-02-10T20:17:04.494060Z","shell.execute_reply.started":"2022-02-10T20:17:04.487442Z","shell.execute_reply":"2022-02-10T20:17:04.493318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# frame_id_1 = 0\n# frame_id_2 = 0 \n# frame_id_3 = 0\n\n# image_paths = df[df.num_bbox>1].sample(100).image_path.tolist()\n# for idx, path in enumerate(image_paths):\n#     img = cv2.imread(path)[...,::-1]\n#     conf1 = []\n#     conf2 = []\n#     tmp1 = []\n#     tmp2 = []\n#     bboxes_1, bbclasses, scores = yolox_inference(img[...,::-1], yolox_model, test_size)  \n#     bboxes_1 = voc2coco(bboxes_1.detach().numpy(),img.shape[1],img.shape[2]).astype(int) #change to coco\n# #     scores = scores.detach().numpy().reshape((1,len(scores)))[0]\n    \n#     bboxes_2, confis = predict(model, img, size=IMG_SIZE, augment=AUGMENT)\n# #     print(bboxes_1)\n    \n#     pred_1 =  tracking_function(tracker1, frame_id_1, bboxes_1, scores)\n#     pred_2 =  tracking_function(tracker2, frame_id_2, bboxes_2, confis)\n# #     print(pred_1)\n# #     print(pred_2)\n    \n#     box1 = [list(map(int,box.split(' ')[1:])) for box in pred_1]\n#     box2 = [list(map(int,box.split(' ')[1:])) for box in pred_2]\n\n    \n#     for box in pred_1:\n#         conf1.append(float(box.split(' ')[0]))\n#     conf1 = np.asarray(conf1)\n#     for box in pred_2:\n#         conf2.append(float(box.split(' ')[0]))\n#     conf2 = np.asarray(conf2)\n# #     print(conf1)\n# #     print(conf2)\n\n#     for i in box1:\n#         a = []\n#         for j in i :\n#             a.append(j)\n#         c = np.asarray(a)\n#         tmp1.append(c)\n#     box1 = np.asarray(tmp1)\n    \n#     for i in box2:\n#         a = []\n#         for j in i :\n#             a.append(j)\n#         c = np.asarray(a)\n#         tmp2.append(c)\n#     box2 = np.asarray(tmp2)\n# #     print(box1)\n# #     print(box2)\n\n#     boxes, scores, labels = run_wbf([box1,box2], [conf1,conf2], image_size = IMG_SIZE)\n# #     print(boxes)\n#     predictions = tracking_function(tracker3, frame_id_3, boxes, scores)\n# #     print(predictions)\n    \n#     print('\\nEnsemble (WBF) Predictions: ')\n#     display(show_img(img, boxes, bbox_format='coco'))\n    \n#     if frame_id_1 < 7:\n#         if len(predictions)>0:\n#             box = [list(map(int,box.split(' ')[1:])) for box in predictions]\n#         else:\n#             box = []\n#         print('\\n Tracking')\n#         display(show_img(img, box, bbox_format='coco'))\n#     print('\\n----------------------------------------\\n')\n#     frame_id_1 += 1   \n#     frame_id_2 += 1\n#     frame_id_3 += 1\n    \n#     if idx>5:\n#         break","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:17:04.495537Z","iopub.execute_input":"2022-02-10T20:17:04.496033Z","iopub.status.idle":"2022-02-10T20:17:04.503471Z","shell.execute_reply.started":"2022-02-10T20:17:04.495996Z","shell.execute_reply":"2022-02-10T20:17:04.502663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame_id_1 = 0\nframe_id_2 = 0 \nframe_id_3 = 0\n\nimage_paths = df[df.num_bbox>1].sample(100).image_path.tolist()\nfor idx, path in enumerate(image_paths):\n    img = cv2.imread(path)[...,::-1]\n    conf1 = []\n    conf2 = []\n    tmp1 = []\n    tmp2 = []\n#     bboxes_1, bbclasses, scores = yolox_inference(img[...,::-1], yolox_model, test_size)  \n#     bboxes_1 = voc2coco(bboxes_1.detach().numpy(),img.shape[1],img.shape[2]).astype(int) #change to coco\n# #     scores = scores.detach().numpy().reshape((1,len(scores)))[0]\n    bboxes_1, confis1 = predict(model, img, size=IMG_SIZE, augment=AUGMENT)\n    bboxes_2, confis2 = predict(model, img, size=IMG_SIZE, augment=AUGMENT)\n#     print(bboxes_1)\n    \n    pred_1 =  tracking_function(tracker1, frame_id_1, bboxes_1, confis1)\n    pred_2 =  tracking_function(tracker2, frame_id_2, bboxes_2, confis2)\n#     print(pred_1)\n#     print(pred_2)\n    \n    box1 = [list(map(int,box.split(' ')[1:])) for box in pred_1]\n    box2 = [list(map(int,box.split(' ')[1:])) for box in pred_2]\n\n    \n    for box in pred_1:\n        conf1.append(float(box.split(' ')[0]))\n    conf1 = np.asarray(conf1)\n    for box in pred_2:\n        conf2.append(float(box.split(' ')[0]))\n    conf2 = np.asarray(conf2)\n#     print(conf1)\n#     print(conf2)\n\n    for i in box1:\n        a = []\n        for j in i :\n            a.append(j)\n        c = np.asarray(a)\n        tmp1.append(c)\n    box1 = np.asarray(tmp1)\n    \n    for i in box2:\n        a = []\n        for j in i :\n            a.append(j)\n        c = np.asarray(a)\n        tmp2.append(c)\n    box2 = np.asarray(tmp2)\n#     print(box1)\n#     print(box2)\n\n    boxes, scores, labels = run_wbf([box1,box2], [conf1,conf2], image_size = IMG_SIZE)\n#     print(boxes)\n    predictions = tracking_function(tracker3, frame_id_3, boxes, scores)\n#     print(predictions)\n    \n    print('\\nEnsemble (WBF) Predictions: ')\n    display(show_img(img, boxes, bbox_format='coco'))\n    \n    if frame_id_1 < 7:\n        if len(predictions)>0:\n            box = [list(map(int,box.split(' ')[1:])) for box in predictions]\n        else:\n            box = []\n        print('\\n Tracking')\n        display(show_img(img, box, bbox_format='coco'))\n    print('\\n----------------------------------------\\n')\n    frame_id_1 += 1   \n    frame_id_2 += 1\n    frame_id_3 += 1\n    \n    if idx>5:\n        break","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:17:04.508415Z","iopub.execute_input":"2022-02-10T20:17:04.508904Z","iopub.status.idle":"2022-02-10T20:17:11.611249Z","shell.execute_reply.started":"2022-02-10T20:17:04.508875Z","shell.execute_reply":"2022-02-10T20:17:11.610639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submit Prediction","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working/\nimport greatbarrierreef\n\nenv = greatbarrierreef.make_env()   # initialize the environment\niter_test = env.iter_test()  ","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:17:11.612332Z","iopub.execute_input":"2022-02-10T20:17:11.612783Z","iopub.status.idle":"2022-02-10T20:17:11.669847Z","shell.execute_reply.started":"2022-02-10T20:17:11.612746Z","shell.execute_reply":"2022-02-10T20:17:11.669171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tracker1 = Tracker(\n    distance_function=euclidean_distance, \n    distance_threshold=30,\n    hit_inertia_min=3,\n    hit_inertia_max=6,\n    initialization_delay=1,\n)\ntracker2 = Tracker(\n    distance_function=euclidean_distance, \n    distance_threshold=30,\n    hit_inertia_min=3,\n    hit_inertia_max=6,\n    initialization_delay=1,\n)\ntracker3 = Tracker(\n    distance_function=euclidean_distance, \n    distance_threshold=30,\n    hit_inertia_min=3,\n    hit_inertia_max=6,\n    initialization_delay=1,\n)","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:17:11.671236Z","iopub.execute_input":"2022-02-10T20:17:11.671723Z","iopub.status.idle":"2022-02-10T20:17:11.678213Z","shell.execute_reply.started":"2022-02-10T20:17:11.671662Z","shell.execute_reply":"2022-02-10T20:17:11.677391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_dict = {\n    'id': [],\n    'prediction_string': [],\n}\nframe_id_1 = 0\nframe_id_2 = 0 \nframe_id_3 = 0\nfor (image_np, pred_df) in iter_test:\n    \n    conf1 = []\n    conf2 = []\n    tmp1 = []\n    tmp2 = []   \n    \n#     bboxes_1, bbclasses, scores = yolox_inference(image_np[...,::-1], yolox_model, test_size)\n    bboxes_1, confis1 = predict(model, image_np, size=IMG_SIZE, augment=AUGMENT)\n    bboxes_2, confis2 = predict(model, image_np, size=IMG_SIZE, augment=AUGMENT)  \n\n    if len(bboxes_1) > 0 and len(bboxes_2) > 0: \n#         bboxes_1, bboxes_2 = voc2coco(bboxes_1.detach().numpy(), image_np.shape[1],image_np.shape[2]).astype(int), bboxes_2\n        pred_1 =  tracking_function(tracker1, frame_id_1, bboxes_1, confis1)\n        pred_2 =  tracking_function(tracker2, frame_id_2, bboxes_2, confis2)\n\n        box1 = [list(map(int,box.split(' ')[1:])) for box in pred_1]\n        box2 = [list(map(int,box.split(' ')[1:])) for box in pred_2]\n\n\n        for box in pred_1:\n            conf1.append(float(box.split(' ')[0]))\n        conf1 = np.asarray(conf1)\n        for box in pred_2:\n            conf2.append(float(box.split(' ')[0]))\n        conf2 = np.asarray(conf2)\n    #     print(conf1)\n    #     print(conf2)\n\n        for i in box1:\n            a = []\n            for j in i :\n                a.append(j)\n            c = np.asarray(a)\n            tmp1.append(c)\n        box1 = np.asarray(tmp1)\n\n        for i in box2:\n            a = []\n            for j in i :\n                a.append(j)\n            c = np.asarray(a)\n            tmp2.append(c)\n        box2 = np.asarray(tmp2)\n    #     print(box1)\n    #     print(box2)\n\n        bboxes, scores, labels = run_wbf([box1,box2], [conf1,conf2], image_size = IMG_SIZE)\n#         bboxes, scores, labels = run_wbf([pred_1, pred_2], [scores, confis], image_size = IMG_SIZE)\n    elif len(bboxes_1) > 0: bboxes, scores = voc2coco(bboxes_1.detach().numpy(), image_np.shape[1], image_np.shape[2]).astype(int), scores        \n    elif len(bboxes_2) > 0: bboxes, scores = bboxes_2, confis        \n    else: bboxes = []\n\n    predictions = tracking_function(tracker3, frame_id_3, bboxes, scores)\n#     predictions = tracking_function(tracker, frame_id, bboxes, scores)\n    \n    prediction_str = ' '.join(predictions)\n    pred_df['annotations'] = prediction_str\n    env.predict(pred_df)\n    if frame_id_1 < 3:\n        if len(predictions)>0:\n            box = [list(map(int,box.split(' ')[1:])) for box in predictions]\n        else:\n            box = []\n        display(show_img(image_np, box, bbox_format='coco'))\n#     print('Prediction:', pred_df)\n    frame_id_1 += 1\n    frame_id_2 += 1\n    frame_id_3 += 1","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:17:11.679602Z","iopub.execute_input":"2022-02-10T20:17:11.679866Z","iopub.status.idle":"2022-02-10T20:17:14.456419Z","shell.execute_reply.started":"2022-02-10T20:17:11.679830Z","shell.execute_reply":"2022-02-10T20:17:14.455773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv('submission.csv')\nsub_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-10T20:17:14.457675Z","iopub.execute_input":"2022-02-10T20:17:14.458086Z","iopub.status.idle":"2022-02-10T20:17:14.472430Z","shell.execute_reply.started":"2022-02-10T20:17:14.458048Z","shell.execute_reply":"2022-02-10T20:17:14.471875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}