{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"first = True\nuse_mm = False","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:25:51.897532Z","iopub.execute_input":"2022-02-13T21:25:51.897918Z","iopub.status.idle":"2022-02-13T21:25:51.930231Z","shell.execute_reply.started":"2022-02-13T21:25:51.897795Z","shell.execute_reply":"2022-02-13T21:25:51.929119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:25:55.002856Z","iopub.execute_input":"2022-02-13T21:25:55.003156Z","iopub.status.idle":"2022-02-13T21:25:55.008767Z","shell.execute_reply.started":"2022-02-13T21:25:55.003123Z","shell.execute_reply":"2022-02-13T21:25:55.007772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\nif use_mm:    \n    # pytorch version compatible with openmmlab\n    !pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/torch-1.9.0+cu111-cp37-cp37m-linux_x86_64.whl\n    # torchvision for yolov5\n    !pip install ../input/pytorch-190/torchvision-0.10.0+cu111-cp37-cp37m-linux_x86_64.whl\n    # mmcv install\n    !pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/yapf-0.32.0-py2.py3-none-any.whl\n    !pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/addict-2.4.0-py3-none-any.whl\n    !pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/mmcv_full-1.4.2-cp37-cp37m-manylinux1_x86_64.whl\n\n    # mmdetection install\n    !pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/pycocotools-2.0.3/pycocotools-2.0.3.tar\n    !pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/terminaltables-3.1.10-py2.py3-none-any.whl\n\n    !cp -r ../input/openmmlab-essential-repositories/openmmlab-repos/mmdetection /kaggle/working/\n    %cd /kaggle/working/mmdetection\n    !pip install -e .\n    %cd ..\n    \n    sys.path.append('./mmdetection')","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:25:55.830721Z","iopub.execute_input":"2022-02-13T21:25:55.831712Z","iopub.status.idle":"2022-02-13T21:25:55.859130Z","shell.execute_reply.started":"2022-02-13T21:25:55.831666Z","shell.execute_reply":"2022-02-13T21:25:55.858050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n# norfair dependencies\n%cd /kaggle/input/norfair031py3/\n!pip install commonmark-0.9.1-py2.py3-none-any.whl -f ./ --no-index\n!pip install rich-9.13.0-py3-none-any.whl\n\n!mkdir /kaggle/working/tmp\n!cp -r /kaggle/input/norfair031py3/filterpy-1.4.5/filterpy-1.4.5/ /kaggle/working/tmp/\n%cd /kaggle/working/tmp/filterpy-1.4.5/\n!pip install .\n!rm -rf /kaggle/working/tmp\n\n# norfair\n%cd /kaggle/input/norfair031py3/\n!pip install norfair-0.3.1-py3-none-any.whl -f ./ --no-index","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:25:56.251895Z","iopub.execute_input":"2022-02-13T21:25:56.252588Z","iopub.status.idle":"2022-02-13T21:27:20.358988Z","shell.execute_reply.started":"2022-02-13T21:25:56.252543Z","shell.execute_reply":"2022-02-13T21:27:20.357506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n# https://www.kaggle.com/c/severstal-steel-defect-detection/discussion/109679\n!mkdir -p /tmp/pip/cache2/\n!cp /kaggle/input/yolov5-package/sahi-0.8.22-py3-none-any.whl /tmp/pip/cache2/\n!cp /kaggle/input/yolov5-package/terminaltables-3.1.10-py2.py3-none-any.whl /tmp/pip/cache2/\n!cp /kaggle/input/yolov5-package/thop-0.0.31.post2005241907-py3-none-any.whl /tmp/pip/cache2/\n!cp /kaggle/input/yolov5-package/yolov5-6.0.6-py36.py37.py38-none-any.whl /tmp/pip/cache2/\n!cp /kaggle/input/yolov5-package/fire-0.4.0.xyz /tmp/pip/cache2/fire-0.4.0.tar.gz\n!pip install --no-index --find-links /tmp/pip/cache2/ yolov5\n!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/\n\n!mkdir -p /tmp/pip/cache/\n!cp /kaggle/input/sahi-0822/sahi-0.8.22-py3-none-any.whl /tmp/pip/cache/\n!cp /kaggle/input/sahi-0822/terminaltables-3.1.10-py2.py3-none-any.whl /tmp/pip/cache/\n!cp /kaggle/input/sahi-0822/fire-0.4.0.xyz /tmp/pip/cache/fire-0.4.0.tar.gz\n!pip install --no-index --find-links /tmp/pip/cache/ sahi","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:27:20.363652Z","iopub.execute_input":"2022-02-13T21:27:20.363922Z","iopub.status.idle":"2022-02-13T21:27:50.910981Z","shell.execute_reply.started":"2022-02-13T21:27:20.363890Z","shell.execute_reply":"2022-02-13T21:27:50.909731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:27:50.913103Z","iopub.execute_input":"2022-02-13T21:27:50.913466Z","iopub.status.idle":"2022-02-13T21:27:52.581702Z","shell.execute_reply.started":"2022-02-13T21:27:50.913419Z","shell.execute_reply":"2022-02-13T21:27:52.580388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:27:52.586262Z","iopub.execute_input":"2022-02-13T21:27:52.586623Z","iopub.status.idle":"2022-02-13T21:27:52.596201Z","shell.execute_reply.started":"2022-02-13T21:27:52.586576Z","shell.execute_reply":"2022-02-13T21:27:52.594943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport cv2\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np\nimport ast\nimport sys\nsys.path.append('/kaggle/input/weightedboxesfusion/')\nsys.path.append('/kaggle/input/pickle5/pickle5-backport-master')\nimport pickle5 as pickle\nimport tgbr_util as util\nimport torch\nimport yaml\nfrom  glob import glob \nif use_mm:\n    from mmdet.apis import init_detector, inference_detector\n    import mmcv\n\nfrom sahi.model import Yolov5DetectionModel\nfrom sahi.predict import get_sliced_prediction","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:27:52.598438Z","iopub.execute_input":"2022-02-13T21:27:52.598775Z","iopub.status.idle":"2022-02-13T21:27:54.707596Z","shell.execute_reply.started":"2022-02-13T21:27:52.598729Z","shell.execute_reply":"2022-02-13T21:27:54.706527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append('/kaggle/input/raft-pytorch')\nfrom raft.core.raft import RAFT\n#from raft.core.utils import flow_viz\nfrom raft.core.utils.utils import InputPadder\nfrom raft.config import RAFTConfig\n\nraft_config = RAFTConfig(\n    dropout=0,\n    alternate_corr=False,\n    small=False,\n    mixed_precision=True\n)\n\nraft_model = RAFT(raft_config)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'device: {device}')\n\nweights_path = '/kaggle/input/raft-pytorch/raft-sintel.pth'\n# weights_path = '/kaggle/input/raft-pytorch/raft-things.pth'\n\nckpt = torch.load(weights_path, map_location=device)\nraft_model.to(device)\nraft_model.load_state_dict(ckpt)\n\nSMALL_RATE = 0.25 # image scaling rate \nSTD_TH = 12 # threshold of std of optical flow","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:27:54.709296Z","iopub.execute_input":"2022-02-13T21:27:54.710931Z","iopub.status.idle":"2022-02-13T21:27:59.217238Z","shell.execute_reply.started":"2022-02-13T21:27:54.710868Z","shell.execute_reply":"2022-02-13T21:27:59.216035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torchvision\ntorch.__version__, torchvision.__version__","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:27:59.219140Z","iopub.execute_input":"2022-02-13T21:27:59.219527Z","iopub.status.idle":"2022-02-13T21:27:59.551624Z","shell.execute_reply.started":"2022-02-13T21:27:59.219457Z","shell.execute_reply":"2022-02-13T21:27:59.550554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from norfair import Detection, Tracker\n\n# Helper to convert bbox in format [x_min, y_min, x_max, y_max, score] to norfair.Detection class\ndef to_norfair(detects, frame_id):\n    result = []\n    for x_min, y_min, x_max, y_max, score in detects:\n        xc, yc = (x_min + x_max) / 2, (y_min + y_max) / 2\n        w, h = x_max - x_min, y_max - y_min\n        result.append(Detection(points=np.array([xc, yc]), scores=np.array([score]), data=np.array([w, h, frame_id])))\n        \n    return result\n\n# Euclidean distance function to match detections on this frame with tracked_objects from previous frames\ndef euclidean_distance(detection, tracked_object):\n    return np.linalg.norm(detection.points - tracked_object.estimate)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:27:59.555141Z","iopub.execute_input":"2022-02-13T21:27:59.555562Z","iopub.status.idle":"2022-02-13T21:28:00.176811Z","shell.execute_reply.started":"2022-02-13T21:27:59.555527Z","shell.execute_reply":"2022-02-13T21:28:00.175873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clip_pred(x_min, y_min, width, height):\n    x_min = min(max(int(x_min + 0.5), 0), 1279)\n    y_min = min(max(int(y_min + 0.5), 0), 719)\n    width = int(max(min(1280 - x_min, int(width + 0.5)),1))\n    height = int(max(min(720 - y_min, int(height + 0.5)),1))\n    return x_min, y_min, width, height\n    \n    \ndef tracking_function(tracker, frame_id, bboxes, scores, verbose=False):\n    detects = []\n    predictions = []\n    if len(scores)>0:\n        for i in range(len(bboxes)):\n            box = bboxes[i]\n            score = scores[i]\n            x_min, y_min, bbox_width, bbox_height = clip_pred(box[0], box[1], box[2], box[3])            \n            detects.append([x_min, y_min, x_min+bbox_width, y_min+bbox_height, score])\n            predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n#             print(predictions[:-1])\n    # Update tracks using detects from current frame\n    tracked_objects = tracker.update(detections=to_norfair(detects, frame_id))\n    if verbose:\n        print(frame_id, tracked_objects)\n    to_add_preds = []\n    if USE_TRACKING:\n        for tobj in tracked_objects:\n            bbox_width, bbox_height, last_detected_frame_id = tobj.last_detection.data\n            if verbose:\n                print(\"last_detected_frame_id:\", last_detected_frame_id)\n            if last_detected_frame_id == frame_id:  # Skip objects that were detected on current frame\n                continue\n            # Add objects that have no detections on current frame to predictions\n            xc, yc = tobj.estimate[0]\n            x_min, y_min = int(round(xc - bbox_width / 2)), int(round(yc - bbox_height / 2))\n            #exclude those in the edge\n            if (x_min + bbox_width >= 1279) or (y_min + bbox_height) >= 719 or (x_min <= 1) or (y_min <= 1):\n                continue\n            \n            score = tobj.last_detection.scores[0]\n            x_min, y_min, bbox_width, bbox_height = clip_pred(x_min, y_min, bbox_width, bbox_height)\n            predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n            \n    return predictions, tracker","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:28:00.178504Z","iopub.execute_input":"2022-02-13T21:28:00.178789Z","iopub.status.idle":"2022-02-13T21:28:00.194658Z","shell.execute_reply.started":"2022-02-13T21:28:00.178745Z","shell.execute_reply":"2022-02-13T21:28:00.193327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_conf_dict = {\n    \"0208_m6_B_LS02_newGT_clahe\": 0.25,\n    \"0209_m6_B_LS02_newLGBT\": 0.1,\n    \"best_s6_3600_GT\":0.3,\n    'yolov5-v79-addgt':0.4,\n    'r27-0210-yolov5m6-2400': 0.27,\n    \n    # 0211\n    'r27-0210-yolov5m6-2400':0.27,\n    \"0210_m6_B_LS02_LGBT_newP\": 0.2,\n    \"0209_m6_B_LS02_LGBT_newP_clahe\":0.25,\n    'GT2_4680': 0.3,\n    'GT_transfer_4680':0.3,\n    '0209_s6_B_LS02_newGT_img4096':0.35,\n    '0212_crcnn_1600':0.25,\n}\nSAHI_dict = {\n    '0203_m6_1200_sliced_S720_ma025_lr5e4': {\n        \"SH\":720,\n        \"SW\":720,\n        \"OHR\":0.2,\n        \"OWR\":0.2,\n    }\n}\nbest_iou_dict = {}\nbest_size_dict = {\n    \"yolov5-v74-fpimage\": 9000,\n    'yolov5-v79-addgt':9000,\n    'r22-0206-yolov5x6-1920-full':int(1920*1.3),\n    'best_s6_3600_GT':int(3600 * 1.3),\n    'GT2_4680':4680,\n    'GT_transfer_4680':4680,\n    '0209_s6_B_LS02_newGT_img4096': int(4096 * 1.3),\n    'r27-0210-yolov5m6-2400': 3120,\n    'r27_m6_add_gt':3120,\n}\nckpt_path_dict = {\n    \"yolov5-v74-fpimage\": \"/kaggle/input/yolov5-v74-fpimage/v74/weights/best.pt\",    \n    \"yolov5-v79-addgt\": \"/kaggle/input/yolov5-v79-addgt/v79/weights/best.pt\",    \n    'r22-0206-yolov5x6-1920-full': \"/kaggle/input/r22-0206-yolov5x6-1920-full/v1/weights/last.pt\",\n    'best_s6_3600_GT': \"/kaggle/input/yolov5-models/best_s6_3600_GT.pt\",\n    \n    'GT2_4680': \"/kaggle/input/yolov5-models/last_s6_3600_GT2_all.pt\",\n    'GT_transfer_4680':  \"/kaggle/input/yolov5-models/last_s6_3600_GT_transfer_all.pt\",\n    \n    '0209_s6_B_LS02_newGT_img4096': \"/kaggle/input/0209-s6-b-ls02-newgt-img4096/best_all.pt\",    \n    'r27-0210-yolov5m6-2400': \"/kaggle/input/r27-yolov5m6-2400-whole/weights/last.pt\",\n    'r27_m6_add_gt': \"/kaggle/input/r27-yolov5m6-2400-whole/full_add_gt/weights/last.pt\",\n}\noutside_models = ['yolov5-v79-addgt','best_s6_3600_GT','GT2_4680', 'GT_transfer_4680','0209_s6_B_LS02_newGT_img4096','r27-0210-yolov5m6-2400', 'r27_m6_add_gt']","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:28:00.198237Z","iopub.execute_input":"2022-02-13T21:28:00.199036Z","iopub.status.idle":"2022-02-13T21:28:00.212629Z","shell.execute_reply.started":"2022-02-13T21:28:00.198984Z","shell.execute_reply":"2022-02-13T21:28:00.211400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RUN_TEST = True\nAUGMENT = True\nIOU = 0.4\nIOU_TH=0.4\n\nUSE_RAFT = True\nUSE_TRACKING = True\nUSE_LOW_CONF = True\nUSE_MULTI_TRACKER = True\nsecond_WBF_LB = True\nPRE_CONF_TH = 0.07\nSKIP_BOX_TH = 0.2\n\nLB_model = 'yolov5-v79-addgt'\nrcnn_models = ['0212_crcnn_1600_whole']","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:32:27.115029Z","iopub.execute_input":"2022-02-13T21:32:27.115340Z","iopub.status.idle":"2022-02-13T21:32:27.122491Z","shell.execute_reply.started":"2022-02-13T21:32:27.115302Z","shell.execute_reply":"2022-02-13T21:32:27.121318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_params = {}\n# 4 models + LB\nmodel_versions = ['GT2_4680', 'r27-0210-yolov5m6-2400', '0209_s6_B_LS02_newGT_img4096', '0209_m6_B_LS02_LGBT_newP_clahe_whole', 'yolov5-v79-addgt']\n# 5 yolo + 1 rcnn\n# model_versions = ['GT2_4680', 'r27-0210-yolov5m6-2400', '0209_s6_B_LS02_newGT_img4096', '0209_m6_B_LS02_LGBT_newP_clahe_whole', \"GT_transfer_4680\", \n#                   '0212_crcnn_1600_whole']\n# 7 yolo\n# model_versions = ['GT2_4680', 'r27-0210-yolov5m6-2400', '0209_s6_B_LS02_newGT_img4096', '0209_m6_B_LS02_LGBT_newP_clahe_whole', \"GT_transfer_4680\", \n#                   '0210_m6_B_LS02_LGBT_newP_whole', '0208_m6_B_LS02_newGT_clahe_whole']\n\n# '0210_m6_B_LS02_LGBT_newP_whole',\n#                 '0209_m6_B_LS02_LGBT_newP_clahe_whole', 'r27_m6_add_gt']\n#                   ,'0210_m6_B_LS02_LGBT_newP_whole', \"0209_m6_B_LS02_LGBT_newP_clahe_whole\", \n#                   'best_s6_3600_GT','0209_m6_B_LS02_newLGBT_whole']\n#model_versions = ['0211_crcnn_1600']\nfor model_version in model_versions:\n    if model_version in outside_models:\n        # from others dataset\n        meta_params[model_version] = {\"ckpt_path\": ckpt_path_dict[model_version]}\n        continue\n    \n    model_version_k = \"-\".join(model_version.lower().split(\"_\"))\n    model_folder = Path(f\"/kaggle/input/{model_version_k}/\")\n    params_path = model_folder / \"config\" / \"params.yaml\"\n    with open(params_path) as file:\n        params = yaml.load(file, Loader=yaml.FullLoader)    \n    for key, val in params.items():\n        if \"dir\" in key or \"path\" in key or \"file\" in key:\n            params[key] = Path(val)         \n    meta_params[model_version] = params","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:32:28.462088Z","iopub.execute_input":"2022-02-13T21:32:28.462762Z","iopub.status.idle":"2022-02-13T21:32:28.490604Z","shell.execute_reply.started":"2022-02-13T21:32:28.462724Z","shell.execute_reply":"2022-02-13T21:32:28.489546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"home_dir = Path(\"../input/\")\ndata_dir = Path(\"../input/tensorflow-great-barrier-reef\")\ndef transform_params(version, params, home_dir, data_dir):\n    version_k = \"-\".join(version.lower().split(\"_\"))\n    for key, val in params.items():\n        if \"path\" in key or \"dir\" in key:\n            old_val = val.resolve()\n            if key not in [\"data_path\", \"root_dir\"]:\n                new_val = home_dir / \"/\".join(str(old_val).split(\"/\")[6:])\n            else:\n                new_val = data_dir\n            if version in str(new_val):\n                if key != \"ckpt_path\":\n                    new_val = Path(str(new_val).replace(version, version_k))\n                else:\n                    if params['tools'] == 'mmdetection':\n                        ckp = home_dir / version_k\n                        if \"whole\" in version:\n                            ckp = glob(str(ckp) + \"/\" + \"epoch*.pth\")[0]\n                        else:\n                            ckp = glob(str(ckp) + \"/\" + \"best*.pth\")[0]\n                        new_val = Path(ckp)\n                    else:\n                        # YOLOv5\n                        new_val = home_dir / f\"{version_k}/TGBR/{version}/weights/last.pt\"\n                    \n            params[key] = new_val\n    return params","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:32:30.080718Z","iopub.execute_input":"2022-02-13T21:32:30.081033Z","iopub.status.idle":"2022-02-13T21:32:30.092308Z","shell.execute_reply.started":"2022-02-13T21:32:30.081001Z","shell.execute_reply":"2022-02-13T21:32:30.090560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for model_version, params in meta_params.items():\n    params[\"repo\"] = '/kaggle/input/yolov5-vincent/yolov5'\n    old_model_version = model_version\n    if model_version[-6:] == \"_whole\": \n        model_version = model_version[:-6]\n    \n    params[\"conf\"] = PRE_CONF_TH if (USE_LOW_CONF and model_version != LB_model) else best_conf_dict.get(model_version, 0.2) \n    params[\"iou\"] = best_iou_dict.get(model_version, IOU)\nfor model_version in model_versions:\n    if model_version in outside_models:\n        continue\n    params = meta_params[model_version]\n    params = transform_params(model_version, params, home_dir, data_dir)\n    meta_params[model_version] = params\nclahe = cv2.createCLAHE(clipLimit=5, tileGridSize=(10, 10))","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:32:30.353066Z","iopub.execute_input":"2022-02-13T21:32:30.353458Z","iopub.status.idle":"2022-02-13T21:32:30.365745Z","shell.execute_reply.started":"2022-02-13T21:32:30.353355Z","shell.execute_reply":"2022-02-13T21:32:30.364682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ensemble_boxes import weighted_boxes_fusion, nms, non_maximum_weighted, soft_nms\n\ndef modify_size(boxes, to01=True):\n    if len(boxes) > 0:\n        if to01:\n            boxes[:,2:] = boxes[:,2:] + boxes[:,:2]\n            boxes[:,0] = boxes[:,0] / 1279.\n            boxes[:,1] = boxes[:,1] / 719.\n            boxes[:,2] = boxes[:,2] / 1279.\n            boxes[:,3] = boxes[:,3] / 719.\n        else:\n            boxes[:,0] = boxes[:,0] * 1279.\n            boxes[:,1] = boxes[:,1] * 719.\n            boxes[:,2] = boxes[:,2] * 1279.\n            boxes[:,3] = boxes[:,3] * 719.\n            boxes[:,2:] = boxes[:,2:] - boxes[:,:2]\n        \n    return boxes\n    \ndef run_wbf(bboxes, confs, iou_thr=0.5, skip_box_thr=0.1, verbose=False, weights=None):\n    if len(bboxes) == 1:\n        return bboxes[0], confs[0], []   \n    else:\n        for i in range(len(bboxes)):\n            bboxes[i] = modify_size(bboxes[i], to01=True)    \n        \n    labels = [np.ones(len(conf)) for conf in confs]\n    boxes, scores, labels = weighted_boxes_fusion(bboxes, confs, labels, iou_thr=iou_thr, skip_box_thr=0.001, allows_overflow=True, conf_type='avg', weights=weights)\n    \n    boxes = modify_size(boxes, to01=False)  \n    boxes = [box for i,box in enumerate(boxes) if scores[i] >= skip_box_thr]\n    scores = [conf for conf in scores if conf >= skip_box_thr]\n\n    return boxes, scores, labels","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:32:30.593074Z","iopub.execute_input":"2022-02-13T21:32:30.593325Z","iopub.status.idle":"2022-02-13T21:32:30.608118Z","shell.execute_reply.started":"2022-02-13T21:32:30.593295Z","shell.execute_reply":"2022-02-13T21:32:30.606936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(params):\n    try:\n        model = torch.hub.load(params['repo'],\n                               'custom',\n                               path=params['ckpt_path'],\n                               source='local',\n                               force_reload=True)  # local repo\n    except:\n        print(\"torch.hub.load failed, try torch.load\")\n        model = torch.load(params['ckpt_path'])\n    model.conf = params[\"conf\"]  # NMS confidence threshold\n    model.iou  = params[\"iou\"]  # NMS IoU threshold\n    model.classes = None   # (optional list) filter by class, i.e. = [0, 15, 16] for persons, cats and dogs\n    model.multi_label = False  # NMS multiple labels per box\n    model.max_det = 50  # maximum number of detections per image\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:32:30.780800Z","iopub.execute_input":"2022-02-13T21:32:30.781788Z","iopub.status.idle":"2022-02-13T21:32:30.790715Z","shell.execute_reply.started":"2022-02-13T21:32:30.781738Z","shell.execute_reply":"2022-02-13T21:32:30.789480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\n#colors = [(np.random.randint(255), np.random.randint(255), np.random.randint(255)) for idx in range(1)]\ncolors = [(192, 173, 39)]\ndef show_img(img, bboxes, colors, bbox_format='yolo', labels=None):\n    names  = ['cots']*len(bboxes) if labels is None else [\"pred\"] * (len(labels) - sum(labels)) + [\"real\"] * sum(labels)\n    labels = [0]*len(bboxes) if labels is None else labels\n    img    = util.draw_bboxes(img = img,\n                           bboxes = bboxes, \n                           classes = names,\n                           class_ids = labels,\n                           class_name = True, \n                           colors = colors, \n                           bbox_format = bbox_format,\n                           line_thickness = 2)\n    return Image.fromarray(img).resize((800, 400))","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:32:30.999216Z","iopub.execute_input":"2022-02-13T21:32:31.000008Z","iopub.status.idle":"2022-02-13T21:32:31.010466Z","shell.execute_reply.started":"2022-02-13T21:32:30.999971Z","shell.execute_reply":"2022-02-13T21:32:31.009307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if first:\n    import greatbarrierreef\n    env = greatbarrierreef.make_env()# initialize the environment\n    iter_test = env.iter_test()      # an iterator which loops over the test set and sample submission\n    first = False","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:32:31.353573Z","iopub.execute_input":"2022-02-13T21:32:31.354602Z","iopub.status.idle":"2022-02-13T21:32:31.362219Z","shell.execute_reply.started":"2022-02-13T21:32:31.354546Z","shell.execute_reply":"2022-02-13T21:32:31.361076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {}\nfor model_version, params in meta_params.items():\n    print(model_version)\n    if params.get('tools',\"yolov5\") == 'mmdetection':\n        test_pipeline = [\n            mmcv.utils.config.ConfigDict(type='LoadImageFromFile'),\n            mmcv.utils.config.ConfigDict(\n                type='MultiScaleFlipAug',\n                img_scale=[(params['img_size'], params['img_size'])] ,\n                flip=[False],\n                transforms=[\n                    mmcv.utils.config.ConfigDict(type='Resize', keep_ratio=False),\n                    mmcv.utils.config.ConfigDict(type='Pad', size_divisor=32),\n                    mmcv.utils.config.ConfigDict(type='RandomFlip', direction='horizontal'),\n                    mmcv.utils.config.ConfigDict(\n                        type='Normalize',\n                        mean=[123.675, 116.28, 103.53],\n                        std=[58.395, 57.12, 57.375],\n                        to_rgb=True),\n                    mmcv.utils.config.ConfigDict(type='ImageToTensor', keys=['img']),\n                    mmcv.utils.config.ConfigDict(type='Collect', keys=['img'])\n                ])\n        ]\n\n        mm_config =  mmcv.Config.fromfile(str(params['cfg_dir'] / \"config.py\"))\n        mm_config['data']['test']['pipeline'] = test_pipeline\n        model = init_detector(mm_config, str(params['ckpt_path']), device='cuda:0')            \n    elif \"slice\" in model_version:\n        model = Yolov5DetectionModel(\n            model_path=str(params['ckpt_path']),\n            confidence_threshold=params[\"conf\"],\n            image_size = int(params['img_size'] * 1.3),\n            device=\"cuda:0\", # or 'cuda:0'\n        )    \n    else:            \n        model = load_model(params)\n        device = torch.device(\"cuda\")\n        model.to(device)\n    models[model_version] = model    ","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:32:31.594145Z","iopub.execute_input":"2022-02-13T21:32:31.594473Z","iopub.status.idle":"2022-02-13T21:32:35.909129Z","shell.execute_reply.started":"2022-02-13T21:32:31.594438Z","shell.execute_reply":"2022-02-13T21:32:35.908181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\nnp.random.seed(13)\ncandidates = df.sample(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:32:35.911402Z","iopub.execute_input":"2022-02-13T21:32:35.911826Z","iopub.status.idle":"2022-02-13T21:32:35.945544Z","shell.execute_reply.started":"2022-02-13T21:32:35.911776Z","shell.execute_reply":"2022-02-13T21:32:35.944627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_check = []\nfor idx,row in candidates.iterrows():\n    path = \"/kaggle/input/tensorflow-great-barrier-reef/train_images/video_{}/{}.jpg\".format(row['video_id'], row['video_frame'])\n    img = cv2.imread(path)[...,::-1]\n    iter_check.append(img)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:32:35.946922Z","iopub.execute_input":"2022-02-13T21:32:35.947228Z","iopub.status.idle":"2022-02-13T21:32:36.199094Z","shell.execute_reply.started":"2022-02-13T21:32:35.947183Z","shell.execute_reply":"2022-02-13T21:32:36.198127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_tracker():\n    return Tracker(\n                distance_function=euclidean_distance, \n                distance_threshold=30,\n                hit_inertia_min=3,\n                hit_inertia_max=6,\n                initialization_delay=2,\n            )\n\ndef pred_img(img, tracker, frame_id, idx, verbose=False, show_img_num=10, raft_image_old=None):\n    # RAFT for sequence change detection    \n    if USE_MULTI_TRACKER:\n        trackers = tracker\n        \n    raft_image = None\n    if USE_RAFT:\n        raft_image = cv2.resize(img, dsize=None, fx=SMALL_RATE, fy=SMALL_RATE)\n        raft_image = torch.from_numpy(raft_image).permute(2, 0, 1).float().to(device)\n\n        if raft_image_old is not None:\n            img1 = raft_image[None].to(device)\n            img2 = raft_image_old[None].to(device)\n\n            padder = InputPadder(img1.shape)\n            img1, img2 = padder.pad(img1, img2)\n            with torch.no_grad():\n                flow_low, flow_up = raft_model(img1, img2, iters=10, test_mode=True)\n            flow_std = np.std(flow_up.cpu().numpy())\n            if verbose:\n                print(\"flow_std {:.4f}\".format(flow_std))\n\n            if flow_std > STD_TH: # reinitialize tracker\n                if verbose:\n                    print(\"flow_std {:.4f} > STD_TH {}, reinitialize tracker \".format(flow_std, STD_TH))\n                if USE_MULTI_TRACKER:\n                    for j in trackers.keys():\n                        trackers[j] = init_tracker()\n                else:\n                    tracker = init_tracker()\n                frame_id = 0        \n    \n    prd_bboxs_all = []\n    confis_all = []\n    \n    img_clahe = img.copy()\n    for i in range(3):\n        img_clahe[:, :, i] = clahe.apply((img_clahe[:, :, i])) \n        \n    LB_bboxes = []\n    LB_confs = []\n    second_bbox = []\n    seconf_conf = []\n    for j, (model_version, model) in enumerate(models.items()):\n        CONF = PRE_CONF_TH if USE_LOW_CONF else meta_params[model_version]['conf']\n        if verbose:\n            print(model_version)\n        if model_version in best_size_dict:\n            SIZE = best_size_dict[model_version]\n        else:\n            SIZE = int(meta_params[model_version][\"img_size\"] * 1.3)\n            \n        if meta_params[model_version].get(\"tools\",\"yolov5\") == \"mmdetection\":\n            if meta_params[model_version].get(\"use_clahe\", False):\n                result = inference_detector(model, img_clahe[...,::-1])\n            else:\n                result = inference_detector(model, img[...,::-1])\n                \n            bboxes = result[0][:,:4]\n            bboxes[:,2:] = bboxes[:,2:] - bboxes[:,:2]\n            confs = result[0][:,4]\n            bboxes = [b for i,b in enumerate(bboxes) if confs[i] >= CONF]\n            confs = [c for c in confs if c >= CONF]\n        elif \"slice\" in model_version:\n            if model_version[-6:] == \"_whole\": \n                model_version = model_version[:-6]\n            if meta_params[model_version].get(\"use_clahe\", False):\n                result_sliced = get_sliced_prediction(\n                    img_clahe,\n                    model,\n                    slice_height = SAHI_dict[model_version]['SH'],\n                    slice_width = SAHI_dict[model_version]['SW'],\n                    overlap_height_ratio = SAHI_dict[model_version]['OHR'],\n                    overlap_width_ratio = SAHI_dict[model_version]['OWR'],\n                    postprocess_match_threshold = IOU)\n            else:\n                result_sliced = get_sliced_prediction(\n                    img,\n                    model,\n                    slice_height = SAHI_dict[model_version]['SH'],\n                    slice_width = SAHI_dict[model_version]['SW'],\n                    overlap_height_ratio = SAHI_dict[model_version]['OHR'],\n                    overlap_width_ratio = SAHI_dict[model_version]['OWR'],\n                    postprocess_match_threshold = IOU)\n\n            object_prediction_list = result_sliced.object_prediction_list\n            confs = [obj_pred.score.value for obj_pred in object_prediction_list]\n            bboxes = np.array([obj_pred.bbox.to_coco_bbox() for obj_pred in object_prediction_list])                    \n        else:                \n            if meta_params[model_version].get(\"use_clahe\", False):\n                bboxes, confs = util.predict(model, img_clahe, size=SIZE, augment=AUGMENT)\n            else:\n                bboxes, confs = util.predict(model, img, size=SIZE, augment=AUGMENT)\n                    \n        bboxes = np.array(bboxes, dtype=np.float64)\n        confs = np.array(confs)\n        if USE_MULTI_TRACKER:\n            if verbose:\n                print(\"Use multi tracker\")\n            predictions, tracker = tracking_function(trackers[j], frame_id, bboxes, confs, 0)\n            trackers[j] = tracker\n            bboxes =  np.array([[float(p) for p in pred.split(\" \")[1:]] for pred in predictions])\n            confs =  np.array([float(pred.split(\" \")[0]) for pred in predictions])\n        \n        if model_version in rcnn_models:\n            second_bbox = bboxes\n            second_conf = confs # [c * 0.8 for c in confis]\n        elif second_WBF_LB and model_version == LB_model:\n            LB_bboxes = np.array([bboxes[i] for i, c in enumerate(confs) if c >= meta_params[model_version]['conf']])\n            LB_confs = np.array([c for i, c in enumerate(confs) if c >= meta_params[model_version]['conf']])\n        else:\n            prd_bboxs_all.append(bboxes)\n            confis_all.append(confs)\n    # wbf\n    wbf_boxes, wbf_confs, _ = run_wbf(prd_bboxs_all, confis_all, IOU_TH, SKIP_BOX_TH, verbose)\n    if verbose:\n        print(\"original WBF prediction\")\n        for i, c in enumerate(wbf_confs):\n            print(i, c, wbf_boxes[i])\n    if len(second_bbox) > 0 and len(wbf_boxes) > 0:\n        keep_second_part = util.calc_iou(np.array(second_bbox), np.array(wbf_boxes)).max(axis=1) >= 0.5\n        second_bbox = second_bbox[keep_second_part]\n        second_conf = (np.array(second_conf)[keep_second_part]).tolist()\n        if len(second_bbox) > 0:\n            if verbose:\n                print(\"consider {} second bbox and {} wbf_bbox\".format(len(second_bbox), len(wbf_boxes)))\n            wbf_boxes, wbf_confs, _ = run_wbf([np.array(wbf_boxes)] +[second_bbox], [wbf_confs] + [second_conf], IOU_TH, 0, \n                                              verbose=False,weights=[len(model_versions)-1, 1])            \n    \n    if second_WBF_LB and len(LB_bboxes) > 0:\n        if verbose:\n            print(\"second_WBF_LB, add {} LB predictions: {} with confs {}\".format(len(LB_bboxes), LB_bboxes, LB_confs))\n            \n        wbf_boxes, wbf_confs, _ = run_wbf([np.array(wbf_boxes)] +[LB_bboxes], [wbf_confs] + [LB_confs.tolist()], IOU_TH, 0, \n                                          verbose=False,weights=[len(model_versions)-1, 1])            \n    # tracking\n    if not USE_MULTI_TRACKER:\n        predictions, tracker = tracking_function(tracker, frame_id, wbf_boxes, wbf_confs, verbose)\n    else:\n        predictions = ['{:.2f} {} {} {} {}'.format(wbf_confs[i], b[0], b[1], b[2], b[3]) for i, b in enumerate(wbf_boxes)]\n    if verbose:\n        print(wbf_boxes)\n        print(\" \".join(predictions))\n        \n    if idx < show_img_num:\n        display(util.show_img(img, wbf_boxes, colors, bbox_format='coco'))    \n    if USE_MULTI_TRACKER:    \n        return predictions, trackers, raft_image\n    else:\n        return predictions, tracker, raft_image","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:32:36.202168Z","iopub.execute_input":"2022-02-13T21:32:36.202476Z","iopub.status.idle":"2022-02-13T21:32:36.246576Z","shell.execute_reply.started":"2022-02-13T21:32:36.202433Z","shell.execute_reply":"2022-02-13T21:32:36.245257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_MULTI_TRACKER:\n    trackers = {j: init_tracker() for j in range(len(models))}\nelse:\n    tracker = init_tracker()\nframe_id = 0\nraft_image_old = None\nfor idx, img in enumerate(tqdm(iter_check)):\n    if USE_MULTI_TRACKER:\n        predictions, trackers, raft_image_old = pred_img(img, trackers, frame_id, idx, \n                                                         verbose=True, show_img_num=20, \n                                                         raft_image_old=raft_image_old)\n    else:\n        predictions, tracker, raft_image_old = pred_img(img, tracker, frame_id, idx, \n                                                        verbose=True, show_img_num=20, \n                                                        raft_image_old=raft_image_old)\n        \n    frame_id += 1","metadata":{"execution":{"iopub.status.busy":"2022-02-13T21:32:36.248835Z","iopub.execute_input":"2022-02-13T21:32:36.249461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_MULTI_TRACKER:\n    trackers = {j: init_tracker() for j in range(len(models))}\nelse:\n    tracker = init_tracker()\n\nframe_id = 0\nraft_image_old = None\nif RUN_TEST:\n    print(\"Run test\")    \n    for idx, (img, pred_df) in enumerate(tqdm(iter_test)):\n        if USE_MULTI_TRACKER:\n            predictions, trackers, raft_image_old = pred_img(img, trackers, frame_id, idx, \n                                                             verbose=(idx<=2), show_img_num=3, \n                                                             raft_image_old=raft_image_old)\n        else:\n            predictions, tracker, raft_image_old = pred_img(img, tracker, frame_id, idx, \n                                                            verbose=(idx<=2), show_img_num=3, \n                                                            raft_image_old=raft_image_old)\n        pred_df['annotations'] = \" \".join(predictions)\n        env.predict(pred_df)\n        frame_id += 1\n    sub_df = pd.read_csv('submission.csv')\n    print(sub_df.head())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if use_mm:\n    import shutil\n    shutil.rmtree('/kaggle/working/mmdetection')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nbest_conf_dict= {\n    '0128_yolov5l_v3': 0.25,\n    \"0128_yolov5l_v3_highFP\": 0.25,\n    \"0129_yolov5l_v3_highFP_1800\": 0.25,\n    \"0129_yolov5l_v3_highFP_1800_e11\": 0.16,\n    \"0129_yolov5l_v3_highFP_clahe\": 0.19,\n    '0129_yolov5l_v3_highFP_3600_e11': 0.25,\n    '0129_yolov5l_v3_1800_clahe': 0.15,\n    '0129_yolov5l_v3_2700_clahe': 0.2,\n    '0130_yolov5m6_2300':0.27,\n    \"yolov5-v74-fpimage\": 0.4,\n    '0130_yolov5s6_3100': 0.31,\n    '0131_yolov5s6_3100': 0.31,\n    \"0130_yolov5m6_noHighFP_2300\": .40,\n    \"0130_yolov5m6_noClahe_2300\": .23,\n    \"0131_yolov5m6_noClahe_2300\": .23,\n    \"0130_yolov5m6_2300_sheep\": .11,\n    \"0131_yolov5m6_2300_sheep\": .11,\n    '0202_frcnn_1600_highFP_e8': 0.34,\n    '0203_m6_1200_sliced_S720_ma025_lr5e4' : 0.53, # 0.44 for no AUG\n    \"0205_m_TPH_1900_B\": 0.27,\n    '0204_yolov5s6_B': 0.20,\n    '0206_yolov5s6_B': 0.20,\n    '0205_yolov5m6_B': 0.17,\n    '0206_yolov5m6_B_LS02': 0.3,\n    'r22-0206-yolov5x6-1920-full':0.2,\n    '0207_yolov5m6_B_LS02_newGT': 0.38, # CV 696\n    '0207_yolov5m6_B_LS02_newGT_clahe':0.33, # 696\n}\n\n'''","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:48:39.461525Z","iopub.status.idle":"2022-02-13T04:48:39.462079Z","shell.execute_reply.started":"2022-02-13T04:48:39.46185Z","shell.execute_reply":"2022-02-13T04:48:39.461874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}