{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"CONFIG = {\n 'INFERENCE_TYPE': 'MultiFramePredictor',\n 'ROOT_DIR': '/kaggle/input/tensorflow-great-barrier-reef/',\n 'IOU': 0.2,\n 'CLFR_WEIGHT': '/kaggle/input/two-class-classifier-reef/fold0_best_AUC.pth', \n 'SKIP_BOX_THR': 0.0001,\n 'SIGMA':0.01,\n 'CONF' : 0.05,\n 'NORFAIR': True,\n 'DEEPSORT':False,\n 'PETDET': '/kaggle/input/clf-saks-tf/tf_efficientnet_b0_ns_5.pth',\n 'BETA': 0.3,\n 'YOLOR': False,\n 'TENSORRT': False,\n 'IMG_SIZE':1280,\n 'EXTRA_AUG': True,\n 'MIN_SZ': 4,\n 'MAX_SZ': 250,\n 'ALPHA': 1e-6,\n 'CONF_THRESH': 0.1,\n 'NMS_THRESH': 0.4,\n 'IOU_THRESH': 0.2,\n 'IOU_THRESH_DELETE': 0.01,\n 'MAXLEN': 5,\n 'ENSEMBLE':'multi',\n 'MODELS': [ {'model_type': 'yolov5',\n 'path': '/kaggle/input/yolov5-1920-4/best.pt',\n 'model_dict': {'imgsize':9000,'augment': False},\n 'weight': 5,\n 'conf': 0.275,},\n {'model_type': 'yolov5',\n 'path': '/kaggle/input/yolov5s6/f2_sub2.pt',\n 'model_dict': {'imgsize':6400,'augment': False},\n 'weight': 6,\n 'conf': 0.2,},\n {'model_type': 'yolov5',\n 'path': '/kaggle/input/yolo-v5-s-video-analysis/best.pt',\n 'model_dict': {'imgsize':1920*2,'augment': True},\n 'weight': 3,\n 'conf': 0.25,},\n {'model_type': 'yolov5',\n 'path': '/kaggle/input/fork-of-weights-yolo-v5-l/best.pt',\n 'model_dict': {'imgsize':1920*2,'augment': True},\n 'weight': 1,\n 'conf': 0.25,},\n ]\n }\n","metadata":{"execution":{"iopub.status.busy":"2022-02-11T21:26:13.403993Z","iopub.execute_input":"2022-02-11T21:26:13.404271Z","iopub.status.idle":"2022-02-11T21:26:13.414581Z","shell.execute_reply.started":"2022-02-11T21:26:13.404237Z","shell.execute_reply":"2022-02-11T21:26:13.413425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture cell_print\nif CONFIG['NORFAIR']:\n # norfair dependencies\n %cd /kaggle/input/norfair031py3/\n !pip install commonmark-0.9.1-py2.py3-none-any.whl -f ./ --no-index\n !pip install rich-9.13.0-py3-none-any.whl\n\n !mkdir /kaggle/working/tmp\n !cp -r /kaggle/input/norfair031py3/filterpy-1.4.5/filterpy-1.4.5/ /kaggle/working/tmp/\n %cd /kaggle/working/tmp/filterpy-1.4.5/\n !pip install .\n !rm -rf /kaggle/working/tmp\n\n # norfair\n %cd /kaggle/input/norfair031py3/\n !pip install norfair-0.3.1-py3-none-any.whl -f ./ --no-index\n %cd /kaggle/working/\n from norfair import Detection, Tracker\n \n!pip install ../input/object-detection-libraries/ensemble_boxes-1.0.4-py3-none-any.whl\n!pip install ../input/object-detection-libraries/fire-0.4.0/fire-0.4.0.tar\n!pip install ../input/object-detection-libraries/terminaltables-3.1.10-py2.py3-none-any.whl\n!pip install ../input/object-detection-libraries/thop-0.0.31.post2005241907-py3-none-any.whl\n!pip install -U ../input/object-detection-libraries/sahi-0.8.13-py3-none-any.whl\n!pip install -U ../input/object-detection-libraries/yolov5-6.0.5-py36.py37.py38-none-any.whl\nif CONFIG['TENSORRT']:\n !pip install ../input/tensorrt/nvidia-pyindex-1.0.9.tar.gz\n !pip install ../input/tensorrt/nvidia_cudnn_cu115-8.3.1.22-py3-none-manylinux1_x86_64.whl\n !pip install ../input/tensorrt/nvidia-cudnn-cu11-2021.12.8.tar.gz\n !pip install ../input/tensorrt/nvidia_cuda_runtime_cu115-11.5.117-py3-none-manylinux1_x86_64.whl\n !pip install ../input/tensorrt/nvidia-cuda-runtime-cu11-2021.10.25.tar.gz\n !pip install ../input/tensorrt/nvidia_cublas_cu115-11.7.4.6-py3-none-manylinux1_x86_64.whl\n !pip install ../input/tensorrt/nvidia-cublas-cu11-2021.10.25.tar.gz\n !pip install ../input/tensorrt/nvidia_tensorrt-8.2.1.8-cp37-none-linux_x86_64.whl\n!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-fonts/Arial.ttf /root/.config/Ultralytics/\n%cp -r /kaggle/input/yolox-cots-models /kaggle/working/\n%cd /kaggle/working/yolox-cots-models/yolox-dep\n!pip install pip-21.3.1-py3-none-any.whl -f ./ --no-index\n!pip install loguru-0.5.3-py3-none-any.whl -f ./ --no-index\n!pip install ninja-1.10.2.3-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl -f ./ --no-index\n!pip install onnx-1.8.1-cp37-cp37m-manylinux2010_x86_64.whl -f ./ --no-index\n!pip install onnxruntime-1.8.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -f ./ --no-index\n!pip install onnxoptimizer-0.2.6-cp37-cp37m-manylinux2014_x86_64.whl -f ./ --no-index\n!pip install thop-0.0.31.post2005241907-py3-none-any.whl -f ./ --no-index\n!pip install tabulate-0.8.9-py3-none-any.whl -f ./ --no-index\n%cd /kaggle/working/yolox-cots-models/YOLOX\n!pip install -r requirements.txt\n!pip install -v -e . \n%cd /kaggle/working/yolox-cots-models/yolox-dep/cocoapi/PythonAPI\n!make\n!make install\n!python setup.py install\nimport pycocotools\n%cd /kaggle/working/yolox-cots-models/YOLOX\n\nconfig_file_template = '''\n\n#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom yolox.exp import Exp as MyExp\n\nclass SmallExp(MyExp):\n def __init__(self):\n super(SmallExp, self).__init__()\n self.depth = 0.33\n self.width = 0.50\n self.exp_name = (os.path.split(os.path.realpath(__file__))[1].split(\".\")[0])+'_s'\n self.num_classes = 1\n\nclass LargeExp(MyExp):\n def __init__(self):\n super(LargeExp, self).__init__()\n self.depth = 1\n self.width = 1\n self.exp_name = (os.path.split(os.path.realpath(__file__))[1].split(\".\")[0])+'_l'\n self.num_classes = 1\n \nclass XLargeExp(MyExp):\n def __init__(self):\n super(XLargeExp, self).__init__()\n self.depth = 1.33\n self.width = 1.25\n self.exp_name = (os.path.split(os.path.realpath(__file__))[1].split(\".\")[0])+'_x'\n self.num_classes = 1\n \nclass MediumExp(MyExp):\n def __init__(self):\n super(MediumExp, self).__init__()\n self.depth = 0.67\n self.width = 0.75\n self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]+'_m'\n self.num_classes = 1\n'''\n\nwith open('cots_config.py', 'w') as f:\n f.write(config_file_template)\n\nimport importlib\ncurrent_exp = importlib.import_module('cots_config')\n\nif CONFIG['YOLOR']:\n !cp -r /kaggle/input/yolor-repo /kaggle/working/\n !cd /kaggle/working/yolor-repo/yolor/mish-cuda && python setup.py build install\n !cd /kaggle/working/yolor-repo/yolor/pytorch_wavelets && pip install .\n %cd /kaggle/input/yolor-repo/yolor\n from models.models import *\n from utils.general import non_max_suppression, scale_coords\n from utils.datasets import letterbox\n %cd /kaggle/working/yolox-cots-models/YOLOX\n","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-11T21:08:40.085975Z","iopub.execute_input":"2022-02-11T21:08:40.086233Z","iopub.status.idle":"2022-02-11T21:15:56.795726Z","shell.execute_reply.started":"2022-02-11T21:08:40.086206Z","shell.execute_reply":"2022-02-11T21:15:56.794892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = '''\nDEEPSORT:\n REID_CKPT: \"/kaggle/input/yolov5-deepsort-pytorch/ckpt.t7\"\n MAX_DIST: 30\n MIN_CONFIDENCE: 0.001\n NMS_MAX_OVERLAP: 0.2\n MAX_IOU_DISTANCE: 0.2\n MAX_AGE: 70\n N_INIT: 1\n NN_BUDGET: 100\n'''\nwith open('/kaggle/working/deepsort.yaml', 'w') as f:\n f.write(config)","metadata":{"execution":{"iopub.status.busy":"2022-02-11T21:16:12.728177Z","iopub.execute_input":"2022-02-11T21:16:12.728434Z","iopub.status.idle":"2022-02-11T21:16:12.733152Z","shell.execute_reply.started":"2022-02-11T21:16:12.728405Z","shell.execute_reply":"2022-02-11T21:16:12.732416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from yolox.utils import postprocess\nfrom yolox.data.data_augment import ValTransform\nimport numpy as np\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nimport pandas as pd\nimport os, copy\nfrom collections import deque\nimport cv2\nimport matplotlib.pyplot as plt\nimport glob\nimport torch.nn as nn\nimport shutil\nimport sys\n%cd /kaggle/working\nfrom ensemble_boxes import *\nimport albumentations as A\nsys.path.extend(['/kaggle/input/tensorflow-great-barrier-reef', '/kaggle/input/pytorch-image-models/pytorch-image-models-master',\n '/kaggle/input/yolov5-deepsort-pytorch/Yolov5_DeepSort_Pytorch-master/Yolov5_DeepSort_Pytorch-master/',\n '/kaggle/input/object-detection-libraries/easydict-master/'\n ])\nfrom deep_sort_pytorch.deep_sort.deep_sort import DeepSort\nfrom deep_sort_pytorch.utils.parser import get_config\nimport torch\nimport timm\nfrom PIL import Image\nimport ast\nimport greatbarrierreef\nimport time\nimport warnings\nimport numpy as np\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport pycocotools\nimport torch\nimport torchvision\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection import FasterRCNN\nfrom sahi.model import Yolov5DetectionModel\nfrom sahi.predict import get_prediction, get_sliced_prediction, predict\nimport yolov5\n\nwarnings.filterwarnings('ignore')\n\nenv = greatbarrierreef.make_env()\niter_test = env.iter_test()\n\nnp.random.seed(32)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-11T21:16:13.636282Z","iopub.execute_input":"2022-02-11T21:16:13.636544Z","iopub.status.idle":"2022-02-11T21:16:20.92498Z","shell.execute_reply.started":"2022-02-11T21:16:13.636514Z","shell.execute_reply":"2022-02-11T21:16:20.924026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_path(row):\n row['image_path'] = '{}/train_images/video_{}/{}.jpg'.format(CONFIG['ROOT_DIR'], row.video_id, row.video_frame)\n return row\n\n# Train Data\ndf = pd.read_csv('{}/train.csv'.format(CONFIG['ROOT_DIR']))\ndf = df.progress_apply(get_path, axis=1)\ndf['annotations'] = df['annotations'].progress_apply(lambda x: ast.literal_eval(x))\ndisplay(df.head(2))\ndf['num_bbox'] = df['annotations'].progress_apply(lambda x: len(x))\ndata = (df.num_bbox>0).value_counts()/len(df)*100\nprint(f\"No BBox: {data[0]:0.2f}% | With BBox: {data[1]:0.2f}%\")","metadata":{"execution":{"iopub.status.busy":"2022-02-11T21:16:23.124842Z","iopub.execute_input":"2022-02-11T21:16:23.125567Z","iopub.status.idle":"2022-02-11T21:16:40.322452Z","shell.execute_reply.started":"2022-02-11T21:16:23.125516Z","shell.execute_reply":"2022-02-11T21:16:40.321625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomNet(nn.Module):\n def __init__(self, out_feature = 1, backbone='tf_efficientnet_b1_ns', pretrained=True):\n super(CustomNet, self).__init__()\n self.backbne_name = backbone\n self.backbone = timm.create_model(backbone, pretrained=pretrained)\n self.out_feature = out_feature\n\n if \"efficientnet\" in backbone:\n self.in_features = self.backbone.classifier.in_features\n self.backbone.global_pool = nn.Identity()\n self.backbone.classifier = nn.Identity()\n elif \"nfnet\" in backbone:\n self.in_features = self.backbone.head.fc.in_features\n self.backbone.head.fc = nn.Identity()\n self.backbone.head.global_pool = nn.Identity()\n \n self.pooling = nn.AdaptiveAvgPool2d(1)\n self.fc = nn.Linear(self.in_features, out_feature)\n\n def forward(self, x):\n bs = x.size(0)\n features = self.backbone(x)\n pooled_features = self.pooling(features).view(bs, -1)\n output = self.fc(pooled_features)\n return output\n\nclass PetNet(nn.Module):\n def __init__(\n self, model_name = \"tf_efficientnet_b0_ns\", out_features = 1, inp_channels= 3, pretrained=False):\n super().__init__()\n self.model = timm.create_model(model_name, pretrained=False, in_chans=inp_channels, num_classes = out_features)\n \n def forward(self, image):\n output = self.model(image)\n return output","metadata":{"execution":{"iopub.status.busy":"2022-02-11T21:17:31.498266Z","iopub.execute_input":"2022-02-11T21:17:31.498528Z","iopub.status.idle":"2022-02-11T21:17:31.509284Z","shell.execute_reply.started":"2022-02-11T21:17:31.498499Z","shell.execute_reply":"2022-02-11T21:17:31.508619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Deque(deque):\n def __init__(self, maxlen):\n super(Deque, self).__init__(maxlen = maxlen)\n \n def init_deque(self, data):\n for i in range(self.maxlen):\n super().append(data)\n \n def append(self, __data):\n if len(self) == 0:\n self.init_deque(__data)\n else:\n super().append(__data)\n\n def extend(self, __data):\n if len(self) == 0:\n self.init_deque(__data[0])\n else:\n super().extend(__data)","metadata":{"execution":{"iopub.status.busy":"2022-02-11T21:17:32.948715Z","iopub.execute_input":"2022-02-11T21:17:32.948984Z","iopub.status.idle":"2022-02-11T21:17:32.959361Z","shell.execute_reply.started":"2022-02-11T21:17:32.948945Z","shell.execute_reply":"2022-02-11T21:17:32.95841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SingleFramePredictor:\n def __init__(self, config):\n self.config = config\n self.colors = [(0, 0, 255),(255,0,0)]\n self.clf_deque = Deque(self.config['MAXLEN'])\n if self.config['ALPHA'] != 0.0:\n self.classifier = nn.DataParallel(CustomNet(pretrained = False).cuda()).eval()\n self.classifier.load_state_dict(torch.load(config['CLFR_WEIGHT']))\n self.transform = A.Compose([A.Resize(config['IMG_SIZE'], config['IMG_SIZE']),A.Normalize()])\n \n if self.config['EXTRA_AUG']:\n self.tfs = A.Compose([A.CLAHE(clip_limit=(4,4),always_apply=True,p=1.0),\n A.RandomBrightness(limit = (0.05,0.05),always_apply=True,p=1.0)\n ], p=1.0)\n \n if self.config['BETA'] != 0.0:\n self.pet_model = PetNet()\n self.pet_model.to('cuda:0')\n self.pet_model.load_state_dict(torch.load(self.config['PETDET']))\n self.pet_tfs = A.Compose([A.Resize(64,64),A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])\n\n self.device = torch.device('cuda:0')\n self.yolox_preproc = ValTransform(legacy = False)\n if self.config['NORFAIR']:\n self.tracker = Tracker(distance_function=self.euclidean_distance, distance_threshold=30,hit_inertia_min=3,hit_inertia_max=6,initialization_delay=1,)\n self.height , self.width = 720,1280\n self.ratio = np.array([self.width+20, self.height+20, self.width+20, self.height+20])\n \n for model in self.config['MODELS']:\n if model['model_type'] == 'yolov5':\n model['model'] = self.yolov5_model(model['path'],model['conf'])\n elif model['model_type'] == 'sahi_yolo':\n model['model'] = self.sahi_model(model['path'],model['conf'])\n elif model['model_type'] == 'faster_rcnn':\n model['model'] = self.faster_rcnn_model(model['path'])\n elif 'yolox' in model['model_type']:\n model['model'] = self.yolox_model(model['path'], model['model_type'])\n elif 'yolor' in model['model_type']:\n model['model'] = self.yolor_model(model['path'], model['model_type'])\n print(model['path'],'Sucessfully Loaded of type',model['model_type'])\n \n print('Every model Loaded Sucessfully')\n \n def yolov5_model(self, weights, conf):\n model = torch.hub.load('/kaggle/input/reef-yolo',\n 'custom',\n path=weights,\n source='local',\n force_reload=True) # local repo\n# model = yolov5.load(weights, device = self.device)\n model.conf = conf # NMS confidence threshold\n model.iou = 0.45 # NMS IoU threshold\n model.classes = None # (optional list) filter by class, i.e. = [0, 15, 16] for persons, cats and dogs\n model.multi_label = False # NMS multiple labels per box\n model.max_det = 1000 # maximum number of detections per image\n \n return model\n \n def reset(self, ):\n if self.config['NORFAIR']:\n self.tracker = Tracker(distance_function=self.euclidean_distance, distance_threshold=30,hit_inertia_min=3,hit_inertia_max=6,initialization_delay=1,)\n self.clf_deque = Deque(self.config['MAXLEN'])\n return None\n \n def yolor_model(self, weights, model_type,):\n model = Darknet(f'/kaggle/input/yolor-repo/yolor/cfg/{model_type}.cfg',self.config['IMG_SIZE']).cuda()\n model.load_state_dict(torch.load(weights, map_location=self.device)['model'])\n model.to(self.device).eval()\n return model\n \n def sahi_model(self, weights, conf):\n detection_model = Yolov5DetectionModel(\n model_path= weights,\n confidence_threshold= conf,\n device=\"cuda:0\"\n )\n detection_model.model.max_det = 1000\n detection_model.model.iou = self.config['IOU']\n detection_model.multi_label = False\n return detection_model\n \n def faster_rcnn_model(self, weights):\n model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=False, pretrained_backbone=False)\n num_classes = 2 \n in_features = model.roi_heads.box_predictor.cls_score.in_features\n model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n model.load_state_dict(torch.load(weights))\n model.eval()\n\n model = model.to(self.device)\n return model\n \n def yolox_model(self, weights, _type):\n if _type == 'yolox_l':\n exp = current_exp.LargeExp()\n elif _type == 'yolox_x':\n exp = current_exp.XLargeExp()\n elif _type == 'yolox_s':\n exp = current_exp.SmallExp()\n elif _type == 'yolox_m':\n exp = current_exp.MediumExp()\n \n model = exp.get_model()\n model.cuda()\n model.eval()\n ckpt = torch.load(weights, map_location=\"cpu\")\n model.load_state_dict(ckpt[\"model\"])\n return model\n \n def prepare_classifier_img(self, img):\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n img = self.transform(image = img)['image']\n img = img.astype(np.float32)\n img = img.transpose(2,0,1)\n return torch.from_numpy(img).unsqueeze(0)\n \n def petdet_img(self, img):\n clf_img = self.pet_tfs(image = img)[\"image\"]\n clf_img = clf_img / 255 # convert to 0-1\n clf_img = np.transpose(clf_img, (2, 0, 1)).astype(np.float32)\n clf_img = torch.tensor(clf_img, dtype = torch.float)\n return clf_img\n \n def _petdet_forward(self, img, bboxes):\n idx = []\n for i in range(bboxes.shape[0]):\n with torch.no_grad():\n output = self.pet_model(self.petdet_img(img[int(bboxes[i,1]):int(bboxes[i,3]), int(bboxes[i,0]):int(bboxes[i,2]), :]).unsqueeze(0).to('cuda:0'))\n output = torch.sigmoid(output).detach().cpu().numpy()[0][0]\n if output > self.config['BETA']:\n idx.append(i)\n \n return bboxes[idx], idx\n \n def faster_rcnn_preprocess(self,pixel_array):\n pixel_array = pixel_array.astype(np.float32) / 255.\n return ToTensorV2(p=1.0)(image=pixel_array)['image'].unsqueeze(0)\n \n def _yolov5_forward(self, model, img, conf):\n results = model(img, size = conf['imgsize'], augment = conf['augment']) # custom inference size\n preds = results.pandas().xyxy[0]\n bboxes = preds[['xmin','ymin','xmax','ymax']].values\n if len(bboxes):\n confs = preds.confidence.values\n labels = np.zeros_like(confs)\n return bboxes, confs, labels\n else:\n return np.array([[0.,0.,0.,0.]]),np.array([0.]),np.array([0])\n \n def _yolor_forward(self, model, img, conf):\n img = np.ascontiguousarray(letterbox(img, (1280,1280), scaleFill= True, auto_size = 128)[0])\n img = (torch.from_numpy(img).to(self.device).float().permute(2,0,1)/255).unsqueeze(0)\n with torch.no_grad():\n out = model(img)[0]\n out = non_max_suppression(out, conf , self.config['IOU'], 0, False)[0]\n out[:,:4] = scale_coords(img.shape[2:], out[:,:4], [720,1280])\n out = out.cpu().detach().numpy()\n return out[:, :4], out[:, 4], out[:,5]\n \n def _sahi_yolo_forward(self, model, img):\n bboxes = []\n confs = []\n bclass = []\n result = get_sliced_prediction(img,model, 1280 , slice_height = 360,slice_width = 640,overlap_height_ratio = 0.2,overlap_width_ratio = 0.2, verbose = 0).object_prediction_list\n if len(result):\n for pred in result:\n bboxes.append(pred.bbox.to_voc_bbox())\n confs.append(pred.score.value)\n bclass.append(0)\n return np.array(bboxes), np.array(confs), np.array(bclass)\n else:\n return np.array([[0.,0.,0.,0.]]),np.array([0.]),np.array([0])\n \n def _faster_rcnn_forward(self, model, img, conf):\n with torch.no_grad():\n outputs = model(self.faster_rcnn_preprocess(img).to(self.device))[0]\n \n scores = outputs['scores'].data.cpu().numpy()\n return outputs['boxes'].data.cpu().numpy()[scores >= conf], scores[scores >= conf], np.zeros_like(scores[scores >= conf])\n \n def _yolox_forward(self, model, image, yolox_dict):\n img, _ = self.yolox_preproc(image, None, yolox_dict['test_size'])\n img = torch.from_numpy(img).unsqueeze(0).float().to(self.device)\n with torch.no_grad():\n outputs = model(img)\n outputs = postprocess(outputs, yolox_dict['num_classes'], yolox_dict['confthre'],yolox_dict['nmsthre'], class_agnostic=True)\n if outputs[0] is None:\n return np.array([[0.,0.,0.,0.]]),np.array([0.]),np.array([0])\n\n outputs = outputs[0].cpu().numpy()\n bboxes = outputs[:, 0:4]\n bboxes /= min(yolox_dict['test_size'][0] / image.shape[0], yolox_dict['test_size'][1] / image.shape[1])\n bbclasses = outputs[:, 6]\n scores = outputs[:, 4] * outputs[:, 5]\n select_bboxes = scores > yolox_dict['confthre']\n \n if scores[select_bboxes].shape[0] < 1:\n return np.array([[0.,0.,0.,0.]]),np.array([0.]),np.array([0])\n \n return np.abs(bboxes[select_bboxes, :]), scores[select_bboxes], bbclasses[select_bboxes]\n \n def forward(self, img):\n if self.config['EXTRA_AUG']:\n img = self.tfs(image = img)[\"image\"]\n multi_bboxes = []\n multi_confs = []\n multi_labels = []\n thr_ = self.config['ALPHA']\n if thr_ > 0.0:\n with torch.no_grad():\n conf = torch.sigmoid(self.classifier(self.prepare_classifier_img(img))).cpu().detach().numpy()[0][0].tolist()\n self.clf_deque.append(conf)\n conf = max(self.clf_deque)\n if conf < thr_:\n return multi_bboxes, multi_confs, multi_labels\n else:\n conf = 0.0\n \n for model in self.config['MODELS']:\n if model['model_type'] == 'yolov5':\n bboxes, confs, labels = self._yolov5_forward(model['model'], img, model['model_dict'])\n elif model['model_type'] == 'sahi_yolo':\n bboxes, confs, labels = self._sahi_yolo_forward(model['model'], img)\n elif model['model_type'] == 'faster_rcnn':\n bboxes, confs, labels = self._faster_rcnn_forward(model['model'], img, model['conf'])\n elif 'yolox' in model['model_type']:\n bboxes, confs, labels = self._yolox_forward(model['model'], img[:,:,::-1], model['model_dict'])\n elif 'yolor' in model['model_type']:\n bboxes, confs, labels = self._yolor_forward(model['model'], img, model['conf'])\n tmp = bboxes[bboxes[:,3]*bboxes[:,2] > 0.0]\n if tmp.shape[0] > 0 and self.config['BETA'] > 0.0:\n bboxes,idx = self._petdet_forward(img, tmp)\n confs = confs[idx]\n labels = labels[idx]\n bboxes = self.border_bboxes(bboxes, img.shape)\n multi_bboxes.append((bboxes/self.ratio).tolist())\n# multi_confs.append(((confs*(1 -( thr_+0.001))) + (conf *( thr_+0.001))).tolist())\n multi_confs.append(confs.tolist())\n multi_labels.append(labels.tolist())\n \n return multi_bboxes, multi_confs, multi_labels\n \n def check_result(self, bboxes, confs,shape):\n if len(bboxes)==0:\n return []\n bboxes=np.array(bboxes)\n confs = np.array(confs)\n idx = np.where(conf > self.config['CONF'])\n bboxes = bboxes[idx]\n confs = confs[idx]\n bboxes[bboxes<0]=0\n bboxes=bboxes[(bboxes[:,0]+bboxes[:,2]) < shape[1]]\n bboxes=bboxes[(bboxes[:,1]+bboxes[:,3]) < shape[0]]\n bboxes=bboxes[bboxes[:,2]>self.config['MIN_SZ']]\n bboxes=bboxes[bboxes[:,3]>self.config['MIN_SZ']]\n bboxes=bboxes[bboxes[:,2] shape[1] - self.config['MIN_SZ']] -= 1.5*self.config['MIN_SZ']\n bboxes[bboxes[:,3] > shape[0] - self.config['MIN_SZ']] -= 1.5*self.config['MIN_SZ']\n return bboxes\n \n def __call__(self, img, mode = 'multi'):\n multi_bboxes, multi_confs, multi_labels = self.forward(img)\n bboxes, confs = self.ensemble(multi_bboxes, multi_confs, multi_labels, mode = mode)\n if len(bboxes) > 0:\n bboxes = self.check_result(self.voc2coco(bboxes,self.height, self.width), img.shape)\n \n return bboxes.tolist(), confs.tolist()\n \n def ensemble(self, boxes_list, scores_list, labels_list, mode, weights = None, iou = None):\n if weights is None:\n weights = [x['weight'] for x in self.config['MODELS']]\n IOU = self.config['IOU'] if iou is None else iou\n SKIP_BOX_THR = self.config['SKIP_BOX_THR']\n try:\n if mode == 'nms':\n boxes, scores, _ = nms(boxes_list, scores_list, labels_list, weights=weights, iou_thr=IOU)\n elif mode == 'soft_nms':\n boxes, scores, _ = soft_nms(boxes_list, scores_list, labels_list, weights=weights, iou_thr=IOU,\\\n sigma=self.config['SIGMA'], thresh=SKIP_BOX_THR)\n elif mode == 'nmw':\n boxes, scores, _ = non_maximum_weighted(boxes_list, scores_list, labels_list, weights=weights,\\\n iou_thr=IOU, skip_box_thr=SKIP_BOX_THR)\n elif mode == 'wbf':\n boxes, scores, _ = weighted_boxes_fusion(boxes_list, scores_list, labels_list, weights=weights,\\\n iou_thr=IOU, skip_box_thr=SKIP_BOX_THR)\n elif mode == 'multi':\n boxes1, scores1, labels1 = nms(boxes_list, scores_list, labels_list, weights = weights, iou_thr=IOU)\n boxes2, scores2, labels2 = soft_nms(boxes_list, scores_list, labels_list, weights=weights, iou_thr=IOU,\\\n sigma=self.config['SIGMA'], thresh=SKIP_BOX_THR)\n boxes3, scores3, labels3 = non_maximum_weighted(boxes_list, scores_list, labels_list, weights=weights,\\\n iou_thr=IOU, skip_box_thr=SKIP_BOX_THR)\n boxes4, scores4, labels4 = weighted_boxes_fusion(boxes_list, scores_list, labels_list, weights=weights,\\\n iou_thr=IOU, skip_box_thr=SKIP_BOX_THR)\n alg_weights = [2,3,4,5]\n boxes, scores, _ = weighted_boxes_fusion([boxes1, boxes2, boxes3, boxes4],\\\n [scores1, scores2, scores3, scores4],\\\n [labels1, labels2, labels3, labels4],\\\n weights=alg_weights, iou_thr=IOU*0.5, skip_box_thr=SKIP_BOX_THR)\n else:\n boxes, scores = np.array(boxes_list[0]), np.array(scores_list[0])\n\n boxes = np.array(boxes)*self.ratio\n except:\n boxes,scores = np.array([]), np.array([])\n \n return boxes,scores\n \n def to_norfair(self, detects, frame_id):\n result = []\n for x_min, y_min, x_max, y_max, score in detects:\n xc, yc = (x_min + x_max) / 2, (y_min + y_max) / 2\n w, h = x_max - x_min, y_max - y_min\n result.append(Detection(points=np.array([xc, yc]), scores=np.array([score]), data=np.array([w, h, frame_id])))\n\n return result\n\n # Euclidean distance function to match detections on this frame with tracked_objects from previous frames\n def euclidean_distance(self, detection, tracked_object):\n return np.linalg.norm(detection.points - tracked_object.estimate)\n \n def tracking_function(self, frame_id, bboxes, scores, ):\n \n detects = []\n bbboxes, confs = [], []\n if len(scores)>0:\n for i in range(len(bboxes)):\n box = bboxes[i]\n score = scores[i]\n x_min = int(box[0])\n y_min = int(box[1])\n bbox_width = int(box[2])\n bbox_height = int(box[3])\n detects.append([x_min, y_min, x_min+bbox_width, y_min+bbox_height, score])\n bbboxes.append([x_min, y_min, bbox_width, bbox_height])\n confs .append(score)\n # Update tracks using detects from current frame\n tracked_objects = self.tracker.update(detections=self.to_norfair(detects, frame_id))\n \n for tobj in tracked_objects:\n bbox_width, bbox_height, last_detected_frame_id = tobj.last_detection.data\n if last_detected_frame_id == frame_id: # Skip objects that were detected on current frame\n continue\n # Add objects that have no detections on current frame to predictions\n xc, yc = tobj.estimate[0]\n x_min, y_min = int(round(xc - bbox_width / 2)), int(round(yc - bbox_height / 2))\n score = tobj.last_detection.scores[0]\n\n bbboxes.append([x_min, y_min, bbox_width, bbox_height])\n confs .append(score)\n\n return bbboxes, confs\n \n def voc2yolo(self, bboxes, image_height=720, image_width=1280):\n \"\"\"\n voc => [x1, y1, x2, y1]\n yolo => [xmid, ymid, w, h] (normalized)\n \"\"\"\n\n bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n\n bboxes[..., [0, 2]] = bboxes[..., [0, 2]]/ image_width\n bboxes[..., [1, 3]] = bboxes[..., [1, 3]]/ image_height\n\n w = bboxes[..., 2] - bboxes[..., 0]\n h = bboxes[..., 3] - bboxes[..., 1]\n\n bboxes[..., 0] = bboxes[..., 0] + w/2\n bboxes[..., 1] = bboxes[..., 1] + h/2\n bboxes[..., 2] = w\n bboxes[..., 3] = h\n\n return bboxes\n \n def voc2yolo_nn(self, bboxes, image_height=720, image_width=1280):\n \"\"\"\n voc => [x1, y1, x2, y1]\n yolo => [xmid, ymid, w, h] \n \"\"\"\n\n bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n\n bboxes[..., [0, 2]] = bboxes[..., [0, 2]]\n bboxes[..., [1, 3]] = bboxes[..., [1, 3]]\n\n w = bboxes[..., 2] - bboxes[..., 0]\n h = bboxes[..., 3] - bboxes[..., 1]\n\n bboxes[..., 0] = bboxes[..., 0] + w/2\n bboxes[..., 1] = bboxes[..., 1] + h/2\n bboxes[..., 2] = w\n bboxes[..., 3] = h\n\n return bboxes\n\n def yolo2voc(self, bboxes, image_height=720, image_width=1280):\n \"\"\"\n yolo => [xmid, ymid, w, h] (normalized)\n voc => [x1, y1, x2, y2]\n\n \"\"\" \n bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n\n bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width\n bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height\n\n bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]\n\n return bboxes\n\n def coco2yolo(self, bboxes, image_height=720, image_width=1280):\n \"\"\"\n coco => [xmin, ymin, w, h]\n yolo => [xmid, ymid, w, h] (normalized)\n \"\"\"\n\n bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n\n # normolizinig\n bboxes[..., [0, 2]]= bboxes[..., [0, 2]]/ image_width\n bboxes[..., [1, 3]]= bboxes[..., [1, 3]]/ image_height\n\n # converstion (xmin, ymin) => (xmid, ymid)\n bboxes[..., [0, 1]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]/2\n\n return bboxes\n \n def coco2voc(self, bboxes, image_height=720, image_width=1280):\n bboxes = self.coco2yolo(bboxes, image_height, image_width)\n bboxes = self.yolo2voc(bboxes, image_height, image_width)\n return bboxes\n\n def yolo2coco(self, bboxes, image_height=720, image_width=1280):\n \"\"\"\n yolo => [xmid, ymid, w, h] (normalized)\n coco => [xmin, ymin, w, h]\n\n \"\"\" \n bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n\n # denormalizing\n bboxes[..., [0, 2]]= bboxes[..., [0, 2]]* image_width\n bboxes[..., [1, 3]]= bboxes[..., [1, 3]]* image_height\n\n # converstion (xmid, ymid) => (xmin, ymin) \n bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n\n return bboxes\n\n def voc2coco(self, bboxes, image_height=720, image_width=1280):\n bboxes = self.voc2yolo(bboxes, image_height, image_width)\n bboxes = self.yolo2coco(bboxes, image_height, image_width)\n return bboxes\n\n\n def plot_one_box(self, x, img, color=None, label=None, line_thickness=None):\n # Plots one bounding box on image img\n tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1 # line/font thickness\n color = color or [random.randint(0, 255) for _ in range(3)]\n c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\n cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)\n if label:\n tf = max(tl - 1, 1) # font thickness\n t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\n c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\n cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA) # filled\n cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)\n\n def draw_bboxes(self, img, bboxes, classes, class_ids, colors = None, show_classes = None, bbox_format = 'yolo', class_name = False, line_thickness = 2): \n\n image = img.copy()\n show_classes = classes if show_classes is None else show_classes\n colors = (0, 255 ,0) if colors is None else colors\n\n if bbox_format == 'yolo':\n\n for idx in range(len(bboxes)): \n\n bbox = bboxes[idx]\n cls = classes[idx]\n cls_id = class_ids[idx]\n color = colors[cls_id] if type(colors) is list else colors\n\n if cls in show_classes:\n\n x1 = round(float(bbox[0])*image.shape[1])\n y1 = round(float(bbox[1])*image.shape[0])\n w = round(float(bbox[2])*image.shape[1]/2) #w/2 \n h = round(float(bbox[3])*image.shape[0]/2)\n\n voc_bbox = (x1-w, y1-h, x1+w, y1+h)\n self.plot_one_box(voc_bbox, \n image,\n color = color,\n label = cls if class_name else str(get_label(cls)),\n line_thickness = line_thickness)\n\n elif bbox_format == 'coco':\n\n for idx in range(len(bboxes)): \n\n bbox = bboxes[idx]\n cls = classes[idx]\n cls_id = class_ids[idx]\n color = colors[cls_id] if type(colors) is list else colors\n\n if cls in show_classes: \n x1 = int(round(bbox[0]))\n y1 = int(round(bbox[1]))\n w = int(round(bbox[2]))\n h = int(round(bbox[3]))\n\n voc_bbox = (x1, y1, x1+w, y1+h)\n self.plot_one_box(voc_bbox, \n image,\n color = color,\n label = cls if class_name else str(cls_id),\n line_thickness = line_thickness)\n\n elif bbox_format == 'voc_pascal':\n\n for idx in range(len(bboxes)): \n\n bbox = bboxes[idx]\n cls = classes[idx]\n cls_id = class_ids[idx]\n color = colors[cls_id] if type(colors) is list else colors\n\n if cls in show_classes: \n x1 = int(round(bbox[0]))\n y1 = int(round(bbox[1]))\n x2 = int(round(bbox[2]))\n y2 = int(round(bbox[3]))\n voc_bbox = (x1, y1, x2, y2)\n self.plot_one_box(voc_bbox, \n image,\n color = color,\n label = cls if class_name else str(cls_id),\n line_thickness = line_thickness)\n else:\n raise ValueError('wrong bbox format')\n\n return image\n \n def decode_gt(self, gt_str):\n bboxes = []\n for i in gt_str:\n bboxes.append([x for x in i.values()])\n return np.array(bboxes, dtype = np.float32)\n\n def get_bbox(self, annots):\n bboxes = [list(annot.values()) for annot in annots]\n return bboxes\n\n def get_imgsize(self,row):\n row['width'], row['height'] = imagesize.get(row['image_path'])\n return row\n\n def format_prediction(self, bboxes, confs):\n annot = ''\n if len(bboxes)>0:\n for idx in range(len(bboxes)):\n xmin, ymin, w, h = bboxes[idx]\n conf = confs[idx]\n annot += f'{conf} {xmin} {ymin} {w} {h}'\n annot +=' '\n annot = annot.strip(' ')\n return annot\n \n def show_img(self, img, bboxes, gt_boxes = None, bbox_format='yolo'):\n names = ['starfish']*len(bboxes)\n labels = [0]*len(bboxes)\n if gt_boxes is not None:\n gt_boxes = self.decode_gt(gt_boxes)\n try:\n bboxes = np.concatenate([bboxes, gt_boxes], axis = 0)\n except:\n bboxes = gt_boxes\n names += ['gt_starfish']*len(gt_boxes)\n labels += [1]*len(gt_boxes)\n \n img = self.draw_bboxes(img = img,\n bboxes = bboxes, \n classes = names,\n class_ids = labels,\n class_name = True, \n colors = self.colors, \n bbox_format = bbox_format,\n line_thickness = 2)\n return Image.fromarray(img).resize((800, 400))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-11T21:31:28.648389Z","iopub.execute_input":"2022-02-11T21:31:28.648678Z","iopub.status.idle":"2022-02-11T21:31:28.795643Z","shell.execute_reply.started":"2022-02-11T21:31:28.648632Z","shell.execute_reply":"2022-02-11T21:31:28.794736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SingleFrameTiledPredictor(SingleFramePredictor):\n def __init__(self,*args, **kwargs):\n super().__init__(*args, **kwargs)\n\n def _yolov5_forward(self, imgs):\n results = model(imgs, size = self.config['IMG_SIZE']) # custom inference size\n preds = results.pandas().xyxy\n bboxes, confs, labels = [], [], []\n for i in range(len(preds)):\n bboxes = preds[i][['xmin','ymin','xmax','ymax']].values\n if not len(bboxes):\n bboxes.append(np.array([[0.,0.,0.,0.]]));confs.append(np.array([0.]));labels.append(np.array([0]))\n continue\n \n confs.append(preds.confidence.values)\n labels.append(np.zeros_like(confs))\n bboxes, confs, labels = self.ensemble(bboxes + self.tiles[i], confs, labels, mode = 'nms')\n return bboxes, confs, labels\n \n def __call__(self, img, mode= 'multi'):\n img = self.split_tile(img, 360, 360,0.2,0.2)\n return super().__call__(img, mode)\n \n def split_tile(self, img, tile_h, tile_w, h_overlap, w_overlap):\n images = []\n self.tiles = []\n for i in np.ceil(np.arange(0, self.heigth+1-tile_h, (1 - h_overlap)*tile_h)).astype(int):\n if i + 2*tile_h > self.height:\n tmp_img = img[i:self.heights, :, :]\n else:\n tmp_img = img[i:i+tile_h, :, :]\n \n for j in np.ceil(np.arange(0,self.width+1-tile_w, (1- w_overlap)*tile_w)).astype(int):\n if j + 2*tile_w > self.width:\n tmp_img = tmp_img[:, j:self.width, :]\n else:\n tmp_img = tmp_img[:,j:j+tile_w, :]\n \n images.append(tmp_img)\n self.tiles.append([i, j]*2) \n self.tiles = np.array(self.tiles)\n return images\n\nclass MultiFramePredictor(SingleFramePredictor):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n self.maxlen = self.config['MAXLEN']\n self.img_deque = Deque(self.maxlen-1)\n self.bb_deque = Deque(self.maxlen-1)\n self.deep_sort_load()\n \n def deep_sort_load(self):\n cfg = get_config()\n cfg.merge_from_file('/kaggle/working/deepsort.yaml') \n self.deepsort = DeepSort(cfg.DEEPSORT.REID_CKPT,\n max_dist=cfg.DEEPSORT.MAX_DIST,\n min_confidence=cfg.DEEPSORT.MIN_CONFIDENCE,\n nms_max_overlap=cfg.DEEPSORT.NMS_MAX_OVERLAP,\n max_iou_distance=cfg.DEEPSORT.MAX_IOU_DISTANCE,\n max_age=cfg.DEEPSORT.MAX_AGE,\n n_init=cfg.DEEPSORT.N_INIT,\n nn_budget=cfg.DEEPSORT.NN_BUDGET,\n use_cuda=True)\n \n \n def __call__(self, img, mode = 'multi'):\n multi_bboxes, multi_confs, multi_labels = self.forward(img)\n bboxes, confs = self.ensemble(multi_bboxes, multi_confs, multi_labels, mode = mode)\n if not len(bboxes):\n bboxes, confs = np.array([[0.,0.,0.,0.]]) , np.array([0.])\n \n res = np.hstack([bboxes, np.expand_dims(confs, 0).T]).astype(np.float32)\n if not len(self.bb_deque):\n self.bb_deque.append(res)\n bboxes, confs = self.dsnms(list(self.bb_deque)+[res])\n \n if len(bboxes[-1]):\n bboxes, confs = np.stack(bboxes[-1]), confs[-1]\n if self.config['DEEPSORT']:\n bboxes, confs = self.deep_sort(bboxes, confs, img)\n self.bb_deque.append(np.hstack([bboxes, np.array([confs]).T]).astype(np.float32))\n return self.voc2coco(bboxes,self.height, self.width).tolist(), confs\n else:\n bboxes, confs = np.array([[0.,0.,0.,0.]]), np.array([0.])\n self.bb_deque.append(np.hstack([bboxes, np.array([confs]).T]).astype(np.float32))\n return [],[] \n \n def deep_sort(self, bboxes, confs, img):\n clss = np.zeros_like(confs)\n outputs = np.array(self.deepsort.update(self.voc2yolo_nn(bboxes), confs, clss, img))\n if not len(outputs) and len(bboxes):\n return bboxes, confs\n confs = confs[:len(outputs)]\n if len(outputs) > len(confs):\n confs += np.ones(len(outputs) - len(confs)).astype('float32').tolist()\n return outputs[:,:-2], confs\n \n def reset_frames(self):\n self.img_deque = Deque(self.maxlen-1)\n self.bb_deque = Deque(self.maxlen-1)\n self.deepsort.tracker.tracks = []\n self.deepsort.tracker._next_id = 1\n\n def createLinks(self, dets_all):\n links_all=[]\n frame_num=len(dets_all)\n for frame_ind in range(frame_num-1): \n dets1=dets_all[frame_ind]\n dets2=dets_all[frame_ind+1]\n box1_num=len(dets1)\n box2_num=len(dets2)\n if frame_ind==0:\n areas1=np.empty(box1_num)\n for box1_ind,box1 in enumerate(dets1):\n areas1[box1_ind]=(box1[2]-box1[0]+1)*(box1[3]-box1[1]+1)\n else: \n areas1=areas2\n areas2=np.empty(box2_num)\n for box2_ind,box2 in enumerate(dets2):\n areas2[box2_ind]=(box2[2]-box2[0]+1)*(box2[3]-box2[1]+1)\n links_frame=[]\n for box1_ind,box1 in enumerate(dets1):\n area1=areas1[box1_ind]\n x1=np.maximum(box1[0],dets2[:,0])\n y1=np.maximum(box1[1],dets2[:,1])\n x2=np.minimum(box1[2],dets2[:,2])\n y2=np.minimum(box1[3],dets2[:,3])\n w =np.maximum(0.0, x2 - x1 + 1)\n h =np.maximum(0.0, y2 - y1 + 1)\n inter = w * h\n ovrs = inter / (area1 + areas2 - inter)\n links_box=[ovr_ind for ovr_ind,ovr in enumerate(ovrs) if ovr >= self.config['IOU_THRESH']]\n links_frame.append(links_box)\n links_all.append(links_frame)\n return links_all\n\n def maxPath(self,dets_all,links_all):\n while True:\n rootindex,maxpath,maxsum=self.findMaxPath(links_all,dets_all)\n if len(maxpath) <= 1 or rootindex == -1:\n break\n dets_all = self.rescore(dets_all,rootindex,maxpath,maxsum)\n dets_all,links_all = self.deleteLink(dets_all,links_all,rootindex,maxpath,self.config['IOU_THRESH_DELETE'])\n return dets_all,links_all\n\n def nms(self, dets, thresh):\n \"\"\"Pure Python NMS baseline.\"\"\"\n x1 = dets[:, 0]\n y1 = dets[:, 1]\n x2 = dets[:, 2]\n y2 = dets[:, 3]\n scores = dets[:, 4]\n areas = (x2 - x1 + 1) * (y2 - y1 + 1)\n order = scores.argsort()[::-1]\n keep = []\n while order.size > 0:\n i = order[0]\n keep.append(i)\n xx1 = np.maximum(x1[i], x1[order[1:]])\n yy1 = np.maximum(y1[i], y1[order[1:]])\n xx2 = np.minimum(x2[i], x2[order[1:]])\n yy2 = np.minimum(y2[i], y2[order[1:]])\n w = np.maximum(0.0, xx2 - xx1 + 1)\n h = np.maximum(0.0, yy2 - yy1 + 1)\n inter = w * h\n ovr = inter / (areas[i] + areas[order[1:]] - inter)\n inds = np.where(ovr <= thresh)[0]\n order = order[inds + 1]\n return keep\n\n def NMS(self, dets_all):\n for frame_ind,dets in enumerate(dets_all):\n keep=self.nms(dets, self.config['NMS_THRESH'])\n dets_all[frame_ind]=dets[keep, :]\n \n return dets_all\n\n def findMaxPath(self,links,dets):\n maxpaths=[]\n roots=[]\n maxpaths.append([ (box[4],[ind]) for ind,box in enumerate(dets[-1])])\n for link_ind,link in enumerate(links[::-1]):\n curmaxpaths=[]\n linkflags=np.zeros(len(maxpaths[0]),int)\n det_ind=len(links)-link_ind-1\n for ind,linkboxes in enumerate(link):\n if linkboxes == []:\n curmaxpaths.append((dets[det_ind][ind][4],[ind]))\n continue\n linkflags[linkboxes]=1\n prev_ind=np.argmax([maxpaths[0][linkbox][0] for linkbox in linkboxes])\n prev_score=maxpaths[0][linkboxes[prev_ind]][0]\n prev_path=copy.copy(maxpaths[0][linkboxes[prev_ind]][1])\n prev_path.insert(0,ind)\n curmaxpaths.append((dets[det_ind][ind][4]+prev_score,prev_path))\n root=[maxpaths[0][ind] for ind,flag in enumerate(linkflags) if flag == 0]\n roots.insert(0,root)\n maxpaths.insert(0,curmaxpaths)\n roots.insert(0,maxpaths[0])\n maxscore=0\n maxpath=[]\n for index,paths in enumerate(roots):\n if paths==[]:\n continue\n maxindex=np.argmax([path[0] for path in paths])\n if paths[maxindex][0]>maxscore:\n maxscore=paths[maxindex][0]\n maxpath=paths[maxindex][1]\n rootindex=index\n try:\n return rootindex,maxpath,maxscore\n except:\n return -1, [], -1\n \n\n def rescore(self, dets, rootindex, maxpath, maxsum):\n newscore=maxsum/len(maxpath)\n for i,box_ind in enumerate(maxpath):\n dets[rootindex+i][box_ind][4]=newscore\n return dets\n\n def deleteLink(self, dets,links, rootindex, maxpath,thesh):\n for i,box_ind in enumerate(maxpath):\n areas=[(box[2]-box[0]+1)*(box[3]-box[1]+1) for box in dets[rootindex+i]]\n area1=areas[box_ind]\n box1=dets[rootindex+i][box_ind]\n x1=np.maximum(box1[0],dets[rootindex+i][:,0])\n y1=np.maximum(box1[1],dets[rootindex+i][:,1])\n x2=np.minimum(box1[2],dets[rootindex+i][:,2])\n y2=np.minimum(box1[3],dets[rootindex+i][:,3])\n w =np.maximum(0.0, x2 - x1 + 1)\n h =np.maximum(0.0, y2 - y1 + 1)\n inter = w * h\n ovrs = inter / (area1 + areas - inter)\n deletes=[ovr_ind for ovr_ind,ovr in enumerate(ovrs) if ovr >= thesh] \n for delete_ind in deletes:\n if delete_ind!=box_ind:\n dets[rootindex+i][delete_ind, 4] = 0\n if rootindex+i 0 or rootindex>0:\n for priorbox in links[rootindex+i-1]:\n for delete_ind in deletes:\n if delete_ind in priorbox:\n priorbox.remove(delete_ind)\n return dets, links\n\n def dsnms(self, dets):\n links=self.createLinks(dets)\n dets, links = self.maxPath(dets,links)\n dets = self.NMS(dets)\n boxes=[[] for i in dets]\n scores=[[] for i in dets]\n for frame_id, frame in enumerate(dets):\n for box_id, box in enumerate(frame):\n if box[4] >= self.config['CONF_THRESH']:\n ymin = box[1]\n xmin = box[0]\n ymax = box[3]\n xmax = box[2]\n boxes[frame_id].append(np.array([xmin, ymin, xmax, ymax]))\n scores[frame_id].append(box[4])\n return boxes, scores\n \nclass MultiFrameTiledPredictor(SingleFrameTiledPredictor, MultiFramePredictor):\n def __init__(self,*args, **kwargs):\n super().__init__(*args, **kwargs)\n self.img_deque = Deque(5)\n self.bb_deque = Deque(5)\n \n def __call__(self, img, mode = 'multi'):\n return super().__call__(img, mode)","metadata":{"execution":{"iopub.status.busy":"2022-02-11T21:31:29.480915Z","iopub.execute_input":"2022-02-11T21:31:29.481242Z","iopub.status.idle":"2022-02-11T21:31:29.552817Z","shell.execute_reply.started":"2022-02-11T21:31:29.481208Z","shell.execute_reply":"2022-02-11T21:31:29.552109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference train","metadata":{}},{"cell_type":"code","source":"predict = eval(CONFIG['INFERENCE_TYPE'])(CONFIG)\npredict.reset()\ntmp = df[df['num_bbox'] > 11].reset_index(drop =True)\nimage_paths = tmp.image_path.tolist()\nannotations = tmp.annotations.tolist()\nframe_id = 0\ntmp['pred'] = ''\nfor idx, path in tqdm(enumerate(image_paths), total = len(image_paths)):\n img = cv2.imread(path)[...,::-1]\n bboxes, confis = predict(img, mode = CONFIG['ENSEMBLE'])\n if len(bboxes) > 0:\n if CONFIG['NORFAIR']:\n bboxes, confis = predict.tracking_function(frame_id, bboxes, confis)\n bboxes, confis = predict.ensemble([(predict.coco2voc(np.array(bboxes))/(predict.ratio)).tolist()], [confis], [[0]*len(confis)], mode = 'multi', weights = [1])\n bboxes = predict.voc2coco(bboxes).tolist()\n try:\n annot = predict.format_prediction(bboxes.tolist(), confis.tolist())\n except:\n annot = predict.format_prediction(bboxes, confis)\n tmp.loc[idx, 'pred'] = annot\n display(predict.show_img(img, bboxes, bbox_format='coco'))\n frame_id += 1\n if idx>10:\n break\ntmp.to_csv('train_pred.csv', index = True)","metadata":{"execution":{"iopub.status.busy":"2022-02-11T21:31:30.107902Z","iopub.execute_input":"2022-02-11T21:31:30.108175Z","iopub.status.idle":"2022-02-11T21:32:10.018989Z","shell.execute_reply.started":"2022-02-11T21:31:30.108146Z","shell.execute_reply":"2022-02-11T21:32:10.017817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'Multi' in CONFIG['INFERENCE_TYPE']:\n predict.reset_frames()\npredict.reset()\nframe_id = 0\nfor idx, path in enumerate(image_paths):\n img = cv2.imread(path)[...,::-1]\n bboxes, confis = predict(img, mode = CONFIG['ENSEMBLE'])\n if len(bboxes) > 0:\n if CONFIG['NORFAIR']:\n bboxes, confis = predict.tracking_function(frame_id, bboxes, confis)\n bboxes, confis = predict.ensemble([(predict.coco2voc(np.array(bboxes))/(predict.ratio)).tolist()], [confis], [[0]*len(confis)], mode = 'wbf', weights = [1])\n bboxes = predict.voc2coco(bboxes).tolist()\n display(predict.show_img(img, bboxes,annotations[idx], bbox_format='coco'))\n frame_id += 1\n if idx>10:\n break","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-11T21:32:10.020897Z","iopub.execute_input":"2022-02-11T21:32:10.021229Z","iopub.status.idle":"2022-02-11T21:32:46.902486Z","shell.execute_reply.started":"2022-02-11T21:32:10.02119Z","shell.execute_reply":"2022-02-11T21:32:46.901795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference test","metadata":{}},{"cell_type":"code","source":"frame_id = 0\nlast_video_id = -1\ntest_ = pd.read_csv('../input/tensorflow-great-barrier-reef/test.csv')\nfor idx, (img, pred_df) in enumerate(tqdm(iter_test)):\n if 'Multi' in CONFIG['INFERENCE_TYPE']:\n tmp = test_.iloc[pred_df.iloc[0,0], 1]\n if tmp != last_video_id:\n predict.reset_frames()\n predict.reset()\n last_video_id = tmp\n if CONFIG['NORFAIR']:\n tmp = test_.iloc[pred_df.iloc[0,0], 0]\n if tmp != last_video_id:\n predict.reset()\n last_video_id = tmp\n frame_id = 0\n \n bboxes, confis = predict(img, mode = CONFIG['ENSEMBLE'])\n if len(bboxes) > 0:\n if CONFIG['NORFAIR']:\n bboxes, confis = predict.tracking_function(frame_id, bboxes, confis)\n bboxes, confis = predict.ensemble([(predict.coco2voc(np.array(bboxes))/(predict.ratio)).tolist()], [confis], [[0]*len(confis)], mode = 'wbf', weights = [1])\n bboxes = predict.voc2coco(bboxes).tolist()\n try:\n annot = predict.format_prediction(bboxes.tolist(), confis.tolist())\n except:\n annot = predict.format_prediction(bboxes, confis)\n pred_df['annotations'] = annot\n env.predict(pred_df)\n frame_id += 1\n if idx<3:\n display(predict.show_img(img, bboxes, bbox_format='coco'))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-11T21:32:46.903811Z","iopub.execute_input":"2022-02-11T21:32:46.904153Z","iopub.status.idle":"2022-02-11T21:32:46.999737Z","shell.execute_reply.started":"2022-02-11T21:32:46.904121Z","shell.execute_reply":"2022-02-11T21:32:46.998731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check Submission","metadata":{}},{"cell_type":"code","source":"sub_df = pd.read_csv('/kaggle/working/submission.csv')\nsub_df.head(1)['annotations'].values","metadata":{"execution":{"iopub.status.busy":"2022-02-05T06:08:03.964406Z","iopub.execute_input":"2022-02-05T06:08:03.965199Z","iopub.status.idle":"2022-02-05T06:08:03.979144Z","shell.execute_reply.started":"2022-02-05T06:08:03.965131Z","shell.execute_reply":"2022-02-05T06:08:03.977767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r /kaggle/working/yolox-cots-models\n!rm -r /kaggle/working/yolor-repo","metadata":{"execution":{"iopub.status.busy":"2022-02-05T06:08:03.980866Z","iopub.execute_input":"2022-02-05T06:08:03.981963Z","iopub.status.idle":"2022-02-05T06:08:04.653383Z","shell.execute_reply.started":"2022-02-05T06:08:03.981915Z","shell.execute_reply":"2022-02-05T06:08:04.65217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}