{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## References\n* Evaluate F2 for each CONF level in validation dataset using the Yolov5 model.\n* The \"F2\" is the comptetation metric, IOU @ 0.3 to 0.8 with step of 0.05.\n\n\n## Used model\nI use sheep's model.\n* https://www.kaggle.com/steamedsheep/yolov5-is-all-you-need\n* https://www.kaggle.com/steamedsheep/reef-baseline-fold12\n\n\n\n\n\nIf you find any bugs or mistakes, please let me know.\nThank you.","metadata":{}},{"cell_type":"code","source":"\nimport numpy as np \nimport pandas as pd \nimport torch\nfrom tqdm import tqdm\nimport sys\nimport cv2\n\nimport ast\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nsys.path.append('../input/tensorflow-great-barrier-reef')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working\nckpt_path = '../input/yolov5-002/YOLOv5/yolov5-3600-sheepsame-001-fold-1/weights/best.pt'\ninfer_size = 5000\nselect_fold = 1\nmodel = torch.hub.load('../input/yolov5-lib-ds', \n                       'custom', \n                       path=ckpt_path,\n                       source='local',\n                       force_reload=True)  # local repo\nmodel.conf = 0.01\n\n# model.iou  = 0.65 ####","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def IOU_coco(bbox1, bbox2):\n    '''\n        adapted from https://stackoverflow.com/questions/25349178/calculating-percentage-of-bounding-box-overlap-for-image-detector-evaluation\n    '''\n    x_left = max(bbox1[0], bbox2[0])\n    y_top = max(bbox1[1], bbox2[1])\n    x_right = min(bbox1[0] + bbox1[2], bbox2[0] + bbox2[2])\n    y_bottom = min(bbox1[1] + bbox1[3], bbox2[1] + bbox2[3])\n    if x_right < x_left or y_bottom < y_top:\n        return 0.0\n    intersection_area = (x_right - x_left) * (y_bottom - y_top)\n    bb1_area = bbox1[2] * bbox1[3]\n    bb2_area = bbox2[2] * bbox2[3]\n    iou = intersection_area / float(bb1_area + bb2_area - intersection_area)\n\n    assert iou >= 0.0\n    assert iou <= 1.0\n    return iou","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working\n\nfrom sklearn.model_selection import GroupKFold\n# import ast\n\ndef get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndef get_path(row):\n    row['image_path'] = f'{ROOT_DIR}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    return row\n\nROOT_DIR  = '/kaggle/input/tensorflow-great-barrier-reef/'\n\ndf = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\n\n# Don't filter for annotated frames. Include frames with no bboxes as well!\ndf[\"num_bbox\"] = df['annotations'].apply(lambda x: str.count(x, 'x'))\ndf_train = df\n\n# Annotations \ndf_train['annotations'] = df_train['annotations'].progress_apply(lambda x: ast.literal_eval(x))\ndf_train['bboxes'] = df_train.annotations.progress_apply(get_bbox)\n\ndf_train = df_train.progress_apply(get_path, axis=1)\n\n# kf = GroupKFold(n_splits = 5) \n# df_train = df_train.reset_index(drop=True)\n# df_train['fold'] = -1\n# for fold, (train_idx, val_idx) in enumerate(kf.split(df_train, y = df_train.video_id.tolist(), groups=df_train.sequence)):\n#     df_train.loc[val_idx, 'fold'] = fold\n\ndf_train.head(5)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select_fold = 1\ndf_test = df_train[df_train.video_id == select_fold]\nprint(len(df_test))\ndf_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path = df_test[df_test.num_bbox>3].image_path.tolist()[0]\nimg = cv2.imread(image_path)[...,::-1]\nr = model(img, size=infer_size, augment=False)\nr.save(\"/kaggle/working\")\nbbox_img = cv2.imread(\"/kaggle/working/image0.jpg\")[...,::-1]\nfrom PIL import Image\ndisplay(Image.fromarray(cv2.resize(bbox_img, dsize=(640,360))))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.query(\"num_bbox>0\").head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import List\n\nimport numpy as np\nimport torch\nfrom torchvision.ops import box_iou\n\n\ndef calculate_score(\n    preds: List[torch.Tensor],\n    gts: List[torch.Tensor],\n    iou_th: float\n) -> float:\n    num_tp = 0\n    num_fp = 0\n    num_fn = 0\n    for p, gt in zip(preds, gts):\n        if len(p) and len(gt):\n            iou_matrix = box_iou(p, gt)\n            tp = len(torch.where(iou_matrix.max(0)[0] >= iou_th)[0])\n            fp = len(p) - tp\n            fn = len(torch.where(iou_matrix.max(0)[0] < iou_th)[0])\n            num_tp += tp\n            num_fp += fp\n            num_fn += fn\n        elif len(p) == 0 and len(gt):\n            num_fn += len(gt)\n        elif len(p) and len(gt) == 0:\n            num_fp += len(p)\n    score = 5 * num_tp / (5 * num_tp + 4 * num_fn + num_fp)\n    return score\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import copy\n\ndf_sample = df_test\nimage_paths = df_sample.image_path.tolist()\ngt = copy.deepcopy(df_sample.bboxes.tolist())\ngtmem = copy.deepcopy(df_sample.bboxes.tolist())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%cd /kaggle/working\nimport cv2\n\npreds_list = [] # Confidence scores of true positives\ngts_list = [] # Confidence scores of true positives\n\nfor i in tqdm(range(len(image_paths))):\n# for i in tqdm(range(470, 500)):\n# for i in tqdm(range(475, 480)):\n    TEST_IMAGE_PATH = image_paths[i]\n    img = cv2.imread(TEST_IMAGE_PATH)[...,::-1]\n    h, w, _ = img.shape\n    gts = []\n    for j in gt[i]: # [[x,y,w,h],...]\n        gts.append([j[0], j[1], j[0]+j[2], j[1]+j[3]])\n    r = model(img, size=infer_size, augment=False)\n    preds_list.append(r.pandas().xyxy[0])\n    gts_list.append(torch.Tensor(gts))\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nF2list = []\nF2max = 0.0\nF2maxat = -1.0\nfor conf in tqdm(np.arange(0.0, 1.0, 0.01)):\n    preds_conf = [torch.Tensor(i.query(\"confidence>@conf\")[['xmin','ymin','xmax','ymax']].values) if len(i)!=0 else torch.tensor([]) for i in preds_list ]\n    iou_ths = np.arange(0.3, 0.85, 0.05)\n#     iou_ths = [0.65,]\n    F2 = np.mean([calculate_score(preds_conf, gts_list, iou_th) for iou_th in iou_ths])\n#     print(score)\n    F2list.append((conf, F2))\n    if F2max < F2:\n        F2max = F2\n        F2maxat = conf\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\nfrom matplotlib import pyplot as plt\nplt.scatter(*zip(*F2list))\nplt.title(\"CONF vs F2 score\")\nplt.xlabel('CONF')\nplt.ylabel('F2')\nplt.show()\nprint(f'F2 max is {F2max} at CONF = {F2maxat}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}