{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"V2: Fixed corner case in image-wise metric and also added global f2 score implementation.  \n    Thanks for pointing out @lobrien\n    \nV3: Fixed a stupid bug in fbeta score calculation, thanks @philippsinger!\n\nV7: There was still bug in the function, fixed by @yoichi7yamakawa. Thanks for pointing out!","metadata":{}},{"cell_type":"code","source":"import glob\nimport os\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nplt.rcParams['figure.figsize'] = 30, 30\nnp.set_printoptions(precision=3, suppress=True)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-22T14:07:56.417548Z","iopub.execute_input":"2021-12-22T14:07:56.417956Z","iopub.status.idle":"2021-12-22T14:07:56.424473Z","shell.execute_reply.started":"2021-12-22T14:07:56.417906Z","shell.execute_reply":"2021-12-22T14:07:56.423355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/tensorflow-great-barrier-reef/train.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:07:56.427039Z","iopub.execute_input":"2021-12-22T14:07:56.427635Z","iopub.status.idle":"2021-12-22T14:07:56.478785Z","shell.execute_reply.started":"2021-12-22T14:07:56.427581Z","shell.execute_reply":"2021-12-22T14:07:56.477938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## visualization snnipet","metadata":{}},{"cell_type":"code","source":"from ast import literal_eval\n\n\ndef load_image(video_id, video_frame, image_dir):\n    img_path = f'{image_dir}/video_{video_id}/{video_frame}.jpg'\n    assert os.path.exists(img_path), f'{img_path} does not exist.'\n    img = cv2.imread(img_path)\n    return img\n\n\ndef decode_annotations(annotaitons_str):\n    \"\"\"decode annotations in string to list of dict\"\"\"\n    return literal_eval(annotaitons_str)\n\ndef load_image_with_annotations(video_id, video_frame, image_dir, annotaitons_str):\n    img = load_image(video_id, video_frame, image_dir)\n    annotations = decode_annotations(annotaitons_str)\n    if len(annotations) > 0:\n        for ann in annotations:\n            cv2.rectangle(img, (ann['x'], ann['y']),\n                (ann['x'] + ann['width'], ann['y'] + ann['height']),\n                (255, 0, 0), thickness=2,)\n    return img\n\ndef draw_predictions(img, pred_bboxes):\n    img = img.copy()\n    if len(pred_bboxes) > 0:\n        for bbox in pred_bboxes:\n            conf = bbox[0]\n            x, y, w, h = bbox[1:].round().astype(int)\n            cv2.rectangle(img, (x, y),\n                (x+w, y+h),\n                (0, 255, 255), thickness=2,)\n            cv2.putText(\n                img,\n                f\"{conf:.2}\",\n                (x, max(0, y-5)),\n                cv2.FONT_HERSHEY_SIMPLEX,\n                0.5,\n                (0, 0, 255),\n                thickness=1,\n            )\n    return img\n\n# test\n# index = 0\nindex = 19668\n# index = 16\nrow = df.iloc[index]\nvideo_id = row.video_id\nvideo_frame = row.video_frame\nannotations_str = row.annotations\nimage_dir = '../input/tensorflow-great-barrier-reef/train_images'\nimg = load_image_with_annotations(video_id, video_frame, image_dir, annotations_str)\nplt.imshow(img[:, :, ::-1])","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:07:56.480042Z","iopub.execute_input":"2021-12-22T14:07:56.480745Z","iopub.status.idle":"2021-12-22T14:07:58.397185Z","shell.execute_reply.started":"2021-12-22T14:07:56.480689Z","shell.execute_reply":"2021-12-22T14:07:58.396239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## generate ground truth and dummy prediction","metadata":{}},{"cell_type":"code","source":"def generate_gt_and_pred(annotations_str):\n    annotations = decode_annotations(annotations_str)\n\n    gt_bboxes = []\n    pred_bboxes = []\n\n    for ann in annotations:\n        gt_bboxes.append(np.array([ann['x'], ann['y'], ann['width'], ann['height']]))\n\n        # pseudo pred bbox\n        conf = np.random.uniform()\n#         noise = (np.random.randn(4)*5).round()\n        pred_bbox = np.array([conf, ann['x'], ann['y'], ann['width']*1.25, ann['height']*1.25])\n#         pred_bbox[1:] = pred_bbox[1:] + noise\n        pred_bboxes.append(pred_bbox)\n\n    gt_bboxes = np.array(gt_bboxes)\n    pred_bboxes = np.array(pred_bboxes)\n    return gt_bboxes, pred_bboxes\n\nannotations_str = df.iloc[index]['annotations']\ngt_bboxes, pred_bboxes = generate_gt_and_pred(annotations_str)\n\nif len(pred_bboxes) > 0:\n    pred_bboxes = pred_bboxes[pred_bboxes[:,0].argsort()[::-1]] # sort by conf\n    pred_img = draw_predictions(img, pred_bboxes)\n    plt.imshow(pred_img[:, :, ::-1])","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:07:58.398516Z","iopub.execute_input":"2021-12-22T14:07:58.398982Z","iopub.status.idle":"2021-12-22T14:08:00.275465Z","shell.execute_reply.started":"2021-12-22T14:07:58.398945Z","shell.execute_reply":"2021-12-22T14:08:00.274758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## image wise metric implementation\n\nf2 score is the case of beta=2 in fbeta score.  \nhttps://en.wikipedia.org/wiki/F-score\n\n![](https://wikimedia.org/api/rest_v1/media/math/render/svg/6fbeb471033fdd63a2c2ca7830afc7abdf8b8134)","metadata":{}},{"cell_type":"code","source":"def calc_iou(bboxes1, bboxes2, bbox_mode='xywh'):\n    assert len(bboxes1.shape) == 2 and bboxes1.shape[1] == 4\n    assert len(bboxes2.shape) == 2 and bboxes2.shape[1] == 4\n    \n    bboxes1 = bboxes1.copy()\n    bboxes2 = bboxes2.copy()\n    \n    if bbox_mode == 'xywh':\n        bboxes1[:, 2:] += bboxes1[:, :2]\n        bboxes2[:, 2:] += bboxes2[:, :2]\n\n    x11, y11, x12, y12 = np.split(bboxes1, 4, axis=1)\n    x21, y21, x22, y22 = np.split(bboxes2, 4, axis=1)\n    xA = np.maximum(x11, np.transpose(x21))\n    yA = np.maximum(y11, np.transpose(y21))\n    xB = np.minimum(x12, np.transpose(x22))\n    yB = np.minimum(y12, np.transpose(y22))\n    interArea = np.maximum((xB - xA + 1), 0) * np.maximum((yB - yA + 1), 0)\n    boxAArea = (x12 - x11 + 1) * (y12 - y11 + 1)\n    boxBArea = (x22 - x21 + 1) * (y22 - y21 + 1)\n    iou = interArea / (boxAArea + np.transpose(boxBArea) - interArea)\n    return iou\n\ndef f_beta(tp, fp, fn, beta=2):\n    return (1+beta**2)*tp / ((1+beta**2)*tp + beta**2*fn+fp)\n\ndef imagewise_f2_score_at_iou_th(gt_bboxes, pred_bboxes, iou_th, verbose=False):\n    gt_bboxes = gt_bboxes.copy()\n    pred_bboxes = pred_bboxes.copy()\n\n    tp = 0\n    fp = 0\n    for k, pred_bbox in enumerate(pred_bboxes): # fixed in ver.7\n        ious = calc_iou(gt_bboxes, pred_bbox[None, 1:])\n        max_iou = ious.max()\n        if max_iou > iou_th:\n            tp += 1\n            gt_bboxes = np.delete(gt_bboxes, ious.argmax(), axis=0)\n        else:\n            fp += 1\n        if len(gt_bboxes) == 0:\n            fp += len(pred_bboxes) - (k + 1) # fix in ver.7\n            break\n\n    fn = len(gt_bboxes)\n    score = f_beta(tp, fp, fn, beta=2)\n    if verbose:\n        print(f'iou_th:{iou_th.round(2):<4} tp:{tp:<2}, fp:{fp:<2}, fn:{fn:<2} f2:{score:.3}')\n    return score\n\n\n\ndef imagewise_f2_score(gt_bboxes, pred_bboxes, verbose=False):\n    \"\"\"\n    gt_bboxes: (N, 4) np.array in xywh format\n    pred_bboxes: (N, 5) np.array in conf+xywh format\n    \"\"\"\n    # v2: add corner case hundling.\n    if len(gt_bboxes) == 0 and len(pred_bboxes) == 0:\n        return 1.0\n    elif len(gt_bboxes) == 0 or len(pred_bboxes) == 0:\n        return 0.0\n    \n    pred_bboxes = pred_bboxes[pred_bboxes[:,0].argsort()[::-1]] # sort by conf\n    \n    scores = []\n    for iou_th in np.arange(0.3, 0.85, 0.05):\n        scores.append(imagewise_f2_score_at_iou_th(gt_bboxes, pred_bboxes, iou_th, verbose))\n    return np.mean(scores)","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:08:00.277577Z","iopub.execute_input":"2021-12-22T14:08:00.27798Z","iopub.status.idle":"2021-12-22T14:08:00.29937Z","shell.execute_reply.started":"2021-12-22T14:08:00.277931Z","shell.execute_reply":"2021-12-22T14:08:00.298365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imagewise_f2_score(gt_bboxes, pred_bboxes, verbose=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:08:00.300599Z","iopub.execute_input":"2021-12-22T14:08:00.300825Z","iopub.status.idle":"2021-12-22T14:08:00.355413Z","shell.execute_reply.started":"2021-12-22T14:08:00.300797Z","shell.execute_reply":"2021-12-22T14:08:00.354435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# V2: Global f2 metric implementation\nIn version 1 I've implemented image-wise metric, but competition metric shoud gather tp/fp/fn over all images and then calculate f2 score.  \nIt means above metric is somehow wrong, but I'll keep it here as it might be useful for someone want to calculate image-wise metric.\n\nAnyway, let's implement correct metric:)","metadata":{}},{"cell_type":"code","source":"def calc_iou(bboxes1, bboxes2, bbox_mode='xywh'):\n    assert len(bboxes1.shape) == 2 and bboxes1.shape[1] == 4\n    assert len(bboxes2.shape) == 2 and bboxes2.shape[1] == 4\n    \n    bboxes1 = bboxes1.copy()\n    bboxes2 = bboxes2.copy()\n    \n    if bbox_mode == 'xywh':\n        bboxes1[:, 2:] += bboxes1[:, :2]\n        bboxes2[:, 2:] += bboxes2[:, :2]\n\n    x11, y11, x12, y12 = np.split(bboxes1, 4, axis=1)\n    x21, y21, x22, y22 = np.split(bboxes2, 4, axis=1)\n    xA = np.maximum(x11, np.transpose(x21))\n    yA = np.maximum(y11, np.transpose(y21))\n    xB = np.minimum(x12, np.transpose(x22))\n    yB = np.minimum(y12, np.transpose(y22))\n    interArea = np.maximum((xB - xA + 1), 0) * np.maximum((yB - yA + 1), 0)\n    boxAArea = (x12 - x11 + 1) * (y12 - y11 + 1)\n    boxBArea = (x22 - x21 + 1) * (y22 - y21 + 1)\n    iou = interArea / (boxAArea + np.transpose(boxBArea) - interArea)\n    return iou\n\ndef f_beta(tp, fp, fn, beta=2):\n    return (1+beta**2)*tp / ((1+beta**2)*tp + beta**2*fn+fp)\n\ndef calc_is_correct_at_iou_th(gt_bboxes, pred_bboxes, iou_th, verbose=False):\n    gt_bboxes = gt_bboxes.copy()\n    pred_bboxes = pred_bboxes.copy()\n    \n    tp = 0\n    fp = 0\n    for k, pred_bbox in enumerate(pred_bboxes): # fixed in ver.7\n        ious = calc_iou(gt_bboxes, pred_bbox[None, 1:])\n        max_iou = ious.max()\n        if max_iou > iou_th:\n            tp += 1\n            gt_bboxes = np.delete(gt_bboxes, ious.argmax(), axis=0)\n        else:\n            fp += 1\n        if len(gt_bboxes) == 0:\n            fp += len(pred_bboxes) - (k + 1) # fix in ver.7\n            break\n\n    fn = len(gt_bboxes)\n    return tp, fp, fn\n\ndef calc_is_correct(gt_bboxes, pred_bboxes):\n    \"\"\"\n    gt_bboxes: (N, 4) np.array in xywh format\n    pred_bboxes: (N, 5) np.array in conf+xywh format\n    \"\"\"\n    if len(gt_bboxes) == 0 and len(pred_bboxes) == 0:\n        tps, fps, fns = 0, 0, 0\n        return tps, fps, fns\n    \n    elif len(gt_bboxes) == 0:\n        tps, fps, fns = 0, len(pred_bboxes), 0\n        return tps, fps, fns\n    \n    elif len(pred_bboxes) == 0:\n        tps, fps, fns = 0, 0, len(gt_bboxes)\n        return tps, fps, fns\n    \n    pred_bboxes = pred_bboxes[pred_bboxes[:,0].argsort()[::-1]] # sort by conf\n    \n    tps, fps, fns = 0, 0, 0\n    for iou_th in np.arange(0.3, 0.85, 0.05):\n        tp, fp, fn = calc_is_correct_at_iou_th(gt_bboxes, pred_bboxes, iou_th)\n        tps += tp\n        fps += fp\n        fns += fn\n    return tps, fps, fns\n\ndef calc_f2_score(gt_bboxes_list, pred_bboxes_list, verbose=False):\n    \"\"\"\n    gt_bboxes_list: list of (N, 4) np.array in xywh format\n    pred_bboxes_list: list of (N, 5) np.array in conf+xywh format\n    \"\"\"\n    tps, fps, fns = 0, 0, 0\n    for gt_bboxes, pred_bboxes in zip(gt_bboxes_list, pred_bboxes_list):\n        tp, fp, fn = calc_is_correct(gt_bboxes, pred_bboxes)\n        tps += tp\n        fps += fp\n        fns += fn\n        if verbose:\n            num_gt = len(gt_bboxes)\n            num_pred = len(pred_bboxes)\n            print(f'num_gt:{num_gt:<3} num_pred:{num_pred:<3} tp:{tp:<3} fp:{fp:<3} fn:{fn:<3}')\n    return f_beta(tps, fps, fns, beta=2)","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:08:00.357622Z","iopub.execute_input":"2021-12-22T14:08:00.358572Z","iopub.status.idle":"2021-12-22T14:08:00.382842Z","shell.execute_reply.started":"2021-12-22T14:08:00.358524Z","shell.execute_reply":"2021-12-22T14:08:00.381813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Generate gt bboxes list and dummy pred bboxes list","metadata":{}},{"cell_type":"code","source":"gt_df = df.iloc[15:20]\ndisplay(gt_df)\n\ngt_bboxes_list = []\npred_bboxes_list = []\nfor ann_str in gt_df['annotations']:\n    gt_bboxes, pred_bboxes = generate_gt_and_pred(ann_str)\n    gt_bboxes_list.append(gt_bboxes)\n    pred_bboxes_list.append(pred_bboxes)","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:08:00.384281Z","iopub.execute_input":"2021-12-22T14:08:00.385611Z","iopub.status.idle":"2021-12-22T14:08:00.412082Z","shell.execute_reply.started":"2021-12-22T14:08:00.385552Z","shell.execute_reply":"2021-12-22T14:08:00.411409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calc_f2_score(gt_bboxes_list, pred_bboxes_list, verbose=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-22T14:08:00.413148Z","iopub.execute_input":"2021-12-22T14:08:00.413606Z","iopub.status.idle":"2021-12-22T14:08:00.436425Z","shell.execute_reply.started":"2021-12-22T14:08:00.413562Z","shell.execute_reply":"2021-12-22T14:08:00.435554Z"},"trusted":true},"execution_count":null,"outputs":[]}]}