{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import ast\nimport numpy as np\nimport pandas as pd\nfrom typing import List\n\nimport torch\nfrom torchvision.ops import box_iou","metadata":{"execution":{"iopub.status.busy":"2022-02-16T03:25:02.314696Z","iopub.execute_input":"2022-02-16T03:25:02.315001Z","iopub.status.idle":"2022-02-16T03:25:02.320136Z","shell.execute_reply.started":"2022-02-16T03:25:02.314968Z","shell.execute_reply":"2022-02-16T03:25:02.319284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# \n\ndef calculate_score(\n    preds: List[torch.Tensor],\n    gts: List[torch.Tensor],\n    iou_th: float\n) -> float:\n    num_tp = 0\n    num_fp = 0\n    num_fn = 0\n    for p, GT in zip(preds, gts):\n        if len(p) and len(GT):\n            gt = GT.clone()\n            gt[:, 2] = gt[:, 0] + gt[:, 2]\n            gt[:, 3] = gt[:, 1] + gt[:, 3]\n            pp = p.clone()\n            pp[:, 2] = pp[:, 0] + pp[:, 2]\n            pp[:, 3] = pp[:, 1] + pp[:, 3]\n            iou_matrix = box_iou(pp, gt)\n            tp = len(torch.where(iou_matrix.max(0)[0] >= iou_th)[0])\n            fp = len(p) - tp\n            fn = len(torch.where(iou_matrix.max(0)[0] < iou_th)[0])\n            num_tp += tp\n            num_fp += fp\n            num_fn += fn\n        elif len(p) == 0 and len(GT):\n            num_fn += len(GT)\n        elif len(p) and len(GT) == 0:\n            num_fp += len(p)\n    score = 5 * num_tp / (5 * num_tp + 4 * num_fn + num_fp)\n    return score","metadata":{"execution":{"iopub.status.busy":"2022-02-16T03:26:49.833657Z","iopub.execute_input":"2022-02-16T03:26:49.834093Z","iopub.status.idle":"2022-02-16T03:26:49.845181Z","shell.execute_reply.started":"2022-02-16T03:26:49.834044Z","shell.execute_reply":"2022-02-16T03:26:49.844317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf = pd.read_csv('../input/tensorflow-great-barrier-reef/train.csv')\n\ndef get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndf['annotations'] = df['annotations'].apply(lambda x: ast.literal_eval(x))\ndf['bboxes'] = df.annotations.apply(get_bbox)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T03:26:50.085774Z","iopub.execute_input":"2022-02-16T03:26:50.086222Z","iopub.status.idle":"2022-02-16T03:26:50.645072Z","shell.execute_reply.started":"2022-02-16T03:26:50.086155Z","shell.execute_reply":"2022-02-16T03:26:50.644454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['preds'] = df['bboxes']  # just assume that predictions is totally the same as GT\ndf","metadata":{"execution":{"iopub.status.busy":"2022-02-16T03:26:50.646360Z","iopub.execute_input":"2022-02-16T03:26:50.646677Z","iopub.status.idle":"2022-02-16T03:26:50.673528Z","shell.execute_reply.started":"2022-02-16T03:26:50.646649Z","shell.execute_reply":"2022-02-16T03:26:50.672709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = []\ngts = []\nfor i, row in (df.iterrows()):\n    if type(row.preds) != float and len(row.preds) > 0:\n        preds = torch.tensor(row.preds)\n        predictions.append(preds)\n    else:\n        predictions.append([])\n    if type(row.bboxes) != float and len(row.bboxes) > 0:\n        gts.append(torch.tensor(row.bboxes))\n    else:\n        gts.append([])\n\n\niou_ths = np.arange(0.3, 0.85, 0.05)\nscores = [calculate_score(predictions, gts, iou_th) for iou_th in iou_ths]\nnp.mean(scores)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T03:26:50.674762Z","iopub.execute_input":"2022-02-16T03:26:50.675006Z","iopub.status.idle":"2022-02-16T03:27:05.363551Z","shell.execute_reply.started":"2022-02-16T03:26:50.674977Z","shell.execute_reply":"2022-02-16T03:27:05.362932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So the F2 = 1.0 when predictions is totally the same as GT.\n\nWe use this to reach F2 = 0.74+ by 3-fold cross validation (video_id split)\n\nIf you are interested you can use this algorithm to compare your CV with ours.\n\nThanks!","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}