{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"References\n* F2 score : [competition metric implementation](https://www.kaggle.com/bamps53/competition-metric-implementation)\n* CV strategy: [subsequences splitting](https://www.kaggle.com/julian3833/reef-a-cv-strategy-subsequences)\n* Training (with modification on local run other than config augs): [higher res training](https://www.kaggle.com/steamedsheep/yolov5-high-resolution-training)\n\n* [Discussion link](https://www.kaggle.com/c/tensorflow-great-barrier-reef/discussion/300638#1651347) which made me aware of this pit-fall\n\nCorrection(s):\n* Only taking predictions above 0.15 confidence added (not in original implemetation) -> gave significant CV boost for higher image size inference but trend is still decreasing\n\nOther Observations:\n* You should correctly tune inference confidence thss since higher image size seems to be requiring a higher threshold (so they probably have more false positives)","metadata":{}},{"cell_type":"markdown","source":"This notebook is an implementation of calculating CV scores using subsequences split and higher resolution setting during validation. \n\nNote: you need to create model using same fold and resolution to get accurate results. ","metadata":{}},{"cell_type":"markdown","source":"## Init","metadata":{}},{"cell_type":"code","source":"from itertools import groupby\nimport numpy as np\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nimport pandas as pd\nimport os\nimport pickle\nimport cv2\nfrom multiprocessing import Pool\nimport matplotlib.pyplot as plt\nimport ast\nimport glob\nimport time\nimport torch\n\nimport shutil\nfrom shutil import copyfile\nimport sys\nsys.path.append('../input/tensorflow-great-barrier-reef')\n\nfrom joblib import Parallel, delayed\n\nfrom IPython.display import display, HTML\n\nfrom matplotlib import animation, rc\nrc('animation', html='jshtml')\n\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-01-16T16:35:14.274228Z","iopub.execute_input":"2022-01-16T16:35:14.274578Z","iopub.status.idle":"2022-01-16T16:35:15.883220Z","shell.execute_reply.started":"2022-01-16T16:35:14.274488Z","shell.execute_reply":"2022-01-16T16:35:15.882409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define Validation Images and Labels Dir","metadata":{}},{"cell_type":"code","source":"!rm -r images labels","metadata":{"execution":{"iopub.status.busy":"2022-01-16T16:37:59.335668Z","iopub.execute_input":"2022-01-16T16:37:59.335950Z","iopub.status.idle":"2022-01-16T16:38:00.002642Z","shell.execute_reply.started":"2022-01-16T16:37:59.335919Z","shell.execute_reply":"2022-01-16T16:38:00.001776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p ./images/train\n!mkdir -p ./images/val\n\n!mkdir -p ./labels/train\n!mkdir -p ./labels/val","metadata":{"execution":{"iopub.status.busy":"2022-01-16T16:38:00.005103Z","iopub.execute_input":"2022-01-16T16:38:00.005425Z","iopub.status.idle":"2022-01-16T16:38:02.705600Z","shell.execute_reply.started":"2022-01-16T16:38:00.005386Z","shell.execute_reply":"2022-01-16T16:38:02.704567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"splitting = 'yolorules' # 'subseq' else video_id","metadata":{"execution":{"iopub.status.busy":"2022-01-16T16:38:02.708645Z","iopub.execute_input":"2022-01-16T16:38:02.709275Z","iopub.status.idle":"2022-01-16T16:38:02.713604Z","shell.execute_reply.started":"2022-01-16T16:38:02.709239Z","shell.execute_reply":"2022-01-16T16:38:02.712325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/reef-a-cv-strategy-subsequences/cross-validation/train-5folds.csv')\n\nfold = 1\n\nannos = []\nfor i, x in train.iterrows():\n    if splitting == 'subseq' and x.fold == fold: mode = 'val'\n    else if splitting != 'subseq' and x.video_id == fold: mode = 'val'\n    else:\n        mode = 'train'\n        if not x.has_annotations: continue\n    \n    copyfile(x.image_path, f'./images/{mode}/{x.image_id}.jpg')\n    \n    if not x.has_annotations: continue  \n    \n    r = ''; anno = eval(x.annotations)\n    for an in anno:\n        r += '0 {} {} {} {}\\n'.format((an['x'] + an['width'] / 2) / 1280,\n                                        (an['y'] + an['height'] / 2) / 720,\n                                        an['width'] / 1280, an['height'] / 720)\n    with open(f'./labels/{mode}/{x.image_id}.txt', 'w') as fp:\n        fp.write(r)","metadata":{"execution":{"iopub.status.busy":"2022-01-16T16:38:17.267049Z","iopub.execute_input":"2022-01-16T16:38:17.267322Z","iopub.status.idle":"2022-01-16T16:39:56.842353Z","shell.execute_reply.started":"2022-01-16T16:38:17.267291Z","shell.execute_reply":"2022-01-16T16:39:56.841529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/working/labels/train | wc -l #884 for fold1","metadata":{"execution":{"iopub.status.busy":"2022-01-16T16:39:56.844130Z","iopub.execute_input":"2022-01-16T16:39:56.844380Z","iopub.status.idle":"2022-01-16T16:39:57.586218Z","shell.execute_reply.started":"2022-01-16T16:39:56.844345Z","shell.execute_reply":"2022-01-16T16:39:57.585353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## YOLOV5 Install","metadata":{}},{"cell_type":"code","source":"!rm -r /kaggle/working/yolov5\n!git clone https://github.com/ultralytics/yolov5 # clone\n%cd yolov5\n%pip install -qr requirements.txt  # install\n\nfrom yolov5 import utils\ndisplay = utils.notebook_init()  # check","metadata":{"execution":{"iopub.status.busy":"2022-01-16T16:39:57.587733Z","iopub.execute_input":"2022-01-16T16:39:57.588000Z","iopub.status.idle":"2022-01-16T16:40:12.587264Z","shell.execute_reply.started":"2022-01-16T16:39:57.587964Z","shell.execute_reply":"2022-01-16T16:40:12.586462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Config","metadata":{}},{"cell_type":"code","source":"data = '''\npath: /kaggle/working  # dataset root dir\ntrain: images/train  # train images (relative to 'path') 128 images\nval: images/val  # val images (relative to 'path') 128 images\ntest:  # test images (optional)\n\nnc: 1  # number of classes\nnames: ['reef']  # class names\n'''\n\nwith open('fold0.yaml', 'w') as fp:\n    fp.write(data)","metadata":{"execution":{"iopub.status.busy":"2022-01-16T16:40:12.589998Z","iopub.execute_input":"2022-01-16T16:40:12.590315Z","iopub.status.idle":"2022-01-16T16:40:12.596733Z","shell.execute_reply.started":"2022-01-16T16:40:12.590284Z","shell.execute_reply":"2022-01-16T16:40:12.594623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## F2 Score Helpers\nreference : [competition metric implementation](https://www.kaggle.com/bamps53/competition-metric-implementation)","metadata":{}},{"cell_type":"code","source":"def calc_iou(bboxes1, bboxes2, bbox_mode='xywh'):\n    assert len(bboxes1.shape) == 2 and bboxes1.shape[1] == 4\n    assert len(bboxes2.shape) == 2 and bboxes2.shape[1] == 4\n    bboxes1 = bboxes1.copy()\n    bboxes2 = bboxes2.copy()\n    \n    if bbox_mode == 'xywh':\n        bboxes1[:, 2:] += bboxes1[:, :2]\n        bboxes2[:, 2:] += bboxes2[:, :2]\n\n    x11, y11, x12, y12 = np.split(bboxes1, 4, axis=1)\n    x21, y21, x22, y22 = np.split(bboxes2, 4, axis=1)\n    xA = np.maximum(x11, np.transpose(x21))\n    yA = np.maximum(y11, np.transpose(y21))\n    xB = np.minimum(x12, np.transpose(x22))\n    yB = np.minimum(y12, np.transpose(y22))\n    interArea = np.maximum((xB - xA + 1), 0) * np.maximum((yB - yA + 1), 0)\n    boxAArea = (x12 - x11 + 1) * (y12 - y11 + 1)\n    boxBArea = (x22 - x21 + 1) * (y22 - y21 + 1)\n    iou = interArea / (boxAArea + np.transpose(boxBArea) - interArea)\n    return iou\n\ndef f_beta(tp, fp, fn, beta=2):\n    return (1+beta**2)*tp / ((1+beta**2)*tp + beta**2*fn+fp)\n\ndef calc_is_correct_at_iou_th(gt_bboxes, pred_bboxes, iou_th, verbose=False):\n    gt_bboxes = gt_bboxes.copy()\n    pred_bboxes = pred_bboxes.copy()\n    \n    tp = 0\n    fp = 0\n    for k, pred_bbox in enumerate(pred_bboxes): # fixed in ver.7\n        ious = calc_iou(gt_bboxes, pred_bbox[None, 1:])\n        max_iou = ious.max()\n        if max_iou > iou_th:\n            tp += 1\n            gt_bboxes = np.delete(gt_bboxes, ious.argmax(), axis=0)\n        else:\n            fp += 1\n        if len(gt_bboxes) == 0:\n            fp += len(pred_bboxes) - (k + 1) # fix in ver.7\n            break\n\n    fn = len(gt_bboxes)\n    return tp, fp, fn\n\ndef calc_is_correct(gt_bboxes, pred_bboxes):\n    \"\"\"\n    gt_bboxes: (N, 4) np.array in xywh format\n    pred_bboxes: (N, 5) np.array in conf+xywh format\n    \"\"\"\n    if len(gt_bboxes) == 0 and len(pred_bboxes) == 0:\n        tps, fps, fns = 0, 0, 0\n        return tps, fps, fns\n    \n    elif len(gt_bboxes) == 0:\n        tps, fps, fns = 0, len(pred_bboxes)*11, 0\n        return tps, fps, fns\n    \n    elif len(pred_bboxes) == 0:\n        tps, fps, fns = 0, 0, len(gt_bboxes)*11\n        return tps, fps, fns\n    \n    pred_bboxes = pred_bboxes[pred_bboxes[:,0].argsort()[::-1]] # sort by conf\n    \n    tps, fps, fns = 0, 0, 0\n    for iou_th in np.arange(0.3, 0.85, 0.05):\n        tp, fp, fn = calc_is_correct_at_iou_th(gt_bboxes, pred_bboxes, iou_th)\n        tps += tp\n        fps += fp\n        fns += fn\n    return tps, fps, fns\n\ndef calc_f2_score(gt_bboxes_list, pred_bboxes_list, verbose=False):\n    \"\"\"\n    gt_bboxes_list: list of (N, 4) np.array in xywh format\n    pred_bboxes_list: list of (N, 5) np.array in conf+xywh format\n    \"\"\"\n    tps, fps, fns = 0, 0, 0\n    for gt_bboxes, pred_bboxes in zip(gt_bboxes_list, pred_bboxes_list):\n        tp, fp, fn = calc_is_correct(gt_bboxes, pred_bboxes)\n        tps += tp\n        fps += fp\n        fns += fn\n        if verbose:\n            num_gt = len(gt_bboxes)\n            num_pred = len(pred_bboxes)\n            print(f'num_gt:{num_gt:<3} num_pred:{num_pred:<3} tp:{tp:<3} fp:{fp:<3} fn:{fn:<3}')\n    return f_beta(tps, fps, fns, beta=2)","metadata":{"execution":{"iopub.status.busy":"2022-01-16T16:40:12.598592Z","iopub.execute_input":"2022-01-16T16:40:12.599152Z","iopub.status.idle":"2022-01-16T16:40:12.622408Z","shell.execute_reply.started":"2022-01-16T16:40:12.599114Z","shell.execute_reply":"2022-01-16T16:40:12.621664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\npaths = glob('/kaggle/working/labels/val/*')\nval_len = len(paths) ","metadata":{"execution":{"iopub.status.busy":"2022-01-16T16:40:12.625217Z","iopub.execute_input":"2022-01-16T16:40:12.625746Z","iopub.status.idle":"2022-01-16T16:40:12.643526Z","shell.execute_reply.started":"2022-01-16T16:40:12.625705Z","shell.execute_reply":"2022-01-16T16:40:12.642819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Iterate over different up-sizing values to note CV at each step","metadata":{}},{"cell_type":"code","source":"CV=True\n\nif CV:\n    for i in range(3600,10000,1200):\n\n        print(\"#######################################\\n\"*3, f'Starting Inference for image size {i}')\n        start_time = time.time()\n\n        !python val.py --data ./fold0.yaml\\\n            --weights /kaggle/input/reef-baseline-fold12/l6_3600_uflip_vm5_f12_up/f1/best.pt\\\n            --imgsz $i\\\n            --batch 4\\\n            --conf-thres 0.01\\\n            --iou-thres 0.3\\\n            --save-txt\\\n            --save-conf\\\n            --exist-ok\n        t=(time.time() - start_time)/60\n        print(f'Inference Complete in {t:.3f} minutes')\n        print('Starting Cross Validation')\n        start_time = time.time()\n        scores = []\n        for j in range(15,40):\n            confidence=j/100\n            gt_bboxs_list, prd_bboxs_list = [], []\n\n            count=0\n            for image_file in paths:\n                gt_bboxs = []; prd_bboxs = []\n                with open(image_file, 'r') as f:\n                    while True:\n                        r = f.readline().rstrip()\n                        if not r: break\n                        r = r.split()[1:]\n                        bbox = np.array(list(map(float, r))); gt_bboxs.append(bbox)\n\n                pred_path = '/kaggle/working/yolov5/runs/val/exp/labels/'\n                pred_file = pred_path+image_file[27:]\n\n                no_anns = True\n                if os.path.exists(pred_file):\n                    with open(pred_file, 'r') as f:\n                        while True:\n                            r = f.readline().rstrip()\n                            if not r: break\n                            r = r.split()[1:]; r = [r[4], *r[:4]]\n                            conf=float(r[0])\n                            if conf>confidence: \n                                bbox = np.array(list(map(float, r)))\n                                prd_bboxs.append(bbox)\n                                no_anns = False\n\n                if no_anns: count+=1\n\n                gt_bboxs, prd_bboxs= np.array(gt_bboxs), np.array(prd_bboxs)\n                prd_bboxs_list.append(prd_bboxs); gt_bboxs_list.append(gt_bboxs)\n\n            score = calc_f2_score(gt_bboxs_list, prd_bboxs_list, verbose=False)\n            scores.append([score, confidence, count])\n            if confidence%5: print(f'confidence: {confidence}, images w/o anns: {count}, total: {val_len}')\n\n        best = max(scores)\n        print(f'best confidence: {best[1]}, images w/o anns: {best[2]}, total: {val_len}')\n        print(f'img size: {i}, f2 score: {best[0]}') \n        t=(time.time() - start_time)/60\n        print(f'cross validation complete in {t.3f} minutes')\n        torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-01-16T16:40:12.644898Z","iopub.execute_input":"2022-01-16T16:40:12.645432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Seems like upsizing during inference might be causing some kind of overfit to Public LB but more experimentation (with your own augmentations) is required. \n\nYou can also use this notebook to estimate your inference time on hidden test set by multiplying time by 12500/[size of your validation folder]. For image size 3600 and val folder size 884, this came out to 1.5 hours which matched inference time on LB submission\n","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}