{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Why this notebook\n* Each model is different in how the CONF parameter decides PRECISION and RECALL\n* Finding which CONF level is often a matter of guesswork, and you are limited by 5 submissions per day.\n* By systematically evaluating F2 score at each level of CONF, you can have a good idea which CONF will give you the best competition F2 metric.\n\n* This assumes:\n  * Your validation dataset is similar to the hidden test set\n  * Your model has not seen your validation dataset\n\n### What's new in this notebook?\n* F2 is evaluated at every CONF level on validation dataset, similar to F1_curve.png for YoloV5\n* Define 'eval_IOU' for which IOU level to evaluate F2 at\n  * (Competition metric tests IOU @ 0.3 to 0.8 with step of 0.05)\n\n### Example model used in this notebook\n\n* This notebook uses the YOLOX model kindly provided by Remek Kinas (https://www.kaggle.com/remekkinas/yolox-inference-on-kaggle-for-cots-lb-0-507)\n  * And assumes it is trained on FOLD=4 of 5-fold split performed in: https://www.kaggle.com/remekkinas/yolox-training-pipeline-cots-dataset-lb-0-507\n\n### Warning\n* Note the iteration time as an estimate to how long this notebook takes to run.\n* Minimise wasting precious GPU minutes running at large resolutions. Test at small resolutions! The CONF peak should theoretically be the same.","metadata":{}},{"cell_type":"markdown","source":"# Control Panel (change your settings here)","metadata":{}},{"cell_type":"code","source":"# CONTROL PANEL\n\nMODELPATH = \"yolox-cots-models\"\n\nMODELNAME = \"yx_l_003\"\n\nMODEL_TYPE = \"L\" # YOLOX \"S\", M\", \"L\" or \"X\"\n\n# YOLOX Inference Size (h, w) - each must be multiple of 32\nSIZE = (800, 1280)\n\n# YOLOX NMS\nNMS_THRESHOLD = 0.4\n\n# Which IOU level to evaluate (Competition metric tests 0.3 to 0.8 with step of 0.05)\neval_IOU = 0.65","metadata":{"execution":{"iopub.status.busy":"2022-01-19T22:05:50.221338Z","iopub.execute_input":"2022-01-19T22:05:50.221973Z","iopub.status.idle":"2022-01-19T22:05:50.226664Z","shell.execute_reply.started":"2022-01-19T22:05:50.221933Z","shell.execute_reply":"2022-01-19T22:05:50.225682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### YOLOX Inference Model","metadata":{}},{"cell_type":"code","source":"CHECKPOINT_FILE = f\"/kaggle/input/{MODELPATH}/{MODELNAME}.pth\"","metadata":{"execution":{"iopub.status.busy":"2022-01-19T21:58:18.865929Z","iopub.execute_input":"2022-01-19T21:58:18.866207Z","iopub.status.idle":"2022-01-19T21:58:18.870599Z","shell.execute_reply.started":"2022-01-19T21:58:18.866172Z","shell.execute_reply":"2022-01-19T21:58:18.869820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Installation and Code","metadata":{}},{"cell_type":"markdown","source":"## Import Modules","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport torch\nimport importlib\nimport cv2 \nimport pandas as pd\nimport numpy as np\n\nimport ast\nimport shutil\nimport sys\n\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\n\nfrom PIL import Image\nfrom IPython.display import display","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T21:58:20.469059Z","iopub.execute_input":"2022-01-19T21:58:20.469691Z","iopub.status.idle":"2022-01-19T21:58:22.127193Z","shell.execute_reply.started":"2022-01-19T21:58:20.469644Z","shell.execute_reply":"2022-01-19T21:58:22.126339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## YOLOX Installation","metadata":{}},{"cell_type":"markdown","source":"* This notebook only installs the minimum required for YOLOX inference. Each GPU minute is precious!","metadata":{}},{"cell_type":"code","source":"# Copy YOLOX and required modules from local repository (Kaggle dataset -> https://www.kaggle.com/remekkinas/yolox-cots-models)\n%cp -r /kaggle/input/yolox-cots-models /kaggle/working/","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T21:58:23.196442Z","iopub.execute_input":"2022-01-19T21:58:23.196697Z","iopub.status.idle":"2022-01-19T21:58:49.955962Z","shell.execute_reply.started":"2022-01-19T21:58:23.196667Z","shell.execute_reply":"2022-01-19T21:58:49.955157Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install YOLOX required modules (those already installed by default Kaggle are commented out)\n%cd /kaggle/working/yolox-cots-models/yolox-dep\n\n# !pip install pip-21.3.1-py3-none-any.whl -f ./ --no-index\n!pip install loguru-0.5.3-py3-none-any.whl -f ./ --no-index\n!pip install ninja-1.10.2.3-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl -f ./ --no-index\n# !pip install onnx-1.8.1-cp37-cp37m-manylinux2010_x86_64.whl -f ./ --no-index\n# !pip install onnxruntime-1.8.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -f ./ --no-index\n# !pip install onnxoptimizer-0.2.6-cp37-cp37m-manylinux2014_x86_64.whl -f ./ --no-index\n!pip install thop-0.0.31.post2005241907-py3-none-any.whl -f ./ --no-index\n# !pip install tabulate-0.8.9-py3-none-any.whl -f ./ --no-index\n#!pip install onnx-simplifier-0.3.6.tar.gz -f ./ --no-index","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T21:59:24.428969Z","iopub.execute_input":"2022-01-19T21:59:24.429282Z","iopub.status.idle":"2022-01-19T21:59:47.794832Z","shell.execute_reply.started":"2022-01-19T21:59:24.429229Z","shell.execute_reply":"2022-01-19T21:59:47.793993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install YOLOX (this step is actually not necessary for inference)\n# %cd /kaggle/working/yolox-cots-models/YOLOX\n# !pip install -r requirements.txt\n# !pip install -v -e . ","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T22:00:11.314200Z","iopub.execute_input":"2022-01-19T22:00:11.314755Z","iopub.status.idle":"2022-01-19T22:00:11.318223Z","shell.execute_reply.started":"2022-01-19T22:00:11.314710Z","shell.execute_reply":"2022-01-19T22:00:11.317436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install CocoAPI tool\n%cd /kaggle/working/yolox-cots-models/yolox-dep/cocoapi/PythonAPI\n\n!make\n!make install\n!python setup.py install","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T22:00:14.093090Z","iopub.execute_input":"2022-01-19T22:00:14.093380Z","iopub.status.idle":"2022-01-19T22:00:32.705570Z","shell.execute_reply.started":"2022-01-19T22:00:14.093348Z","shell.execute_reply":"2022-01-19T22:00:32.704698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pycocotools","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T22:00:44.773451Z","iopub.execute_input":"2022-01-19T22:00:44.774135Z","iopub.status.idle":"2022-01-19T22:00:44.779715Z","shell.execute_reply.started":"2022-01-19T22:00:44.774093Z","shell.execute_reply":"2022-01-19T22:00:44.778846Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### YOLOX Model and Functions","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working/yolox-cots-models/YOLOX\n\nfrom string import Template\n\nconfig_file_template = '''\n\n#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n    def __init__(self):\n        super(Exp, self).__init__()\n        self.depth = $model_depth\n        self.width = $model_width\n        self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n        self.num_classes = 1\n\n'''\nif MODEL_TYPE == \"X\":\n    pipeline = Template(config_file_template).substitute(\n        model_depth = 1.33,\n        model_width = 1.25\n    )\nelif MODEL_TYPE == \"L\":\n    pipeline = Template(config_file_template).substitute(\n        model_depth = 1.0,\n        model_width = 1.0\n    )\nelif MODEL_TYPE == \"M\":\n    pipeline = Template(config_file_template).substitute(\n        model_depth = 0.67,\n        model_width = 0.75\n    )\nelif MODEL_TYPE == \"S\":\n    pipeline = Template(config_file_template).substitute(\n        model_depth = 0.67,\n        model_width = 0.75\n    )\n\nwith open('cots_config.py', 'w') as f:\n    f.write(pipeline)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-19T22:00:46.496599Z","iopub.execute_input":"2022-01-19T22:00:46.496938Z","iopub.status.idle":"2022-01-19T22:00:46.510996Z","shell.execute_reply.started":"2022-01-19T22:00:46.496899Z","shell.execute_reply":"2022-01-19T22:00:46.510031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/yolox-cots-models/YOLOX\n\nfrom yolox.utils import postprocess\nfrom yolox.data.data_augment import ValTransform\n\nCOCO_CLASSES = (\n  \"starfish\",\n)\n\n# get YOLOX experiment\ncurrent_exp = importlib.import_module('cots_config')\nexp = current_exp.Exp()\n\nnum_classes = 1\n\nnmsthre = NMS_THRESHOLD\n\n# get YOLOX model\nmodel = exp.get_model()\nmodel.cuda()\nmodel.eval()\n\n# get custom trained checkpoint\nckpt_file = CHECKPOINT_FILE\nckpt = torch.load(ckpt_file, map_location=\"cpu\")\nmodel.load_state_dict(ckpt[\"model\"])","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-19T22:00:50.089679Z","iopub.execute_input":"2022-01-19T22:00:50.090232Z","iopub.status.idle":"2022-01-19T22:00:54.399173Z","shell.execute_reply.started":"2022-01-19T22:00:50.090192Z","shell.execute_reply":"2022-01-19T22:00:54.398500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Modified from https://www.kaggle.com/remekkinas/yolox-inference-on-kaggle-for-cots-lb-0-507\n# Additions: \n#     auto converts to xywh format\n#     converts tensors to list of floats\n\ndef yolox_inference(img, model, test_size, conf_threshold = 0.4):\n    bboxes = []\n    bbclasses = []\n    scores = []\n    \n    preproc = ValTransform(legacy = False)\n    \n    tensor_img, _ = preproc(img, None, test_size)\n    tensor_img = torch.from_numpy(tensor_img).unsqueeze(0)\n    tensor_img = tensor_img.float()\n    tensor_img = tensor_img.cuda()\n    \n    with torch.no_grad():\n        outputs = model(tensor_img)\n        outputs = postprocess(\n                    outputs, num_classes, conf_threshold,\n                    nmsthre, class_agnostic=True\n                )\n    \n    if outputs[0] is None:\n        return [], [], []\n    \n    outputs = outputs[0].cpu()\n    bboxes = outputs[:, 0:4]\n    \n    bboxes /= min(test_size[0] / img.shape[0], test_size[1] / img.shape[1])\n    bbclasses = outputs[:, 6]\n    scores = outputs[:, 4] * outputs[:, 5]\n    \n    if len(bboxes) == 0:\n        return [], [], []\n    \n    bboxes = bboxes.numpy()\n    \n    # format to coco\n    bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0]\n    bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1]    \n    \n    # Converts tensors to lists\n    return bboxes, bbclasses.tolist(), scores.tolist()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T22:00:57.217108Z","iopub.execute_input":"2022-01-19T22:00:57.217701Z","iopub.status.idle":"2022-01-19T22:00:57.228868Z","shell.execute_reply.started":"2022-01-19T22:00:57.217660Z","shell.execute_reply":"2022-01-19T22:00:57.228066Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Modified from https://www.kaggle.com/remekkinas/yolox-inference-on-kaggle-for-cots-lb-0-507\n# Additions: \n#     allows customized box color (BGR)\n\ndef draw_yolox_predictions(img, bboxes, scores, bbclasses, classes_dict, boxcolor = (0,0,255)):\n    outimg = img.copy()\n    for i in range(len(bboxes)):\n        box = bboxes[i]\n        cls_id = int(bbclasses[i])\n        score = scores[i]\n        x0 = int(box[0])\n        y0 = int(box[1])\n        x1 = x0 + int(box[2])\n        y1 = y0 + int(box[3])\n\n        cv2.rectangle(outimg, (x0, y0), (x1, y1), boxcolor, 2)\n        cv2.putText(outimg, '{}:{:.1f}%'.format(classes_dict[cls_id], score * 100), (x0, y0 - 3), cv2.FONT_HERSHEY_PLAIN, 0.8, boxcolor, thickness = 1)\n    return outimg","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T22:00:59.301199Z","iopub.execute_input":"2022-01-19T22:00:59.301925Z","iopub.status.idle":"2022-01-19T22:00:59.309139Z","shell.execute_reply.started":"2022-01-19T22:00:59.301885Z","shell.execute_reply":"2022-01-19T22:00:59.308131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## IOU Calculation","metadata":{}},{"cell_type":"code","source":"def IOU_coco(bbox1, bbox2):\n    '''\n        adapted from https://stackoverflow.com/questions/25349178/calculating-percentage-of-bounding-box-overlap-for-image-detector-evaluation\n    '''\n    x_left = max(bbox1[0], bbox2[0])\n    y_top = max(bbox1[1], bbox2[1])\n    x_right = min(bbox1[0] + bbox1[2], bbox2[0] + bbox2[2])\n    y_bottom = min(bbox1[1] + bbox1[3], bbox2[1] + bbox2[3])\n    if x_right < x_left or y_bottom < y_top:\n        return 0.0\n    intersection_area = (x_right - x_left) * (y_bottom - y_top)\n    bb1_area = bbox1[2] * bbox1[3]\n    bb2_area = bbox2[2] * bbox2[3]\n    iou = intersection_area / float(bb1_area + bb2_area - intersection_area)\n\n    assert iou >= 0.0\n    assert iou <= 1.0\n    return iou","metadata":{"execution":{"iopub.status.busy":"2022-01-19T22:01:07.465166Z","iopub.execute_input":"2022-01-19T22:01:07.465911Z","iopub.status.idle":"2022-01-19T22:01:07.473176Z","shell.execute_reply.started":"2022-01-19T22:01:07.465870Z","shell.execute_reply":"2022-01-19T22:01:07.472145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Retrieve video frames not used in training","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working\n\nfrom sklearn.model_selection import GroupKFold\n\ndef get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndef get_path(row):\n    row['image_path'] = f'{ROOT_DIR}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    return row\n\nROOT_DIR  = '/kaggle/input/tensorflow-great-barrier-reef/'\n\ndf = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\n\n# Don't filter for annotated frames. Include frames with no bboxes as well!\ndf[\"num_bbox\"] = df['annotations'].apply(lambda x: str.count(x, 'x'))\ndf_train = df\n\n# Annotations \ndf_train['annotations'] = df_train['annotations'].progress_apply(lambda x: ast.literal_eval(x))\ndf_train['bboxes'] = df_train.annotations.progress_apply(get_bbox)\n\ndf_train = df_train.progress_apply(get_path, axis=1)\n\nkf = GroupKFold(n_splits = 5) \ndf_train = df_train.reset_index(drop=True)\ndf_train['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(kf.split(df_train, y = df_train.video_id.tolist(), groups=df_train.sequence)):\n    df_train.loc[val_idx, 'fold'] = fold\n\ndf_train.head(5)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T22:01:16.971753Z","iopub.execute_input":"2022-01-19T22:01:16.972506Z","iopub.status.idle":"2022-01-19T22:01:33.657147Z","shell.execute_reply.started":"2022-01-19T22:01:16.972452Z","shell.execute_reply":"2022-01-19T22:01:33.656323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Select the dataset that your model hasn't seen!","metadata":{}},{"cell_type":"code","source":"df_test = df_train[df_train.fold == 4]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T22:01:33.658744Z","iopub.execute_input":"2022-01-19T22:01:33.659113Z","iopub.status.idle":"2022-01-19T22:01:33.667610Z","shell.execute_reply.started":"2022-01-19T22:01:33.659072Z","shell.execute_reply":"2022-01-19T22:01:33.666774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get image paths and ground truth BB's","metadata":{}},{"cell_type":"code","source":"import copy\n\n# deepcopy is required to avoid \n\ndf_sample = df_test\nimage_paths = df_sample.image_path.tolist()\ngt = copy.deepcopy(df_sample.bboxes.tolist())\ngtmem = copy.deepcopy(df_sample.bboxes.tolist())","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T22:04:59.849174Z","iopub.execute_input":"2022-01-19T22:04:59.849936Z","iopub.status.idle":"2022-01-19T22:04:59.901312Z","shell.execute_reply.started":"2022-01-19T22:04:59.849895Z","shell.execute_reply":"2022-01-19T22:04:59.900472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test your model is working","metadata":{}},{"cell_type":"code","source":"i = 1350\nTEST_IMAGE_PATH = image_paths[i]\nimg = cv2.imread(TEST_IMAGE_PATH)\n\nbboxes, bbclasses, scores = yolox_inference(img, model, SIZE, 0.01)\n\n# Draw Green ground truth box\nout_image = draw_yolox_predictions(img, gt[i], [1.0] * len(gt[i]), [0] * len(gt[i]), COCO_CLASSES, (0,255,0))\n\n# Draw Red inference box\nout_image = draw_yolox_predictions(out_image, bboxes, scores, bbclasses, COCO_CLASSES, (0,0,255))\n\n# Convert BGR to RGB\nout_image = cv2.cvtColor(out_image, cv2.COLOR_BGR2RGB)\n\ndisplay(Image.fromarray(out_image))","metadata":{"execution":{"iopub.status.busy":"2022-01-19T22:05:57.732761Z","iopub.execute_input":"2022-01-19T22:05:57.733415Z","iopub.status.idle":"2022-01-19T22:05:58.317753Z","shell.execute_reply.started":"2022-01-19T22:05:57.733371Z","shell.execute_reply":"2022-01-19T22:05:58.316184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Assess Model Performance","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working\n\n# Confidence scores of true positives, false positives and count false negatives\nTP = [] # Confidence scores of true positives\nFP = [] # Confidence scores of true positives\nFN = 0  # Count of false negative boxes\n\nfor i in tqdm(range(len(image_paths))):\n# for i in tqdm(range(1250, 1450)):\n    TEST_IMAGE_PATH = image_paths[i]\n    img = cv2.imread(TEST_IMAGE_PATH)\n    bboxes, bbclasses, scores = yolox_inference(img, model, SIZE, 0.01)\n\n    # Test YOLOX\n    gt0 = gt[i]\n    if len(bboxes) == 0:\n        # all gt are false negative\n        FN += len(gt0)\n    else:\n        bb = bboxes.copy().tolist()\n        for idx, b in enumerate(bb):\n            b.append(scores[idx])\n        bb.sort(key = lambda x: x[4], reverse = True)\n        \n        if len(gt0) == 0:\n            # all bboxes are false positives\n            for b in bb:\n                FP.append(b[4])\n        else:\n            # match bbox with gt\n            for b in bb:\n                matched = False\n                for g in gt0:\n                    # check whether gt box is already matched to an inference bb\n                    if len(g) == 4:\n                        # g bbox is unmatched\n                        if IOU_coco(b, g) >= eval_IOU:\n                            g.append(b[4]) # assign confidence values to g; marks g as matched\n                            matched = True\n                            TP.append(b[4])\n                            break\n                if not matched:\n                    FP.append(b[4])\n            for g in gt0:\n                if len(g) == 4:\n                    FN += 1","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T22:06:29.892020Z","iopub.execute_input":"2022-01-19T22:06:29.892323Z","iopub.status.idle":"2022-01-19T22:13:53.498691Z","shell.execute_reply.started":"2022-01-19T22:06:29.892287Z","shell.execute_reply":"2022-01-19T22:13:53.497934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display your model's Metrics","metadata":{}},{"cell_type":"code","source":"from matplotlib import pyplot as plt\n\nplt.hist(TP, 100)\nplt.title(\"CONF of true positives, base YOLOX\")\nplt.xlabel('CONF')\nplt.ylabel('TP count')\nplt.show()\n\nprint(f'True positives = {len(TP)}')\nprint(f'False negatives = {FN}')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T22:13:55.946692Z","iopub.execute_input":"2022-01-19T22:13:55.947527Z","iopub.status.idle":"2022-01-19T22:13:56.313181Z","shell.execute_reply.started":"2022-01-19T22:13:55.947456Z","shell.execute_reply":"2022-01-19T22:13:56.312481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(FP, 100)\nplt.title(\"CONF of false positives, base YOLOX\")\nplt.xlabel('CONF')\nplt.ylabel('FP count')\nplt.show()\n\nprint(f'False positives = {len(FP)}')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T22:13:58.925979Z","iopub.execute_input":"2022-01-19T22:13:58.926251Z","iopub.status.idle":"2022-01-19T22:13:59.424173Z","shell.execute_reply.started":"2022-01-19T22:13:58.926219Z","shell.execute_reply":"2022-01-19T22:13:59.423320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"F2list = []\nF2max = 0.0\nF2maxat = -1.0\n\nfor c in np.arange(0.0, 1.0, 0.01):\n    FNcount = FN + sum(1 for i in TP if i < c)\n    TPcount = sum(1 for i in TP if i >= c)\n    FPcount = sum(1 for i in FP if i >= c)\n    R = TPcount / (TPcount + FNcount + 0.0001)\n    P = TPcount / (TPcount + FPcount + 0.0001)\n    F2 = (5 * P * R) / (4 * P + R + 0.0001)\n    F2list.append((c, F2))\n    if F2max < F2:\n        F2max = F2\n        F2maxat = c\n\nplt.scatter(*zip(*F2list))\nplt.title(\"CONF vs F2 score\")\nplt.xlabel('CONF')\nplt.ylabel('F2')\nplt.show()\n\nprint(f'F2 max is {F2max} at CONF = {F2maxat}')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-19T22:14:34.677580Z","iopub.execute_input":"2022-01-19T22:14:34.677849Z","iopub.status.idle":"2022-01-19T22:14:34.996285Z","shell.execute_reply.started":"2022-01-19T22:14:34.677818Z","shell.execute_reply":"2022-01-19T22:14:34.995547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Cleanup","metadata":{}},{"cell_type":"code","source":"!rm -rf /kaggle/working/*","metadata":{"execution":{"iopub.status.busy":"2022-01-09T11:15:54.282823Z","iopub.execute_input":"2022-01-09T11:15:54.283134Z","iopub.status.idle":"2022-01-09T11:15:54.450751Z","shell.execute_reply.started":"2022-01-09T11:15:54.283102Z","shell.execute_reply":"2022-01-09T11:15:54.449692Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Credits","metadata":{}},{"cell_type":"markdown","source":"* https://www.kaggle.com/remekkinas/yolox-training-pipeline-cots-dataset-lb-0-507\n* https://www.kaggle.com/remekkinas/yolox-inference-on-kaggle-for-cots-lb-0-507","metadata":{}}]}