{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-12T23:38:57.563275Z","iopub.execute_input":"2022-07-12T23:38:57.564089Z","iopub.status.idle":"2022-07-12T23:38:57.599665Z","shell.execute_reply.started":"2022-07-12T23:38:57.563977Z","shell.execute_reply":"2022-07-12T23:38:57.598465Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#What does the notation mAP@[.5:.95] mean?\n\n\n\"mAP.5:.95 means average mAP over different IoU thresholds, from 0.5 to 0.95, step 0.05 (0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95).\"\n\n\"There is an associated MS COCO challenge with a new evaluation metric, that averages mAP over different IoU thresholds, from 0.5 to 0.95 (written as “0.5:0.95”).\n\n\"We evaluate the mAP averaged for IoU ∈ [0.5 : 0.05 : 0.95] (COCO’s standard metric, simply denoted as mAP@[.5, .95]) and mAP@0.5 (PASCAL VOC’s metric). \"\n\n\"To evaluate our final detections, we use the official COCO API [20], which measures mAP averaged over IOU thresholds in [0.5 : 0.05 : 0.95], amongst other metrics. \"\n\n\"BTW, the source code of coco shows exactly what mAP@[.5:.95] is doing:\"\n\nself.iouThrs = np.linspace(.5, 0.95, np.round((0.95 - .5) / .05) + 1, endpoint=True)\n\nReferences:\n\ncocoapi\n\nInside-Outside Net: Detecting Objects in Context with Skip Pooling and Recurrent Neural Networks\n\nFaster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks\n\nSpeed/accuracy trade-offs for modern convolutional object detectors\n\nhttps://datascience.stackexchange.com/questions/16797/what-does-the-notation-map-5-95-mean","metadata":{}},{"cell_type":"markdown","source":"![](https://hasty.ai/media/pages/content-hub/mp-wiki/metrics/map-mean-average-precision/62945307a2-1654857595/snimok-ekrana-2022-06-10-v-13.33.11.png)Hasty.ai","metadata":{}},{"cell_type":"code","source":"#Code by Peter  https://www.kaggle.com/code/pestipeti/explanation-of-map5-scoring-metric/notebook\n\ndef map_per_image(label, predictions):\n    \"\"\"Computes the precision score of one image.\n\n    Parameters\n    ----------\n    label : string\n            The true label of the image\n    predictions : list\n            A list of predicted elements (order does matter, 5 predictions allowed per image)\n\n    Returns\n    -------\n    score : double\n    \"\"\"    \n    try:\n        return 1 / (predictions[:5].index(label) + 1)\n    except ValueError:\n        return 0.0\n\ndef map_per_set(labels, predictions):\n    \"\"\"Computes the average over multiple images.\n\n    Parameters\n    ----------\n    labels : list\n             A list of the true labels. (Only one true label per images allowed!)\n    predictions : list of list\n             A list of predicted elements (order does matter, 5 predictions allowed per image)\n\n    Returns\n    -------\n    score : double\n    \"\"\"\n    return np.mean([map_per_image(l, p) for l,p in zip(labels, predictions)])","metadata":{"execution":{"iopub.status.busy":"2022-07-12T23:39:37.024392Z","iopub.execute_input":"2022-07-12T23:39:37.025703Z","iopub.status.idle":"2022-07-12T23:39:37.033973Z","shell.execute_reply.started":"2022-07-12T23:39:37.025642Z","shell.execute_reply":"2022-07-12T23:39:37.032861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Peter  https://www.kaggle.com/code/pestipeti/explanation-of-map5-scoring-metric/notebook\n\n#                   (true, [predictions])\nassert map_per_image('x', []) == 0.0\nassert map_per_image('x', ['y']) == 0.0\nassert map_per_image('x', ['x']) == 1.0\nassert map_per_image('x', ['x', 'y', 'z']) == 1.0\nassert map_per_image('x', ['y', 'x']) == 0.5\nassert map_per_image('x', ['y', 'x', 'x']) == 0.5\nassert map_per_image('x', ['y', 'z']) == 0.0\nassert map_per_image('x', ['y', 'z', 'x']) == 1/3\nassert map_per_image('x', ['y', 'z', 'a', 'b', 'c']) == 0.0\nassert map_per_image('x', ['x', 'z', 'a', 'b', 'c']) == 1.0\nassert map_per_image('x', ['y', 'z', 'a', 'b', 'x']) == 1/5\nassert map_per_image('x', ['y', 'z', 'a', 'b', 'c', 'x']) == 0.0\n\nassert map_per_set(['x'], [['x', 'y']]) == 1.0\nassert map_per_set(['x', 'z'], [['x', 'y'], ['x', 'y']]) == 1/2\nassert map_per_set(['x', 'z'], [['x', 'y'], ['x', 'y', 'z']]) == 2/3\nassert map_per_set(['x', 'z', 'k'], [['x', 'y'], ['x', 'y', 'z'], ['a', 'b', 'c', 'd', 'e']]) == 4/9","metadata":{"execution":{"iopub.status.busy":"2022-07-12T23:39:59.686233Z","iopub.execute_input":"2022-07-12T23:39:59.686604Z","iopub.status.idle":"2022-07-12T23:39:59.700868Z","shell.execute_reply.started":"2022-07-12T23:39:59.686573Z","shell.execute_reply":"2022-07-12T23:39:59.699491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numba import jit\nfrom typing import List, Union, Tuple","metadata":{"execution":{"iopub.status.busy":"2022-07-12T23:42:35.849226Z","iopub.execute_input":"2022-07-12T23:42:35.850077Z","iopub.status.idle":"2022-07-12T23:42:36.670006Z","shell.execute_reply.started":"2022-07-12T23:42:35.850024Z","shell.execute_reply":"2022-07-12T23:42:36.668908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#IOU Calculation","metadata":{}},{"cell_type":"code","source":"#Code by Peter  https://www.kaggle.com/code/pestipeti/competition-metric-details-script\n\n@jit(nopython=True)\ndef calculate_iou(gt, pr, form='pascal_voc') -> float:\n    \"\"\"Calculates the Intersection over Union.\n\n    Args:\n        gt: (np.ndarray[Union[int, float]]) coordinates of the ground-truth box\n        pr: (np.ndarray[Union[int, float]]) coordinates of the prdected box\n        form: (str) gt/pred coordinates format\n            - pascal_voc: [xmin, ymin, xmax, ymax]\n            - coco: [xmin, ymin, w, h]\n    Returns:\n        (float) Intersection over union (0.0 <= iou <= 1.0)\n    \"\"\"\n    if form == 'coco':\n        gt = gt.copy()\n        pr = pr.copy()\n\n        gt[2] = gt[0] + gt[2]\n        gt[3] = gt[1] + gt[3]\n        pr[2] = pr[0] + pr[2]\n        pr[3] = pr[1] + pr[3]\n\n    # Calculate overlap area\n    dx = min(gt[2], pr[2]) - max(gt[0], pr[0]) + 1\n    \n    if dx < 0:\n        return 0.0\n    \n    dy = min(gt[3], pr[3]) - max(gt[1], pr[1]) + 1\n\n    if dy < 0:\n        return 0.0\n\n    overlap_area = dx * dy\n\n    # Calculate union area\n    union_area = (\n            (gt[2] - gt[0] + 1) * (gt[3] - gt[1] + 1) +\n            (pr[2] - pr[0] + 1) * (pr[3] - pr[1] + 1) -\n            overlap_area\n    )\n\n    return overlap_area / union_area","metadata":{"execution":{"iopub.status.busy":"2022-07-12T23:42:41.174877Z","iopub.execute_input":"2022-07-12T23:42:41.175281Z","iopub.status.idle":"2022-07-12T23:42:41.523867Z","shell.execute_reply.started":"2022-07-12T23:42:41.175247Z","shell.execute_reply":"2022-07-12T23:42:41.522632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#No overlapping","metadata":{}},{"cell_type":"code","source":"#Code by Peter  https://www.kaggle.com/code/pestipeti/competition-metric-details-script\n\nbox1 = np.array([834.0, 222.0, 56.0, 36.0])\nbox2 = np.array([26.0, 144.0, 124.0, 117.0])\n\nassert calculate_iou(box1, box2, form='coco') == 0.0","metadata":{"execution":{"iopub.status.busy":"2022-07-12T23:43:23.023576Z","iopub.execute_input":"2022-07-12T23:43:23.024152Z","iopub.status.idle":"2022-07-12T23:43:24.384445Z","shell.execute_reply.started":"2022-07-12T23:43:23.024108Z","shell.execute_reply":"2022-07-12T23:43:24.383222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Partial (50%) overlapping","metadata":{}},{"cell_type":"code","source":"#Code by Peter  https://www.kaggle.com/code/pestipeti/competition-metric-details-script\n\nbox1 = np.array([100, 100, 100, 100])\nbox2 = np.array([100, 100, 200, 100])\n\nres = calculate_iou(box1, box2, form='coco')\nassert  res > 0.5 and res < 0.50249","metadata":{"execution":{"iopub.status.busy":"2022-07-12T23:44:03.547108Z","iopub.execute_input":"2022-07-12T23:44:03.547594Z","iopub.status.idle":"2022-07-12T23:44:03.883971Z","shell.execute_reply.started":"2022-07-12T23:44:03.547474Z","shell.execute_reply":"2022-07-12T23:44:03.882755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Full overlapping","metadata":{}},{"cell_type":"code","source":"#Code by Peter  https://www.kaggle.com/code/pestipeti/competition-metric-details-script\n\nbox1 = np.array([834.0, 222.0, 56.0, 36.0])\nbox2 = box1\n\nassert calculate_iou(box1, box2, form='coco') == 1.0","metadata":{"execution":{"iopub.status.busy":"2022-07-12T23:44:59.980287Z","iopub.execute_input":"2022-07-12T23:44:59.980728Z","iopub.status.idle":"2022-07-12T23:44:59.986712Z","shell.execute_reply.started":"2022-07-12T23:44:59.980695Z","shell.execute_reply":"2022-07-12T23:44:59.985628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#MAP calculation","metadata":{}},{"cell_type":"code","source":"#Code by Peter  https://www.kaggle.com/code/pestipeti/competition-metric-details-script\n\n@jit(nopython=True)\ndef find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None) -> int:\n    \"\"\"Returns the index of the 'best match' between the\n    ground-truth boxes and the prediction. The 'best match'\n    is the highest IoU. (0.0 IoUs are ignored).\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        pred: (List[Union[int, float]]) Coordinates of the predicted box\n        pred_idx: (int) Index of the current predicted box\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (int) Index of the best match GT box (-1 if no match above threshold)\n    \"\"\"\n    best_match_iou = -np.inf\n    best_match_idx = -1\n\n    for gt_idx in range(len(gts)):\n        \n        if gts[gt_idx][0] < 0:\n            # Already matched GT-box\n            continue\n        \n        iou = -1 if ious is None else ious[gt_idx][pred_idx]\n\n        if iou < 0:\n            iou = calculate_iou(gts[gt_idx], pred, form=form)\n            \n            if ious is not None:\n                ious[gt_idx][pred_idx] = iou\n\n        if iou < threshold:\n            continue\n\n        if iou > best_match_iou:\n            best_match_iou = iou\n            best_match_idx = gt_idx\n\n    return best_match_idx\n\n@jit(nopython=True)\ndef calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None) -> float:\n    \"\"\"Calculates precision for GT - prediction pairs at one threshold.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n = len(preds)\n    tp = 0\n    fp = 0\n    \n    # for pred_idx, pred in enumerate(preds_sorted):\n    for pred_idx in range(n):\n\n        best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,\n                                            threshold=threshold, form=form, ious=ious)\n\n        if best_match_gt_idx >= 0:\n            # True positive: The predicted box matches a gt box with an IoU above the threshold.\n            tp += 1\n            # Remove the matched GT box\n            gts[best_match_gt_idx] = -1\n\n        else:\n            # No match\n            # False positive: indicates a predicted box had no associated gt box.\n            fp += 1\n\n    # False negative: indicates a gt box had no associated predicted box.\n    fn = (gts.sum(axis=1) > 0).sum()\n\n    return tp / (tp + fp + fn)\n\n\n@jit(nopython=True)\ndef calculate_image_precision(gts, preds, thresholds = (0.5, ), form = 'coco') -> float:\n    \"\"\"Calculates image precision.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        thresholds: (float) Different thresholds\n        form: (str) Format of the coordinates\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n_threshold = len(thresholds)\n    image_precision = 0.0\n    \n    ious = np.ones((len(gts), len(preds))) * -1\n    # ious = None\n\n    for threshold in thresholds:\n        precision_at_threshold = calculate_precision(gts.copy(), preds, threshold=threshold,\n                                                     form=form, ious=ious)\n        image_precision += precision_at_threshold / n_threshold\n\n    return image_precision","metadata":{"execution":{"iopub.status.busy":"2022-07-12T23:47:54.039508Z","iopub.execute_input":"2022-07-12T23:47:54.039909Z","iopub.status.idle":"2022-07-12T23:47:54.060709Z","shell.execute_reply.started":"2022-07-12T23:47:54.039877Z","shell.execute_reply":"2022-07-12T23:47:54.059459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#For local validation, you need to calculate image precision for all of your validation images and take the average.","metadata":{}},{"cell_type":"code","source":"#Code by Peter  https://www.kaggle.com/code/pestipeti/competition-metric-details-script\n\nvalidation_image_precision = []\niou_thresholds = [x for x in np.arange(0.5, 0.76, 0.05)]","metadata":{"execution":{"iopub.status.busy":"2022-07-12T23:56:38.402739Z","iopub.execute_input":"2022-07-12T23:56:38.403142Z","iopub.status.idle":"2022-07-12T23:56:38.409327Z","shell.execute_reply.started":"2022-07-12T23:56:38.403106Z","shell.execute_reply":"2022-07-12T23:56:38.407910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#From that part started csv snippets, therefore there is no sense to add any of the lines.","metadata":{}},{"cell_type":"code","source":"#Code by Cyc  https://www.kaggle.com/code/chenyc15/mean-average-precision-metric\n\n# helper function to calculate IoU\ndef iou(box1, box2):\n    x11, y11, w1, h1 = box1\n    x21, y21, w2, h2 = box2\n    assert w1 * h1 > 0\n    assert w2 * h2 > 0\n    x12, y12 = x11 + w1, y11 + h1\n    x22, y22 = x21 + w2, y21 + h2\n\n    area1, area2 = w1 * h1, w2 * h2\n    xi1, yi1, xi2, yi2 = max([x11, x21]), max([y11, y21]), min([x12, x22]), min([y12, y22])\n    \n    if xi2 <= xi1 or yi2 <= yi1:\n        return 0\n    else:\n        intersect = (xi2-xi1) * (yi2-yi1)\n        union = area1 + area2 - intersect\n        return intersect / union\n    \n# simple test\nbox1 = [100, 100, 200, 200]\nbox2 = [100, 100, 300, 200]\nprint(iou(box1, box2))","metadata":{"execution":{"iopub.status.busy":"2022-07-13T00:06:07.865734Z","iopub.execute_input":"2022-07-13T00:06:07.866138Z","iopub.status.idle":"2022-07-13T00:06:07.878128Z","shell.execute_reply.started":"2022-07-13T00:06:07.866104Z","shell.execute_reply":"2022-07-13T00:06:07.876865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Above, \"number 66666 is all about balance and stability, so if you start seeing it, it will bring a financial balance into your life. It may announce a new business opportunity or a salary raise and your current work. This way, angels are reminding you of their presence in your life.\"\n\nReally???\n\nhttps://www.ipublishing.co.in/angel-number-66666-meaning#:~:text=Angel%20number%2066666%20brings%20you,you%20find%20serenity%20within%20yourself.","metadata":{}},{"cell_type":"code","source":"#Code by Cyc  https://www.kaggle.com/code/chenyc15/mean-average-precision-metric\n\ndef map_iou(boxes_true, boxes_pred, scores, thresholds = [0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75]):\n    \"\"\"\n    Mean average precision at differnet intersection over union (IoU) threshold\n    \n    input:\n        boxes_true: Mx4 numpy array of ground true bounding boxes of one image. \n                    bbox format: (x1, y1, w, h)\n        boxes_pred: Nx4 numpy array of predicted bounding boxes of one image. \n                    bbox format: (x1, y1, w, h)\n        scores:     length N numpy array of scores associated with predicted bboxes\n        thresholds: IoU shresholds to evaluate mean average precision on\n    output: \n        map: mean average precision of the image\n    \"\"\"\n    \n    # According to the introduction, images with no ground truth bboxes will not be \n    # included in the map score unless there is a false positive detection (?)\n        \n    # return None if both are empty, don't count the image in final evaluation (?)\n    if len(boxes_true) == 0 and len(boxes_pred) == 0:\n        return None\n    \n    assert boxes_true.shape[1] == 4 or boxes_pred.shape[1] == 4, \"boxes should be 2D arrays with shape[1]=4\"\n    if len(boxes_pred):\n        assert len(scores) == len(boxes_pred), \"boxes_pred and scores should be same length\"\n        # sort boxes_pred by scores in decreasing order\n        boxes_pred = boxes_pred[np.argsort(scores)[::-1], :]\n    \n    map_total = 0\n    \n    # loop over thresholds\n    for t in thresholds:\n        matched_bt = set()\n        tp, fn = 0, 0\n        for i, bt in enumerate(boxes_true):\n            matched = False\n            for j, bp in enumerate(boxes_pred):\n                miou = iou(bt, bp)\n                if miou >= t and not matched and j not in matched_bt:\n                    matched = True\n                    tp += 1 # bt is matched for the first time, count as TP\n                    matched_bt.add(j)\n            if not matched:\n                fn += 1 # bt has no match, count as FN\n                \n        fp = len(boxes_pred) - len(matched_bt) # FP is the bp that not matched to any bt\n        m = tp / (tp + fn + fp)\n        map_total += m\n    \n    return map_total / len(thresholds)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T00:07:27.547900Z","iopub.execute_input":"2022-07-13T00:07:27.548310Z","iopub.status.idle":"2022-07-13T00:07:27.561674Z","shell.execute_reply.started":"2022-07-13T00:07:27.548278Z","shell.execute_reply":"2022-07-13T00:07:27.560798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Cyc  https://www.kaggle.com/code/chenyc15/mean-average-precision-metric\n\n# simple test\nboxes_true = np.array([[100, 100, 200, 200]])\nboxes_pred = np.array([[100, 100, 300, 200]])\nscores = [0.9]\n\nmap_iou(boxes_true, boxes_pred, scores)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T00:07:45.207817Z","iopub.execute_input":"2022-07-13T00:07:45.208186Z","iopub.status.idle":"2022-07-13T00:07:45.220161Z","shell.execute_reply.started":"2022-07-13T00:07:45.208156Z","shell.execute_reply":"2022-07-13T00:07:45.218850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Acknowledgements:\n\nPeter  https://www.kaggle.com/code/pestipeti/explanation-of-map5-scoring-metric/notebook\n\nPeter  https://www.kaggle.com/code/pestipeti/competition-metric-details-script\n\nCyc  https://www.kaggle.com/code/chenyc15/mean-average-precision-metric","metadata":{}}]}