{"cells":[{"metadata":{"_uuid":"4ef2be72aa6687e3c646d971fea47358551d6b52"},"cell_type":"markdown","source":"This is an attempt to implament the LB metric: mean avarage precision at different IoU threshold. It might be helpful for local validation. The `map_iou` function evaluates the metric on one image. \n\nIdea borrowed from https://www.kaggle.com/raresbarbantan/f2-metric and is modified for this competition.\n\n\nI haven't thoroughly tested it so please comment if you found any bugs!"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\n\n# helper function to calculate IoU\ndef iou(box1, box2):\n    x11, y11, w1, h1 = box1\n    x21, y21, w2, h2 = box2\n    assert w1 * h1 > 0\n    assert w2 * h2 > 0\n    x12, y12 = x11 + w1, y11 + h1\n    x22, y22 = x21 + w2, y21 + h2\n\n    area1, area2 = w1 * h1, w2 * h2\n    xi1, yi1, xi2, yi2 = max([x11, x21]), max([y11, y21]), min([x12, x22]), min([y12, y22])\n    \n    if xi2 <= xi1 or yi2 <= yi1:\n        return 0\n    else:\n        intersect = (xi2-xi1) * (yi2-yi1)\n        union = area1 + area2 - intersect\n        return intersect / union\n    \n# simple test\nbox1 = [100, 100, 200, 200]\nbox2 = [100, 100, 300, 200]\nprint(iou(box1, box2))\n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def map_iou(boxes_true, boxes_pred, scores, thresholds = [0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75]):\n    \"\"\"\n    Mean average precision at differnet intersection over union (IoU) threshold\n    \n    input:\n        boxes_true: Mx4 numpy array of ground true bounding boxes of one image. \n                    bbox format: (x1, y1, w, h)\n        boxes_pred: Nx4 numpy array of predicted bounding boxes of one image. \n                    bbox format: (x1, y1, w, h)\n        scores:     length N numpy array of scores associated with predicted bboxes\n        thresholds: IoU shresholds to evaluate mean average precision on\n    output: \n        map: mean average precision of the image\n    \"\"\"\n    \n    # According to the introduction, images with no ground truth bboxes will not be \n    # included in the map score unless there is a false positive detection (?)\n        \n    # return None if both are empty, don't count the image in final evaluation (?)\n    if len(boxes_true) == 0 and len(boxes_pred) == 0:\n        return None\n    \n    assert boxes_true.shape[1] == 4 or boxes_pred.shape[1] == 4, \"boxes should be 2D arrays with shape[1]=4\"\n    if len(boxes_pred):\n        assert len(scores) == len(boxes_pred), \"boxes_pred and scores should be same length\"\n        # sort boxes_pred by scores in decreasing order\n        boxes_pred = boxes_pred[np.argsort(scores)[::-1], :]\n    \n    map_total = 0\n    \n    # loop over thresholds\n    for t in thresholds:\n        matched_bt = set()\n        tp, fn = 0, 0\n        for i, bt in enumerate(boxes_true):\n            matched = False\n            for j, bp in enumerate(boxes_pred):\n                miou = iou(bt, bp)\n                if miou >= t and not matched and j not in matched_bt:\n                    matched = True\n                    tp += 1 # bt is matched for the first time, count as TP\n                    matched_bt.add(j)\n            if not matched:\n                fn += 1 # bt has no match, count as FN\n                \n        fp = len(boxes_pred) - len(matched_bt) # FP is the bp that not matched to any bt\n        m = tp / (tp + fn + fp)\n        map_total += m\n    \n    return map_total / len(thresholds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05e23bd44f18cd8d98040a4250a738b5d93c059a"},"cell_type":"code","source":"# simple test\nboxes_true = np.array([[100, 100, 200, 200]])\nboxes_pred = np.array([[100, 100, 300, 200]])\nscores = [0.9]\n\nmap_iou(boxes_true, boxes_pred, scores)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}