{"cells":[{"metadata":{"_uuid":"4ef2be72aa6687e3c646d971fea47358551d6b52"},"cell_type":"markdown","source":"**Implementation of the competition metric in keras for image segmentation models\n**\n\nCode is partly from these great kernels:\n\nUnet segmentation: https://www.kaggle.com/jonnedtc/cnn-segmentation-connected-components\n\nNumpy implementation of the bbox https://www.kaggle.com/chenyc15/mean-average-precision-metric\n\nIdea borrowed from: https://www.kaggle.com/raresbarbantan/f2-metric and is modified for this competition."},{"metadata":{"trusted":true,"_uuid":"1d45153b741a86218c76f5d8982896996366df38"},"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nfrom skimage import measure\nimport keras.backend as K","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# helper function to calculate IoU\ndef iou_bbox(box1, box2):\n    x11, y11, w1, h1 = box1\n    x21, y21, w2, h2 = box2\n    assert w1 * h1 > 0\n    assert w2 * h2 > 0\n    x12, y12 = x11 + w1, y11 + h1\n    x22, y22 = x21 + w2, y21 + h2\n\n    area1, area2 = w1 * h1, w2 * h2\n    xi1, yi1, xi2, yi2 = max([x11, x21]), max([y11, y21]), min([x12, x22]), min([y12, y22])\n    \n    if xi2 <= xi1 or yi2 <= yi1:\n        return 0\n    else:\n        intersect = (xi2-xi1) * (yi2-yi1)\n        union = area1 + area2 - intersect\n        return intersect / union\n    \n# simple test\nbox1 = [100, 100, 200, 200]\nbox2 = [100, 100, 300, 200]\nprint(iou_bbox(box1, box2))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d2fcd2420c583bac1aca8efeee549a9c44485421"},"cell_type":"markdown","source":"**In contrast to the competition metric, the confidence level is not taken into account for computing the map_iou. **"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def map_iou(boxes_true, boxes_pred, thresholds=(0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75)):\n    \"\"\"\n    Mean average precision at differnet intersection over union (IoU) threshold\n\n    input:\n        boxes_true: Mx4 numpy array of ground true bounding boxes of one image.\n                    bbox format: (x1, y1, w, h)\n        boxes_pred: Nx4 numpy array of predicted bounding boxes of one image.\n                    bbox format: (x1, y1, w, h)\n        thresholds: IoU shresholds to evaluate mean average precision on\n    output:\n        map: mean average precision of the image\n    \"\"\"\n\n    # According to the introduction, images with no ground truth bboxes will not be\n    # included in the map score unless there is a false positive detection (?)\n\n    # return 0 if both are empty, don't count the image in final evaluation (?)\n    if len(boxes_true) == 0 and len(boxes_pred) == 0:\n        return 0\n\n    map_total = 0\n\n    # loop over thresholds\n    for t in thresholds:\n        matched_bt = set()\n        tp, fn = 0, 0\n        for i, bt in enumerate(boxes_true):\n            matched = False\n            for j, bp in enumerate(boxes_pred):\n                miou = iou_bbox(bt, bp)\n                if miou >= t and not matched and j not in matched_bt:\n                    matched = True\n                    tp += 1 # bt is matched for the first time, count as TP\n                    matched_bt.add(j)\n            if not matched:\n                fn += 1 # bt has no match, count as FN\n\n        fp = len(boxes_pred) - len(matched_bt) # FP is the bp that not matched to any bt\n        m = tp / (tp + fn + fp)\n        map_total += m\n\n    return map_total / len(thresholds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc8fd114a9ded2d9fa7812a9e6791ec696f95686"},"cell_type":"code","source":"def unet_mask_to_bbox_coords(mask, threshold=0.5, do_resize=False):\n    '''\n    :param mask: predicted mask, numpy array of shape (widht, height, 1) or (width, height)\n    :param threshold: threshold for binarization of mask\n    :return: bbox coordinates, in form [x, y, width, height] for each bbox coordinate\n    :rtype: numpy array\n    '''\n    if mask.ndim == 3:\n        mask = mask[:, :, 0]\n    # resize predicted mask\n    if do_resize:\n        mask = resize(mask, (1024, 1024), mode='constant')\n    # threshold predicted mask, multiply by 255, since predictions were upscaled for memory performance\n    comp = mask > threshold\n    # apply connected components\n    comp = measure.label(comp)\n\n    bboxes = np.array([]).reshape((0, 4))\n\n    for region in measure.regionprops(comp):\n        # retrieve x, y, height and width\n        y, x, y2, x2 = region.bbox\n        height = y2 - y\n        width = x2 - x\n        bboxes = np.concatenate([bboxes, np.array([[x, y, width, height]])], axis=0)\n\n    return bboxes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b16e65e8a14c9cd6aa2d650953e6784415c8976"},"cell_type":"code","source":"def competitionMetric(y_true, y_pred):\n    '''\n    Implementation of rsna pneumonia competition metric\n    '''\n    def np_competitionMetric(np_true, np_pred):\n        '''\n        Compute the mean map_iou for each sample of the batch\n        '''\n        return np.mean([map_iou(unet_mask_to_bbox_coords(true), unet_mask_to_bbox_coords(pred))\n                               for true, pred in zip(np_true, np_pred)]).astype(np.float32)\n\n    return tf.py_func(np_competitionMetric,\n                      inp=[y_true, y_pred],\n                      Tout=tf.float32,\n                      stateful=False,\n                      name='competitionMetric'\n                      )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"71dc1b4729d4c98c069664d473d3fb1c11d0268c"},"cell_type":"markdown","source":"**> Let's do a simple test**"},{"metadata":{"trusted":true,"_uuid":"293bfda45d560d091df4dc9732bb4c0cb3df3fbe"},"cell_type":"code","source":"y_true_array = np.zeros((1, 128, 128, 1))\ny_true_array[:, 20: 41, 23:45, :] = 1\ny_true = tf.Variable(y_true_array, dtype='float32', name='y_true')\n\ny_pred_array = np.zeros((1, 128, 128, 1))\ny_pred_array[:, 20: 37, 18:37, :] = 1\ny_pred = tf.Variable(y_pred_array, dtype='float32', name='y_pred')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ede9a305109db0933aa52454ac7ee5508d7161e"},"cell_type":"code","source":"sess = K.get_session()\nsess.run(tf.global_variables_initializer())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8cb546e59eec0b8f3415af3b04f9ea4f4c773ec"},"cell_type":"code","source":"sess.run(competitionMetric(y_true, y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5110593b32a68255e371f039a7ab664fd582bb51"},"cell_type":"code","source":"box_true = [[20, 23, 21, 22]] #x, y, width, height\nbox_pred = [[20, 18, 17, 19]]\nmap_iou(box_true, box_pred)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c1159fc7801352c762d644939006ad42fc15711a"},"cell_type":"markdown","source":"Now, let's compute the metric on a batch example"},{"metadata":{"trusted":true,"_uuid":"05e23bd44f18cd8d98040a4250a738b5d93c059a"},"cell_type":"code","source":"sess = K.get_session()\n\ny_true_array = np.zeros((24, 128, 128, 1))\ny_true_array[:, 20: 35, 10:40, :] = 1\ny_true = tf.Variable(y_true_array, dtype='float32', name='y_true')\n\ny_pred_array = np.zeros((24, 128, 128, 1))\ny_pred_array[5:, 20: 37, 19:37, :] = 1\ny_pred_array[:10, 100:115, 105:115, :] = 1\ny_pred = tf.Variable(y_pred_array, dtype='float32', name='y_pred')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2b3391d44abd73cd875b03964dabf01a1008545"},"cell_type":"code","source":"sess.run(tf.global_variables_initializer())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b78c817c3c167b2014532ca308221756b7858c7a"},"cell_type":"code","source":"sess.run(competitionMetric(y_true, y_pred))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b21d701e473502d6ab61f27564069b2210119ec"},"cell_type":"markdown","source":"Create the corresponding batch of bboxes"},{"metadata":{"trusted":true,"_uuid":"13a8839cf69e7a5251f469b751fdcaa278d90f3c"},"cell_type":"code","source":"batch_box_true = [ [[20, 10, 15, 30]] for _ in range(24)]\nbatch_box_pred = [ [[100, 105, 15, 10]] for _ in range(5)] +\\\n                 [ [[100, 105, 15, 10], [20, 19, 17, 18]] for _ in range(5)] +\\\n                 [ [[20, 19, 17, 18]] for _ in range(14)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83ae10700c6d3076127e2f8a3345d43e939be39d"},"cell_type":"code","source":"np.mean([map_iou(box_true, box_pred) for box_true, box_pred in zip(batch_box_true, batch_box_pred)])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}