{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Implementation of the mean Average Precision (mAP) at different intersection over union (IoU)\n\nAs described [here](https://www.kaggle.com/c/sartorius-cell-instance-segmentation/overview/evaluation)\n\nCode adapted from [here](https://www.kaggle.com/wcukierski/example-metric-implementation) (please upvote this kernel as well), as the metric should be the same as the 2018 DSB one.","metadata":{}},{"cell_type":"code","source":"import os\nimport skimage\nimport numpy as np\nimport pandas as pd\nimport skimage.segmentation\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-24T02:13:20.818361Z","iopub.execute_input":"2022-05-24T02:13:20.818801Z","iopub.status.idle":"2022-05-24T02:13:20.824234Z","shell.execute_reply.started":"2022-05-24T02:13:20.818762Z","shell.execute_reply":"2022-05-24T02:13:20.823366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Example","metadata":{}},{"cell_type":"code","source":"def rles_to_mask(encs, shape):\n    \"\"\"\n    Decodes a rle.\n\n    Args:\n        encs (list of str): Rles for each class.\n        shape (tuple [2]): Mask size.\n\n    Returns:\n        np array [shape]: Mask.\n    \"\"\"\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint)\n    for m, enc in enumerate(encs):\n        if isinstance(enc, np.float) and np.isnan(enc):\n            continue\n        enc_split = enc.split()\n        for i in range(len(enc_split) // 2):\n            start = int(enc_split[2 * i]) - 1\n            length = int(enc_split[2 * i + 1])\n            img[start: start + length] = 1 + m\n    return img.reshape(shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T02:14:24.218078Z","iopub.execute_input":"2022-05-24T02:14:24.218914Z","iopub.status.idle":"2022-05-24T02:14:24.226233Z","shell.execute_reply.started":"2022-05-24T02:14:24.218854Z","shell.execute_reply":"2022-05-24T02:14:24.225506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/sartorius-cell-instance-segmentation/train.csv')\ndf = df.groupby('id').agg(list).reset_index()\n\nfor col in df.columns[2:]:\n    df[col] = df[col].apply(\n        lambda x: np.unique(x)[0] if len(np.unique(x)) == 1 else np.unique(x)\n    )","metadata":{"execution":{"iopub.status.busy":"2022-05-24T02:13:29.183520Z","iopub.execute_input":"2022-05-24T02:13:29.183801Z","iopub.status.idle":"2022-05-24T02:13:30.526755Z","shell.execute_reply.started":"2022-05-24T02:13:29.183770Z","shell.execute_reply":"2022-05-24T02:13:30.525815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ground truth","metadata":{}},{"cell_type":"code","source":"i = 0  # feel free to change that\n\nshape = df[['height', 'width']].values[i]\n\nrles = df['annotation'][i]\n\nmasks = rles_to_mask(rles, shape).astype(np.uint16)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T02:14:31.366987Z","iopub.execute_input":"2022-05-24T02:14:31.367239Z","iopub.status.idle":"2022-05-24T02:14:31.392767Z","shell.execute_reply.started":"2022-05-24T02:14:31.367212Z","shell.execute_reply":"2022-05-24T02:14:31.391945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nplt.imshow(masks)\nplt.axis(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T02:14:56.294908Z","iopub.execute_input":"2022-05-24T02:14:56.295503Z","iopub.status.idle":"2022-05-24T02:14:56.581948Z","shell.execute_reply.started":"2022-05-24T02:14:56.295447Z","shell.execute_reply":"2022-05-24T02:14:56.581099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Simulated prediction","metadata":{}},{"cell_type":"code","source":"# offset pixels\noffset = 1\ny_pred = masks[offset:, offset:]\ny_pred = np.pad(y_pred, ((0, offset), (0, offset)), mode=\"constant\")\n\n# Remove a bunch of cells\ny_pred[y_pred > 300] = 0 \n\n# Relabel objects\ny_pred, _, _ = skimage.segmentation.relabel_sequential(y_pred) ","metadata":{"execution":{"iopub.status.busy":"2022-05-24T02:38:15.605122Z","iopub.execute_input":"2022-05-24T02:38:15.605844Z","iopub.status.idle":"2022-05-24T02:38:15.631134Z","shell.execute_reply.started":"2022-05-24T02:38:15.605790Z","shell.execute_reply":"2022-05-24T02:38:15.630304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nplt.imshow(y_pred)\nplt.axis(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T02:38:33.415549Z","iopub.execute_input":"2022-05-24T02:38:33.415826Z","iopub.status.idle":"2022-05-24T02:38:33.680304Z","shell.execute_reply.started":"2022-05-24T02:38:33.415796Z","shell.execute_reply":"2022-05-24T02:38:33.679262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metric","metadata":{}},{"cell_type":"markdown","source":"## IoU","metadata":{}},{"cell_type":"code","source":"def compute_iou(labels, y_pred):\n    \"\"\"\n    Computes the IoU for instance labels and predictions.\n\n    Args:\n        labels (np array): Labels.\n        y_pred (np array): predictions\n\n    Returns:\n        np array: IoU matrix, of size true_objects x pred_objects.\n    \"\"\"\n\n    true_objects = len(np.unique(labels))\n    pred_objects = len(np.unique(y_pred))\n\n    # Compute intersection between all objects\n    intersection = np.histogram2d(\n        labels.flatten(), y_pred.flatten(), bins=(true_objects, pred_objects)\n    )[0]\n\n    # Compute areas (needed for finding the union between all objects)\n    area_true = np.histogram(labels, bins=true_objects)[0]\n    area_pred = np.histogram(y_pred, bins=pred_objects)[0]\n    area_true = np.expand_dims(area_true, -1)\n    area_pred = np.expand_dims(area_pred, 0)\n\n    # Compute union\n    union = area_true + area_pred - intersection\n    iou = intersection / union\n    \n    return iou[1:, 1:]  # exclude background","metadata":{"execution":{"iopub.status.busy":"2022-05-24T02:41:41.257883Z","iopub.execute_input":"2022-05-24T02:41:41.258722Z","iopub.status.idle":"2022-05-24T02:41:41.266366Z","shell.execute_reply.started":"2022-05-24T02:41:41.258685Z","shell.execute_reply":"2022-05-24T02:41:41.265595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Precision","metadata":{}},{"cell_type":"code","source":"def precision_at(threshold, iou):\n    \"\"\"\n    Computes the precision at a given threshold.\n\n    Args:\n        threshold (float): Threshold.\n        iou (np array [n_truths x n_preds]): IoU matrix.\n\n    Returns:\n        int: Number of true positives,\n        int: Number of false positives,\n        int: Number of false negatives.\n    \"\"\"\n    matches = iou > threshold\n    true_positives = np.sum(matches, axis=1) >= 1  # Correct objects\n    false_negatives = np.sum(matches, axis=1) == 0  # Missed objects\n    false_positives = np.sum(matches, axis=0) == 0  # Extra objects\n    tp, fp, fn = (\n        np.sum(true_positives),\n        np.sum(false_positives),\n        np.sum(false_negatives),\n    )\n    return tp, fp, fn","metadata":{"execution":{"iopub.status.busy":"2022-05-24T02:39:01.181866Z","iopub.execute_input":"2022-05-24T02:39:01.182184Z","iopub.status.idle":"2022-05-24T02:39:01.188576Z","shell.execute_reply.started":"2022-05-24T02:39:01.182150Z","shell.execute_reply":"2022-05-24T02:39:01.187788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Overall Metric","metadata":{}},{"cell_type":"markdown","source":"### IoU","metadata":{}},{"cell_type":"code","source":"def iou_map(truths, preds, verbose=0):\n    \"\"\"\n    Computes the metric for the competition.\n    Masks contain the segmented pixels where each object has one value associated,\n    and 0 is the background.\n\n    Args:\n        truths (list of masks): Ground truths.\n        preds (list of masks): Predictions.\n        verbose (int, optional): Whether to print infos. Defaults to 0.\n\n    Returns:\n        float: mAP.\n    \"\"\"\n    ious = [compute_iou(truth, pred) for truth, pred in zip(truths, preds)]\n    \n    print(ious[0].shape)\n\n    if verbose:\n        print(\"Thresh\\tTP\\tFP\\tFN\\tPrec.\")\n\n    prec = []\n    for t in np.arange(0.5, 1.0, 0.05):\n        tps, fps, fns = 0, 0, 0\n        for iou in ious:\n            tp, fp, fn = precision_at(t, iou)\n            tps += tp\n            fps += fp\n            fns += fn\n\n        p = tps / (tps + fps + fns)\n        prec.append(p)\n\n        if verbose:\n            print(\"{:1.3f}\\t{}\\t{}\\t{}\\t{:1.3f}\".format(t, tps, fps, fns, p))\n\n    if verbose:\n        print(\"AP\\t-\\t-\\t-\\t{:1.3f}\".format(np.mean(prec)))\n\n    return np.mean(prec)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T02:41:21.273165Z","iopub.execute_input":"2022-05-24T02:41:21.273579Z","iopub.status.idle":"2022-05-24T02:41:21.282695Z","shell.execute_reply.started":"2022-05-24T02:41:21.273537Z","shell.execute_reply":"2022-05-24T02:41:21.282009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Compute","metadata":{}},{"cell_type":"code","source":"iou_map([masks] * 5, [masks] * 5, verbose=1)  # This should score 1","metadata":{"execution":{"iopub.status.busy":"2021-11-17T20:08:11.517835Z","iopub.execute_input":"2021-11-17T20:08:11.518103Z","iopub.status.idle":"2021-11-17T20:08:11.799653Z","shell.execute_reply.started":"2021-11-17T20:08:11.518076Z","shell.execute_reply":"2021-11-17T20:08:11.798935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iou_map([masks] , [y_pred], verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-11-17T20:08:11.84219Z","iopub.execute_input":"2021-11-17T20:08:11.84245Z","iopub.status.idle":"2021-11-17T20:08:11.900468Z","shell.execute_reply.started":"2021-11-17T20:08:11.842422Z","shell.execute_reply":"2021-11-17T20:08:11.89966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hope this helps !\n\nNote that the averaging is made at cell level here.\nTo compute the evaluation metric as intended, use the previous cell and loop over all the ground truths and predictions.\n\nPlease let me know if I made any mistakes.","metadata":{}}]}