{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Evaluate using competition mAP metric\nThanks @theoviel for the starter kernel [here](https://www.kaggle.com/theoviel/competition-metric-map-iou).<br>\nThis notebook uses rle masks to calculate iou. So, it should be useful for everyone using different models.","metadata":{}},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import os\nimport skimage\nimport numpy as np\nimport pandas as pd\nimport skimage.segmentation\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-12T05:55:34.444218Z","iopub.execute_input":"2021-12-12T05:55:34.444581Z","iopub.status.idle":"2021-12-12T05:55:36.342721Z","shell.execute_reply.started":"2021-12-12T05:55:34.444491Z","shell.execute_reply":"2021-12-12T05:55:36.341738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Helper Functions","metadata":{}},{"cell_type":"code","source":"def rle_decode(mask_rle, shape=[520, 704], color=1):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0::2], s[1::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo : hi] = color\n    return img.reshape(shape)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:55:36.344577Z","iopub.execute_input":"2021-12-12T05:55:36.34488Z","iopub.status.idle":"2021-12-12T05:55:36.357106Z","shell.execute_reply.started":"2021-12-12T05:55:36.34484Z","shell.execute_reply":"2021-12-12T05:55:36.35605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_iou_binary(gt_mask, pred_mask):\n    \"calculate iou between two binary masks.\"\n    try:\n        gt_mask = rle_decode(gt_mask).flatten()\n        pred_mask = rle_decode(pred_mask).flatten()\n        area1 = np.histogram(gt_mask, bins=2)[0]\n        area2 = np.histogram(pred_mask, bins=2)[0]\n        intersection = np.histogram2d(gt_mask, pred_mask, bins=(2, 2) )[0]\n        union = np.expand_dims(area1,1) + np.expand_dims(area2,0) - intersection\n        iou = intersection/union\n        return iou[1,1]\n    except Exception as e:\n        print(\"exception at: calc_iou_binary\")\n        return 0.0\n","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:55:36.358855Z","iopub.execute_input":"2021-12-12T05:55:36.359438Z","iopub.status.idle":"2021-12-12T05:55:36.3694Z","shell.execute_reply.started":"2021-12-12T05:55:36.359398Z","shell.execute_reply":"2021-12-12T05:55:36.368534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_iou(labels, y_pred):\n    \"\"\"\n    Computes the IoU for instance labels and predictions.\n    Args:\n        labels (list): labels.\n        y_pred (list): predictions.\n\n    Returns:\n        np.array: IoU matrix, of size true_objects x pred_objects.\n    \"\"\"\n\n    true_objects = len(labels)\n    pred_objects = len(y_pred)\n    # initialize iou matrix\n    iou_mat = np.zeros((true_objects, pred_objects))\n    # compute iou for each true objects and pred objects\n    for t in range(true_objects):\n        for p in range(pred_objects):\n            iou_mat[t,p] = calc_iou_binary(labels[t], y_pred[p])\n\n    return iou_mat","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:55:36.371302Z","iopub.execute_input":"2021-12-12T05:55:36.371717Z","iopub.status.idle":"2021-12-12T05:55:36.380377Z","shell.execute_reply.started":"2021-12-12T05:55:36.371689Z","shell.execute_reply":"2021-12-12T05:55:36.379587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def precision_at(threshold, iou):\n    \"\"\"\n    Computes the precision at a given threshold.\n\n    Args:\n        threshold (float): Threshold.\n        iou (np array [n_truths x n_preds]): IoU matrix.\n\n    Returns:\n        int: Number of true positives,\n        int: Number of false positives,\n        int: Number of false negatives.\n    \"\"\"\n    matches = iou > threshold\n    true_positives = np.sum(matches, axis=1) >= 1  # Correct objects\n    false_negatives = np.sum(matches, axis=1) == 0  # Missed objects\n    false_positives = np.sum(matches, axis=0) == 0  # Extra objects\n    tp, fp, fn = (\n        np.sum(true_positives),\n        np.sum(false_positives),\n        np.sum(false_negatives),\n    )\n    return tp, fp, fn","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:55:36.381461Z","iopub.execute_input":"2021-12-12T05:55:36.382201Z","iopub.status.idle":"2021-12-12T05:55:36.391978Z","shell.execute_reply.started":"2021-12-12T05:55:36.382141Z","shell.execute_reply":"2021-12-12T05:55:36.391191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def iou_map(truths, preds, verbose=0):\n    \"\"\"\n    Computes the metric for the competition.\n    Masks contain the segmented pixels where each object has one value associated,\n    and 0 is the background.\n\n    Args:\n        truths (list of list of masks): Ground truths.\n        preds (list of list of masks): Predictions.\n        verbose (int, optional): Whether to print infos. Defaults to 0.\n\n    Returns:\n        float: mAP.\n    \"\"\"\n    ious = [compute_iou(truth, pred) for truth, pred in tqdm(zip(truths, preds))]\n    \n\n    if verbose:\n        print(\"Thresh\\tTP\\tFP\\tFN\\tPrec.\")\n\n    prec = []\n    for t in np.arange(0.5, 1.0, 0.05):\n        tps, fps, fns = 0, 0, 0\n        for iou in ious:\n            tp, fp, fn = precision_at(t, iou)\n            tps += tp\n            fps += fp\n            fns += fn\n\n        p = tps / (tps + fps + fns)\n        prec.append(p)\n\n        if verbose:\n            print(\"{:1.3f}\\t{}\\t{}\\t{}\\t{:1.3f}\".format(t, tps, fps, fns, p))\n\n    if verbose:\n        print(\"AP\\t-\\t-\\t-\\t{:1.3f}\".format(np.mean(prec)))\n\n    return np.mean(prec)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:55:36.393083Z","iopub.execute_input":"2021-12-12T05:55:36.393824Z","iopub.status.idle":"2021-12-12T05:55:36.407705Z","shell.execute_reply.started":"2021-12-12T05:55:36.393783Z","shell.execute_reply":"2021-12-12T05:55:36.406833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Prediction masks and Truth masks\nLoad both prediction and truth masks in rle.","metadata":{}},{"cell_type":"markdown","source":"### Prediction masks","metadata":{}},{"cell_type":"code","source":"from ast import literal_eval\npreds_df = pd.read_csv('/kaggle/input/sartvaldataset/val_predictions.csv')\npreds_df['masks'] = preds_df['masks'].apply(lambda x: list(literal_eval(x))) # convert to list from str\nids, masks = preds_df['ids'], preds_df['masks']","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:55:36.409142Z","iopub.execute_input":"2021-12-12T05:55:36.409557Z","iopub.status.idle":"2021-12-12T05:55:36.47645Z","shell.execute_reply.started":"2021-12-12T05:55:36.409514Z","shell.execute_reply":"2021-12-12T05:55:36.475399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Truth masks","metadata":{}},{"cell_type":"code","source":"TRAIN_DIR = '/kaggle/input/sartorius-cell-instance-segmentation/train'\nval_df = pd.read_csv('/kaggle/input/sartoriuskfold/train_folds.csv')\nval_df = val_df[val_df.fold==1]\nval_df['file_path'] = val_df['id'].apply(lambda x: os.path.join(TRAIN_DIR, x+'.png'))","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:55:36.47762Z","iopub.execute_input":"2021-12-12T05:55:36.483014Z","iopub.status.idle":"2021-12-12T05:55:36.506901Z","shell.execute_reply.started":"2021-12-12T05:55:36.482955Z","shell.execute_reply":"2021-12-12T05:55:36.506318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/sartorius-cell-instance-segmentation/train.csv')\ndf = df.groupby('id').agg(list).reset_index()\ndf = df.merge(val_df, on='id')\n\ndf['annotation'] = df['annotation'].apply(\n        lambda x: np.unique(x)[0] if len(np.unique(x)) == 1 else np.unique(x)\n    )\n# create list if list of truth masks\ntruth_df = df[['id','annotation']]\ntruth_masks = []\nfor ann in truth_df['annotation']:\n    truth_masks.append(ann) # append list(ann) to truth list","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:55:36.507864Z","iopub.execute_input":"2021-12-12T05:55:36.50841Z","iopub.status.idle":"2021-12-12T05:55:37.233998Z","shell.execute_reply.started":"2021-12-12T05:55:36.508375Z","shell.execute_reply":"2021-12-12T05:55:37.2331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert len(ids)==len(masks)==len(truth_masks)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:55:37.235829Z","iopub.execute_input":"2021-12-12T05:55:37.236068Z","iopub.status.idle":"2021-12-12T05:55:37.240087Z","shell.execute_reply.started":"2021-12-12T05:55:37.236039Z","shell.execute_reply":"2021-12-12T05:55:37.239352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_masks = [masks[i] for i in [0, 1, 2, 3, 6, 8]] # take only 6 images for each cell type. (manually selected.)\nnew_truth_masks = [truth_masks[i] for i in [0, 1, 2, 3, 6, 8]]","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:55:37.240978Z","iopub.execute_input":"2021-12-12T05:55:37.241439Z","iopub.status.idle":"2021-12-12T05:55:37.25444Z","shell.execute_reply.started":"2021-12-12T05:55:37.241406Z","shell.execute_reply":"2021-12-12T05:55:37.253581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert len(new_masks)==len(new_truth_masks)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:55:37.25553Z","iopub.execute_input":"2021-12-12T05:55:37.255759Z","iopub.status.idle":"2021-12-12T05:55:37.266977Z","shell.execute_reply.started":"2021-12-12T05:55:37.255719Z","shell.execute_reply":"2021-12-12T05:55:37.266022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Compute mAP","metadata":{}},{"cell_type":"code","source":"print(f'Validating on {len(new_masks)} images.')\niou_map(new_truth_masks,new_masks,verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:55:37.268213Z","iopub.execute_input":"2021-12-12T05:55:37.268542Z","iopub.status.idle":"2021-12-12T06:18:52.748361Z","shell.execute_reply.started":"2021-12-12T05:55:37.268514Z","shell.execute_reply":"2021-12-12T06:18:52.747515Z"},"trusted":true},"execution_count":null,"outputs":[]}]}