{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Custom Evaluation Script for MMDetection","metadata":{}},{"cell_type":"markdown","source":"The goal of this notebook is to define an eval function based on the competition metric. I wanted to acknowledge [Theo Viel](https://www.kaggle.com/theoviel) for contributions to the helper functions as desribed here: https://www.kaggle.com/theoviel/competition-metric-map-iou and my teammate [KKY](https://www.kaggle.com/evilpsycho42) for the work done for this notebook. \n\nAt a high level, the masks of the ground truths (val set) are compared to the mask predictions (not RLE) over the range of thresholds specified in the competition metric. \n\n**One pending issue is that the score seems too low. I am hoping that more people taking a look at this can help us root cause the issue.**","metadata":{}},{"cell_type":"markdown","source":"## Dependencies","metadata":{}},{"cell_type":"code","source":"!pip install '/kaggle/input/pytorch-170-cuda-toolkit-110221/torch-1.7.0+cu110-cp37-cp37m-linux_x86_64.whl' --no-deps\n!pip install '/kaggle/input/pytorch-170-cuda-toolkit-110221/torchvision-0.8.1+cu110-cp37-cp37m-linux_x86_64.whl' --no-deps\n!pip install '/kaggle/input/pytorch-170-cuda-toolkit-110221/torchaudio-0.7.0-cp37-cp37m-linux_x86_64.whl' --no-deps\n\n!pip install '/kaggle/input/mmdetectionv2140/addict-2.4.0-py3-none-any.whl' --no-deps\n!pip install '/kaggle/input/mmdetectionv2140/yapf-0.31.0-py2.py3-none-any.whl' --no-deps\n!pip install '/kaggle/input/mmdetectionv2140/terminal-0.4.0-py3-none-any.whl' --no-deps\n!pip install '/kaggle/input/mmdetectionv2140/terminaltables-3.1.0-py3-none-any.whl' --no-deps\n!pip install '/kaggle/input/mmdetectionv2140/mmcv_full-1_3_8-cu110-torch1_7_0/mmcv_full-1.3.8-cp37-cp37m-manylinux1_x86_64.whl' --no-deps\n!pip install '/kaggle/input/mmdetectionv2140/pycocotools-2.0.2/pycocotools-2.0.2' --no-deps\n!pip install '/kaggle/input/mmdetectionv2140/mmpycocotools-12.0.3/mmpycocotools-12.0.3' --no-deps\n\n!rm -rf mmdetection\n\n!cp -r /kaggle/input/mmdetectionv2140/mmdetection-2.14.0 /kaggle/working/\n!mv /kaggle/working/mmdetection-2.14.0 /kaggle/working/mmdetection\n%cd /kaggle/working/mmdetection\n!pip install -e .\n\n# %% [code] {\"papermill\":{\"duration\":0.077806,\"end_time\":\"2021-10-28T17:40:33.91703\",\"exception\":false,\"start_time\":\"2021-10-28T17:40:33.839224\",\"status\":\"completed\"},\"tags\":[],\"execution\":{\"iopub.status.busy\":\"2021-11-10T17:41:11.575075Z\",\"iopub.execute_input\":\"2021-11-10T17:41:11.576605Z\",\"iopub.status.idle\":\"2021-11-10T17:41:11.583348Z\",\"shell.execute_reply.started\":\"2021-11-10T17:41:11.576373Z\",\"shell.execute_reply\":\"2021-11-10T17:41:11.582666Z\"}}\n%cd ..","metadata":{"_uuid":"a419ada0-7a45-4ad7-b86d-4c474f8ebe5a","_cell_guid":"334bcad4-9640-4b58-86b2-61982b9742a3","collapsed":false,"jupyter":{"outputs_hidden":false},"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-03T18:19:51.322917Z","iopub.execute_input":"2021-12-03T18:19:51.323265Z","iopub.status.idle":"2021-12-03T18:25:14.750405Z","shell.execute_reply.started":"2021-12-03T18:19:51.323222Z","shell.execute_reply":"2021-12-03T18:25:14.74948Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd /kaggle/working/mmdetection","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:25:14.752452Z","iopub.execute_input":"2021-12-03T18:25:14.752734Z","iopub.status.idle":"2021-12-03T18:25:14.760807Z","shell.execute_reply.started":"2021-12-03T18:25:14.752697Z","shell.execute_reply":"2021-12-03T18:25:14.759929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Import Libraries**","metadata":{"_uuid":"a65ec487-8dcf-4428-ae3d-65142ec0c082","_cell_guid":"fa2cdd7b-7f9b-4925-bd6a-3518aa43f2c6","papermill":{"duration":0.098388,"end_time":"2021-10-28T17:40:04.980758","exception":false,"start_time":"2021-10-28T17:40:04.88237","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torch.nn.functional as F\nimport sklearn\nimport torchvision\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nimport numpy as np\nimport cupy as cp\nimport gc\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport PIL\nimport json\nfrom PIL import Image, ImageEnhance\nimport albumentations as A\nimport mmdet\nimport mmcv\nfrom albumentations.pytorch import ToTensorV2\nimport seaborn as sns\nimport glob\nfrom pathlib import Path\nimport pycocotools\nfrom pycocotools import mask\nimport numpy.random\nimport random\nimport cv2\nimport re\nimport shutil\nfrom mmdet.datasets import build_dataset\nfrom mmdet.models import build_detector\nfrom mmdet.apis import train_detector\nfrom mmdet.apis import inference_detector, init_detector, show_result_pyplot, set_random_seed\nfrom mmcv import Config\n","metadata":{"_uuid":"c16995fe-3ea6-4b60-b9d0-909b8885373c","_cell_guid":"284f6a96-e100-49de-9c05-bb0a4a81ae7b","collapsed":false,"papermill":{"duration":28.752894,"end_time":"2021-10-28T17:40:33.786328","exception":false,"start_time":"2021-10-28T17:40:05.033434","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-03T18:25:14.762836Z","iopub.execute_input":"2021-12-03T18:25:14.763103Z","iopub.status.idle":"2021-12-03T18:25:38.328988Z","shell.execute_reply.started":"2021-12-03T18:25:14.763066Z","shell.execute_reply":"2021-12-03T18:25:38.328072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd ..","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:25:38.330651Z","iopub.execute_input":"2021-12-03T18:25:38.330938Z","iopub.status.idle":"2021-12-03T18:25:38.340595Z","shell.execute_reply.started":"2021-12-03T18:25:38.330892Z","shell.execute_reply":"2021-12-03T18:25:38.339815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"def rles_to_mask(encs, shape):\n    \"\"\"\n    Decodes a rle.\n    Args:\n        encs (list of str): Rles for each class.\n        shape (tuple [2]): Mask size.\n    Returns:\n        np array [shape]: Mask.\n    \"\"\"\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint)\n    for m, enc in enumerate(encs):\n        if isinstance(enc, np.float) and np.isnan(enc):\n            continue\n        enc_split = enc.split()\n        for i in range(len(enc_split) // 2):\n            start = int(enc_split[2 * i]) - 1\n            length = int(enc_split[2 * i + 1])\n            img[start: start + length] = 1 + m\n    return img.reshape(shape)\n\n\ndef compute_iou(labels, y_pred):\n    \"\"\"\n    Computes the IoU for instance labels and predictions.\n    Args:\n        labels (np array): Labels.\n        y_pred (np array): predictions\n    Returns:\n        np array: IoU matrix, of size true_objects x pred_objects.\n    \"\"\"\n\n    true_objects = len(np.unique(labels))\n    pred_objects = len(np.unique(y_pred))\n\n    # Compute intersection between all objects\n    intersection = np.histogram2d(\n        labels.flatten(), y_pred.flatten(), bins=(true_objects, pred_objects)\n    )[0]\n\n    # Compute areas (needed for finding the union between all objects)\n    area_true = np.histogram(labels, bins=true_objects)[0]\n    area_pred = np.histogram(y_pred, bins=pred_objects)[0]\n    area_true = np.expand_dims(area_true, -1)\n    area_pred = np.expand_dims(area_pred, 0)\n\n    # Compute union\n    union = area_true + area_pred - intersection\n    iou = intersection / union\n    \n    return iou[1:, 1:]  # exclude background\n\n\ndef precision_at(threshold, iou):\n    \"\"\"\n    Computes the precision at a given threshold.\n    Args:\n        threshold (float): Threshold.\n        iou (np array [n_truths x n_preds]): IoU matrix.\n    Returns:\n        int: Number of true positives,\n        int: Number of false positives,\n        int: Number of false negatives.\n    \"\"\"\n    matches = iou > threshold\n    true_positives = np.sum(matches, axis=1) >= 1  # Correct objects\n    false_negatives = np.sum(matches, axis=1) == 0  # Missed objects\n    false_positives = np.sum(matches, axis=0) == 0  # Extra objects\n    tp, fp, fn = (\n        np.sum(true_positives),\n        np.sum(false_positives),\n        np.sum(false_negatives),\n    )\n    return tp, fp, fn\n\n\ndef iou_map(truths, preds, verbose=False):\n    \"\"\"\n    Computes the metric for the competition.\n    Masks contain the segmented pixels where each object has one value associated,\n    and 0 is the background.\n    Args:\n        truths (list of masks): Ground truths.\n        preds (list of masks): Predictions.\n        verbose (int, optional): Whether to print infos. Defaults to 0.\n    Returns:\n        float: mAP.\n    \"\"\"\n    ious = [compute_iou(truth, pred) for truth, pred in zip(truths, preds)]\n    \n\n    if verbose:\n        print(\"Thresh\\tTP\\tFP\\tFN\\tPrec.\")\n\n    prec = []\n    for t in np.arange(0.5, 1.0, 0.05):\n        tps, fps, fns = 0, 0, 0\n        for iou in ious:\n            tp, fp, fn = precision_at(t, iou)\n            tps += tp\n            fps += fp\n            fns += fn\n\n        p = tps / (tps + fps + fns)\n        prec.append(p)\n\n        if verbose:\n            print(\"{:1.3f}\\t{}\\t{}\\t{}\\t{:1.3f}\".format(t, tps, fps, fns, p))\n\n    if verbose:\n        print(\"AP\\t-\\t-\\t-\\t{:1.3f}\".format(np.mean(prec)))\n\n    return np.mean(prec)\n\n\ndef combine_mask(masks):\n    zeros = np.zeros([520, 704])\n    for i, m in enumerate(masks):\n        zeros[m] = i+1\n    return zeros\n\ndef get_mask_from_result(result):\n    d = {True : 1, False : 0}\n    u,inv = np.unique(result,return_inverse = True)\n    mk = np.array([d[x] for x in u])[inv].reshape(result.shape)\n    return mk\n\ndef remove_overlapping_pixels(mask, other_masks):\n    for other_mask in other_masks:\n        if np.sum(np.logical_and(mask, other_mask)) > 0:\n            #print(\"Overlap detected\")\n            mask[np.logical_and(mask, other_mask)] = 0\n    return mask\n\n","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:25:38.343251Z","iopub.execute_input":"2021-12-03T18:25:38.343605Z","iopub.status.idle":"2021-12-03T18:25:38.366954Z","shell.execute_reply.started":"2021-12-03T18:25:38.343567Z","shell.execute_reply":"2021-12-03T18:25:38.366153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile ./mmdetection/configs/mask_rcnn/mask_rcnn_r50_fpn_2x_coco_test.py\n\n_base_ = [\n    '../_base_/models/mask_rcnn_r50_fpn.py',\n    '../_base_/datasets/coco_instance.py',\n    '../_base_/schedules/schedule_2x.py', '../_base_/default_runtime.py'\n]","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:28:06.104467Z","iopub.execute_input":"2021-12-03T18:28:06.105243Z","iopub.status.idle":"2021-12-03T18:28:06.111451Z","shell.execute_reply.started":"2021-12-03T18:28:06.105196Z","shell.execute_reply":"2021-12-03T18:28:06.110664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model paths","metadata":{}},{"cell_type":"code","source":"root_dir = '../input/sartorius-cell-instance-segmentation/'\nconfig_base = './mmdetection/configs/'\n\nclass CustomConfig():\n    config = config_base + 'mask_rcnn/mask_rcnn_r50_fpn_2x_coco.py'\n    model_path = '../input/mrcnn-1123-aug-ep30/epoch_30.pth'\n    debug = True\n    \nargs = CustomConfig()\n","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:29:27.314201Z","iopub.execute_input":"2021-12-03T18:29:27.314577Z","iopub.status.idle":"2021-12-03T18:29:27.319935Z","shell.execute_reply.started":"2021-12-03T18:29:27.314542Z","shell.execute_reply":"2021-12-03T18:29:27.319212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run Eval ","metadata":{}},{"cell_type":"code","source":"val_json = json.load(open(\"../input/sartorius-cell-instance-segmentation-coco/annotations_val.json\", \"r\"))\ncfg = Config.fromfile(args.config)\nmodel_dir = args.model_path\n\ncfg.model.roi_head.bbox_head.num_classes = 3\ncfg.model.roi_head.mask_head.num_classes=3\n\nmodel = init_detector(cfg, model_dir)\n\nconfidence_thresholds = {0: 0.5, 1: 0.5, 2: 0.5}\n\n@torch.no_grad()\ndef calculate_custom_score(model, val_json, root_dir):\n    df = pd.read_csv(root_dir + \"train.csv\")\n    val_ids = [i['id'] for i in val_json['images']]\n    if args.debug:\n        val_ids = val_ids[0:args.debug]\n    df_val = df[df.id.isin(val_ids)].reset_index(drop=True)\n    df_val = df_val.groupby('id').agg(list).reset_index()\n    del df\n    for col in df_val.columns[2:]:\n        df_val[col] = df_val[col].apply(\n            lambda x: np.unique(x)[0] if len(np.unique(x)) == 1 else np.unique(x)\n        )\n    \n    gts = []\n    dts = []\n    for img_id in df_val.id.tolist():\n        \n        img = mmcv.imread(root_dir + f\"train/{img_id}.png\")\n        result = inference_detector(model, img)\n        \n        # dt\n        dt = []\n        for cls, bbs in enumerate(result[0]):\n            if bbs.shape != (0, 5):\n                sgs = result[1][cls]\n                for bb, sg in zip(bbs, sgs):\n                    box = bb[:4]\n                    cnf = bb[4]\n                    if cnf >= confidence_thresholds[cls]:\n                        mask = get_mask_from_result(sg)\n                        mask = remove_overlapping_pixels(mask, dt)\n                        dt.append(mask.astype(bool))\n                        #print(mask)\n        dts.append(combine_mask(dt))\n        \n        # gt\n        shape = df_val.loc[df_val.id == img_id, ['height', 'width']].values[0]\n        gt = rles_to_mask(df_val.loc[df_val.id == img_id, \"annotation\"].item(), shape).astype(np.uint16)\n        gts.append(gt)\n    score = iou_map(gts, dts, True)\n    return score\n\n\nscore  = calculate_custom_score(model, val_json, root_dir)\n","metadata":{"_uuid":"ba46a643-3668-4024-baaa-187cc67a3ee3","_cell_guid":"aa57267c-8611-431a-813d-0e9df6b11446","collapsed":false,"papermill":{"duration":0.784288,"end_time":"2021-10-28T17:40:35.695074","exception":false,"start_time":"2021-10-28T17:40:34.910786","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-12-03T18:29:31.266931Z","iopub.execute_input":"2021-12-03T18:29:31.267806Z","iopub.status.idle":"2021-12-03T18:29:44.222036Z","shell.execute_reply.started":"2021-12-03T18:29:31.267746Z","shell.execute_reply":"2021-12-03T18:29:44.221128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### The average precision (AP) seems too low for the model, but it does provide a framework to eval the model for mmdetection. ","metadata":{}},{"cell_type":"code","source":"shutil.rmtree('/kaggle/working/mmdetection')","metadata":{},"execution_count":null,"outputs":[]}]}