{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Dependencies","metadata":{}},{"cell_type":"code","source":"# ensemble_boxes 1.0.7\n!pip install ../input/sartorius-cell-instance-segmentation-dataset/packages/packages/ensemble_boxes-1.0.7-py3-none-any.whl\n# detectron2 0.5\n!pip install ../input/sartorius-cell-instance-segmentation-dataset/packages/packages/pycocotools-2.0.2/dist/pycocotools-2.0.2.tar --no-index --find-links ../input/sartorius-cell-instance-segmentation-dataset/packages/packages \n!pip install ../input/sartorius-cell-instance-segmentation-dataset/packages/packages/fvcore-0.1.5.post20211019/fvcore-0.1.5.post20211019 --no-index --find-links ../input/sartorius-cell-instance-segmentation-dataset/packages/packages \n!pip install ../input/sartorius-cell-instance-segmentation-dataset/packages/packages/antlr4-python3-runtime-4.8/antlr4-python3-runtime-4.8 --no-index --find-links ../input/sartorius-cell-instance-segmentation-dataset/packages/packages \n!pip install ../input/sartorius-cell-instance-segmentation-dataset/packages/packages/detectron2-0.5/detectron2 --no-index --find-links ../input/sartorius-cell-instance-segmentation-dataset/packages/packages\n# cellpose 0.7.2\n!pip install --no-index ../input/cellposeoffline/cellpose-0.7.2-py3-none-any.whl --find-links=../input/cellposeoffline","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-30T11:45:59.310357Z","iopub.execute_input":"2021-12-30T11:45:59.310991Z","iopub.status.idle":"2021-12-30T11:50:05.893414Z","shell.execute_reply.started":"2021-12-30T11:45:59.310905Z","shell.execute_reply":"2021-12-30T11:50:05.892379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import yaml\nimport os\nimport json\nfrom glob import glob\nfrom tqdm.notebook import tqdm\nfrom typing import Union\nfrom collections import Counter\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nimport cv2\nfrom scipy.stats import mode\nimport matplotlib.pyplot as plt\nfrom numba import jit\nimport torch\nfrom fastai.vision.all import *\nimport detectron2\nimport detectron2.layers\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nimport pycocotools.mask as mask_util\nimport networkx as nx","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-30T11:50:05.906065Z","iopub.execute_input":"2021-12-30T11:50:05.907116Z","iopub.status.idle":"2021-12-30T11:50:08.748501Z","shell.execute_reply.started":"2021-12-30T11:50:05.907066Z","shell.execute_reply":"2021-12-30T11:50:08.747736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RAW_DATASET_PATH = '../input/sartorius-cell-instance-segmentation'\nDETECTRON_MODELS_PATH = '../input/sartorius-cell-instance-segmentation-dataset'\nCLASSIFIER_PATH = '../input/sartorius-fastai-classifier'\nCELLPOSE_MODELS_PATH = '../input/sartoriuscellposemodels'","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:50:08.794016Z","iopub.execute_input":"2021-12-30T11:50:08.794630Z","iopub.status.idle":"2021-12-30T11:50:08.800357Z","shell.execute_reply.started":"2021-12-30T11:50:08.794595Z","shell.execute_reply":"2021-12-30T11:50:08.799699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(f'{RAW_DATASET_PATH}/train.csv')\ndf_test = pd.read_csv(f'{RAW_DATASET_PATH}/sample_submission.csv')\n\nprint(f'Training Set Shape: {df_train.shape} - {df_train[\"id\"].nunique()} Images - Memory Usage: {df_train.memory_usage().sum() / 1024 ** 2:.2f} MB')\nprint(f'Test Set Shape: {df_test.shape} - {df_test[\"id\"].nunique()} Images - Memory Usage: {df_test.memory_usage().sum() / 1024 ** 2:.2f} MB')","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:50:08.802322Z","iopub.execute_input":"2021-12-30T11:50:08.802931Z","iopub.status.idle":"2021-12-30T11:50:09.502875Z","shell.execute_reply.started":"2021-12-30T11:50:08.802893Z","shell.execute_reply":"2021-12-30T11:50:09.502026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotation Utilities","metadata":{}},{"cell_type":"code","source":"def decode_rle_mask(rle_mask, shape, fill_holes=False, is_coco_encoded=False):\n\n    \"\"\"\n    Decode run-length encoded mask string into 2d binary mask array\n\n    Parameters\n    ----------\n    rle_mask (str): Run-length encoded mask string\n    shape (tuple): Height and width of the mask\n    fill_holes (bool): Whether to fill holes in masks or not\n    is_coco_encoded (bool): Whether the mask is encoded with pycocotools or not\n\n    Returns\n    -------\n    mask [numpy.ndarray of shape (height, width)]: Decoded 2d mask\n    \"\"\"\n\n    if is_coco_encoded:\n        # Decoding RLE encoded mask of string\n        mask = np.uint8(mask_utils.decode({'size': shape, 'counts': rle_mask}))\n    else:\n        # Decoding RLE encoded mask of integers\n        rle_mask = rle_mask.split()\n        starts, lengths = [np.asarray(x, dtype=int) for x in (rle_mask[0:][::2], rle_mask[1:][::2])]\n        starts -= 1\n        ends = starts + lengths\n\n        mask = np.zeros((shape[0] * shape[1]), dtype=np.uint8)\n        for start, end in zip(starts, ends):\n            mask[start:end] = 1\n\n        mask = mask.reshape(shape[0], shape[1])\n\n    if fill_holes:\n        mask = ndimage.binary_fill_holes(mask).astype(np.uint8)\n\n    return mask\n\n\ndef encode_rle_mask(mask):\n\n    \"\"\"\n    Encode 2d binary mask array into run-length encoded mask string\n\n    Parameters\n    ----------\n    mask [numpy.ndarray of shape (height, width)]: 2d mask\n\n    Returns\n    -------\n    rle_mask (str): Run-length encoded mask string\n    \"\"\"\n\n    mask = mask.flatten()\n    mask = np.concatenate([[0], mask, [0]])\n    runs = np.where(mask[1:] != mask[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\n\ndef binary_to_multi_object_mask(binary_masks):\n\n    \"\"\"\n    Encode multiple 2d binary masks into a single 2d multi-object segmentation mask\n\n    Parameters\n    ----------\n    binary_masks [numpy.ndarray of shape (n_objects, height, width)]: 2d binary masks\n\n    Returns\n    -------\n    multi_object_mask [numpy.ndarray of shape (height, width)]: 2d multi-object mask\n    \"\"\"\n\n    multi_object_mask = np.zeros((binary_masks.shape[1], binary_masks.shape[2]))\n    for i, binary_mask in enumerate(binary_masks):\n        non_zero_idx = binary_mask == 1\n        multi_object_mask[non_zero_idx] = i + 1\n\n    return multi_object_mask\n\n\ndef mask_to_bounding_box(mask):\n\n    \"\"\"\n    Get bounding box from a binary mask\n\n    Parameters\n    ----------\n    mask [numpy.ndarray of shape (height, width)]: 2d binary mask\n\n    Returns\n    -------\n    bounding_box [list of shape (4)]: Bounding box of the object\n    \"\"\"\n\n    non_zero_idx = np.where(mask == 1)\n    bounding_box = [\n        np.min(non_zero_idx[1]),\n        np.min(non_zero_idx[0]),\n        np.max(non_zero_idx[1]),\n        np.max(non_zero_idx[0])\n    ]\n\n    return bounding_box\n","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:50:09.504365Z","iopub.execute_input":"2021-12-30T11:50:09.504925Z","iopub.status.idle":"2021-12-30T11:50:09.524033Z","shell.execute_reply.started":"2021-12-30T11:50:09.504886Z","shell.execute_reply":"2021-12-30T11:50:09.523011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Metric Utilities","metadata":{}},{"cell_type":"code","source":"def precision_at(ious, threshold):\n\n    \"\"\"\n    Get true positives, false positives, false negatives and precision score from given IoUs at given threshold\n\n    Parameters\n    ----------\n    ious [numpy.ndarray of shape (ground_truth_objects, prediction_objects)]: Intersection over union between all ground-truths and predicted segmentation masks\n    threshold (float): Threshold on which the hits are calculated\n\n    Returns\n    -------\n    tp (int): Number of true positives in IoU hit matrix\n    fp (int): Number of false positives in IoU hit matrix\n    fn (int): Number of false negatives in IoU hit matrix\n    precision (float): Precision score of IoU hit matrix (0.0 <= precision <= 1.0)\n    \"\"\"\n\n    hits = ious > threshold\n    true_positives = np.sum(hits, axis=1) == 1\n    false_positives = np.sum(hits, axis=0) == 0\n    false_negatives = np.sum(hits, axis=1) == 0\n    tp, fp, fn = (\n        np.sum(true_positives),\n        np.sum(false_positives),\n        np.sum(false_negatives),\n    )\n    precision = tp / (tp + fp + fn)\n    return tp, fp, fn, precision\n\n\ndef get_average_precision(ground_truth_masks, prediction_masks, thresholds=(0.50, 0.55, 0.60, 0.65, 0.70, 0.75, 0.80, 0.85, 0.90, 0.95), verbose=True):\n\n    \"\"\"\n    Calculate mean Average Precision (mAP) between ground truth and prediction masks on given thresholds\n\n    Parameters\n    ----------\n    ground_truth_masks [numpy.ndarray of shape (n_objects, height, width)]: Ground truth binary segmentation masks\n    prediction_masks [numpy.ndarray of shape (m_objects, height, width)]: Prediction binary segmentation masks\n    thresholds (tuple): Thresholds on which the hits are calculated\n    verbose (bool): Verbosity flag\n\n    Returns\n    -------\n    average_precision (float): Average precision score of IoU hit matrix (0.0 <= average_precision <= 1.0)\n    \"\"\"\n\n    prediction_masks = [mask_util.encode(np.asarray(mask, order='F')) for mask in prediction_masks]\n    ground_truth_masks = [mask_util.encode(np.asarray(mask, order='F')) for mask in ground_truth_masks]\n    ious = mask_util.iou(prediction_masks, ground_truth_masks, [0] * len(ground_truth_masks))\n\n    precisions = []\n    for threshold in thresholds:\n        tp, fp, fn, precision = precision_at(ious=ious, threshold=threshold)\n        precisions.append(precision)\n        if verbose:\n            print(f'Precision: {precision:.6f} (TP: {tp} FP: {fp} FN: {fn}) at Threshold: {threshold:.2f}')\n\n    average_precision = np.mean(precisions)\n    if verbose:\n        print(f'Image Average Precision: {average_precision:.6f}\\n')\n\n    return average_precision\n","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:50:09.525875Z","iopub.execute_input":"2021-12-30T11:50:09.526474Z","iopub.status.idle":"2021-12-30T11:50:09.542354Z","shell.execute_reply.started":"2021-12-30T11:50:09.526337Z","shell.execute_reply":"2021-12-30T11:50:09.541600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization Utilities","metadata":{}},{"cell_type":"code","source":"def visualize_image(image, masks, title, path=None):\n\n    \"\"\"\n    Visualize image along with its segmentation masks\n\n    Parameters\n    ----------\n    image [numpy.ndarray of shape (height, width)]: Grayscale image\n    masks [numpy.ndarray of shape (n_objects, height, width)]: Segmentation masks\n    title (str): Title of the plot\n    path (str or None): Path of the output file (if path is None, plot is displayed with selected backend)\n    \"\"\"\n\n    fig, ax = plt.subplots(figsize=(16, 16))\n    ax.imshow(image, cmap='gray')\n\n    if masks is not None:\n        masks = np.stack(masks)\n        mask = np.any(masks > 0, axis=0)\n        ax.imshow(mask, alpha=0.4)\n\n    ax.set_xlabel('')\n    ax.set_ylabel('')\n    ax.tick_params(axis='x', labelsize=15, pad=10)\n    ax.tick_params(axis='y', labelsize=15, pad=10)\n    ax.set_title(title, size=20, pad=15)\n\n    if path is None:\n        plt.show()\n    else:\n        plt.savefig(path)\n        plt.close(fig)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:50:09.543919Z","iopub.execute_input":"2021-12-30T11:50:09.544692Z","iopub.status.idle":"2021-12-30T11:50:09.556092Z","shell.execute_reply.started":"2021-12-30T11:50:09.544653Z","shell.execute_reply":"2021-12-30T11:50:09.555205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cell Type Classifier Inference","metadata":{}},{"cell_type":"code","source":"from glob import glob\n\nCLASS_NAMES = ['cort', 'astro', 'shsy5y']\nFASTAI_CLF_LEARNER = f'{CLASSIFIER_PATH}/clf_resnet34.pkl'\nFASTAI_CLF_LEARNERS = [f'{CLASSIFIER_PATH}/clf_resnet34_{fold}.pkl' for fold in range(5)]\n\ndf_test = pd.DataFrame(glob(f'{RAW_DATASET_PATH}/test/*'), columns=['image_path'])\ndf_test['id'] = df_test.image_path.map(lambda x: x.split('/')[-1].split('.')[0])\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:50:09.557840Z","iopub.execute_input":"2021-12-30T11:50:09.558161Z","iopub.status.idle":"2021-12-30T11:50:09.587136Z","shell.execute_reply.started":"2021-12-30T11:50:09.558126Z","shell.execute_reply":"2021-12-30T11:50:09.586364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_RGBY_image(image_id):\n\n    filename = f'{RAW_DATASET_PATH}/test/{image_id}.png'\n    img = cv2.imread(filename)[..., 0]\n    clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8, 8))\n    img2 = clahe.apply(img)\n    img3 = cv2.equalizeHist(img)\n    stacked_images = np.stack([img, img2, img3], axis=-1)\n    return stacked_images\n","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:50:09.588284Z","iopub.execute_input":"2021-12-30T11:50:09.588827Z","iopub.status.idle":"2021-12-30T11:50:09.595407Z","shell.execute_reply.started":"2021-12-30T11:50:09.588790Z","shell.execute_reply":"2021-12-30T11:50:09.594322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_image_dir = '/kaggle/work/mmdet_test/'\n!mkdir -p {out_image_dir}\n\nfor idx in tqdm(range(len(df_test))):\n    image_id = df_test.iloc[idx]['id']\n    img = load_RGBY_image(image_id)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    cv2.imwrite(f'{out_image_dir}/{image_id}.png', img)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:50:09.597409Z","iopub.execute_input":"2021-12-30T11:50:09.597678Z","iopub.status.idle":"2021-12-30T11:50:10.521293Z","shell.execute_reply.started":"2021-12-30T11:50:09.597643Z","shell.execute_reply":"2021-12-30T11:50:10.520554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_test_classes(learner_path, test_folder, cpu=True):\n    ids = [os.path.basename(fn).replace('.png', '') for fn in get_image_files(test_folder)]\n    inference_learn = load_learner(\n        learner_path,\n        cpu=cpu,\n    )\n    test_dl = inference_learn.dls.test_dl(get_image_files(test_folder))\n    preds, _ = inference_learn.get_preds(dl=test_dl)\n    preds = preds.detach().cpu().numpy().argmax(1)\n    return {_id: int(pred) for _id, pred in zip(ids, preds)}\n\n# Single CLF model\nid2class = get_test_classes(\n    learner_path=f'{CLASSIFIER_PATH}/clf_resnet34.pkl',\n    test_folder=out_image_dir,\n    cpu=True\n)\n\n# Ensemble of CLF fold models\nid2classes = [\n    get_test_classes(\n        learner_path=learner_fn,\n        test_folder=out_image_dir,\n        cpu=True\n    ) for learner_fn in FASTAI_CLF_LEARNERS\n]\n\n# Sanity check visualization\nf,axs = plt.subplots(1, 3, figsize=(12, 6))\nfor key, i in zip(id2class.keys(), range(3)):\n    _cl = id2class[key]\n    img_fn = os.path.join(out_image_dir, key + '.png')\n    axs[i].imshow(cv2.imread(img_fn))\n    axs[i].set_title(f'{key} - class {CLASS_NAMES[_cl]}')","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:50:10.523108Z","iopub.execute_input":"2021-12-30T11:50:10.524126Z","iopub.status.idle":"2021-12-30T11:50:22.347348Z","shell.execute_reply.started":"2021-12-30T11:50:10.524086Z","shell.execute_reply":"2021-12-30T11:50:22.346582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"majority_id2classes = {}\nfor key in id2class.keys():\n    preds = [d[key] for d in id2classes]\n    counter = Counter(preds)\n    majority_id2classes[key] = counter.most_common(1)[0][0]\n\n# Save predictions to file\nwith open('id2class.json', 'w') as json_file:\n    json.dump(majority_id2classes, json_file)\n    \nmajority_id2classes","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:50:22.348640Z","iopub.execute_input":"2021-12-30T11:50:22.348880Z","iopub.status.idle":"2021-12-30T11:50:22.359080Z","shell.execute_reply.started":"2021-12-30T11:50:22.348849Z","shell.execute_reply":"2021-12-30T11:50:22.358339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cellpose Inference","metadata":{}},{"cell_type":"code","source":"%%writefile cellpose_inference.py\n\nimport numpy as np\nfrom cellpose import models, io, plot\nfrom pathlib import Path\nimport pandas as pd\nimport os\nimport json\nimport pickle\nimport cv2\nfrom pycocotools import mask as maskUtils\n\n\ndef rle_decode(mask_rle, shape, color=1):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros((shape[0] * shape[1], shape[2]), dtype=np.float32)\n    for lo, hi in zip(starts, ends):\n        img[lo : hi] = color\n    return img.reshape(shape)\n\n\ndef build_binary_mask(labels ,input_shape=(520,704)):\n    \"\"\" \"\"\"\n    height, width = input_shape\n    \n    mask = np.zeros((height, width, 1))\n    for label in labels:\n        mask += rle_decode(label, shape=(height, width, 1))\n        \n    mask = mask.clip(0, 1)\n    return mask.astype(np.uint8)\n\n\ndef ious_of_rles(rles):\n    binary_masks = [build_binary_mask([rle]) for rle in rles]\n    \n    if len(binary_masks) == 0:\n        return np.array([[0. for _ in range(len(binary_masks))]])\n    \n    enc_masks = [maskUtils.encode(np.asarray(p[:,:,0], order='F')) for p in binary_masks]\n    ious = maskUtils.iou(enc_masks, enc_masks, [0]*len(enc_masks))\n    \n    return ious\n\n\ndef rle_encode(img):\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\n\ndef is_outlier(idx, ious, iou_th, n_th):\n    return sum(np.where(ious[idx] > iou_th, 1, 0)) < n_th\n\n\ndef get_overlapping_indices(idx, ious, iou_th):\n    return [i[0] for i in np.argwhere(ious[idx] > iou_th) if i[0] != idx]\n\n\ndef iou_sum(idx, ious, weights=None):\n    if weights is not None:\n        return np.sum(weights * ious[idx])\n    return np.sum(ious[idx])\n\n\ndef ensemble_rles(rles, weights=None, low_iou_th=0.2, min_overlaps=3):\n    \"\"\"\n    Combine predictions from multiple models by filtering outliers and less confident replicates.\n    Two rles that overlap more than low_iou_th are replicates. \n    \n    1.  Prediction outliers are removed. A pred with less than min_overlaps replicates is considered\n    to be an outlier.\n    \n    2.  From replicates, only the pred that has the highest sum of overlapping ious is kept.\n    \n    Note that returned preds may contain overlapping rles (can only overlap less than low_iou_th)\n    \n    Argument:\n        rles: list of rle string\n        weights: optional weight for each rle - used for weighting IOU sums in rle copmarison\n        low_iou_th: rles that overlap more than this are considered replicates\n        min_overlaps: if rle has less than (min_overlaps - 1) replicates, it is discarded\n    \n    Returns:\n        filtered_rles: list of rles that remain after filtering operations\n    \"\"\"\n    ious = ious_of_rles(rles)\n    \n    # Check outliers\n    delete_list = [False for _ in rles]\n    probs = []\n    for n in range(len(rles)):\n        delete_list[n] = is_outlier(n, ious, iou_th=low_iou_th, n_th=min_overlaps)\n        \n    for n in range(len(rles)):\n        replicate_idxs = get_overlapping_indices(n, ious, iou_th=low_iou_th)\n        n_sum = iou_sum(n, ious, weights=weights)\n        weight = 1. if weights is None else weights[n]\n        probs.append(weight * n_sum)\n        \n        # compare against replicates and zero out if some of the reps is higher\n        for idx in replicate_idxs:\n            if iou_sum(idx, ious, weights=weights) > n_sum:\n                delete_list[n] = True\n                break\n    \n    rles = [rle for (rle, delete) in zip(rles, delete_list) if not delete]\n    probs = [-1 * p for (p, delete) in zip(probs, delete_list) if not delete]\n    \n    # sort highest probs first\n    probs, rles = zip(*sorted(zip(probs, rles)))\n    \n    return rles\n\ndef remove_overlap_in_rles(rles, shape=(520,704), min_size_left=0.7):\n    \"\"\" Rles should have highest probs first \"\"\"\n    new_rles = []\n    used = np.zeros(shape, dtype=np.int32)\n    for rle in rles:\n        binary_mask = rle_decode(rle, shape=(*shape,1)).astype(np.uint8)[:,:,0]\n        area_before_overlap_remove = binary_mask.sum()\n        binary_mask = binary_mask * (1-used)\n        area_after_overlap_remove = binary_mask.sum()\n        \n        if (area_after_overlap_remove / (area_before_overlap_remove + 1e-6)) < min_size_left:\n            continue\n        used += binary_mask\n        new_rles.append(rle_encode(binary_mask))\n    return new_rles\n\nCLASS_NAMES = ['cort', 'astro', 'shsy5y']\n\n# Cell type class-specific parameters\nCELLPOSE_DIAMETERS = [\n    [20], # cort \n    [30], # astro\n    [20],  # shsy5y\n]\n\nCELLPOSE_MASK_TH = [\n    [0.1],  # cort\n    [-0.4], # astro\n    [0.2],   # shsy5y\n]\n\n# area less than this will get discarded - not used now\nMIN_PIXEL_THS = [\n    15, # cort\n    15, # astro\n    15  # shsy5y\n]\n\nCELLPOSE_FLOW_THS = [\n    0.4, # cort\n    0.5, # astro\n    0.45 # shsy5y\n] \n\n# Read classifier predictions for each id \nwith open('id2class.json') as f:\n    id2class = json.load(f)\n\ntest_dir = Path('../input/sartorius-cell-instance-segmentation/test')\ntest_files = [fname for fname in test_dir.iterdir()]\n\ncellpose_models = [\n    # cort\n    [models.CellposeModel(\n        gpu=True,\n        pretrained_model=[\n            # cyto-1 fold models\n            \"../input/sartoriuscellposemodels/cyto-1/cort/fold-0/ep499\",\n            \"../input/sartoriuscellposemodels/cyto-1/cort/fold-1/ep499\",\n            \"../input/sartoriuscellposemodels/cyto-1/cort/fold-2/ep499\",\n            \"../input/sartoriuscellposemodels/cyto-1/cort/fold-3/ep499\",\n            \"../input/sartoriuscellposemodels/cyto-1/cort/fold-4/ep499\",\n            \n            # cyto-2 (only fold 4 was trained)\n            \"../input/sartoriuscellposemodels/cyto-2/cort/fold-4/ep299\",\n            \n            # cyto-2-pseudo fold models\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/cort/fold-0/ep299\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/cort/fold-1/ep299\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/cort/fold-2/ep299\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/cort/fold-3/ep299\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/cort/fold-4/ep299\",\n            \n            # pseudo-2\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/cort/fold-0/ep101\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/cort/fold-1/ep101\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/cort/fold-2/ep101\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/cort/fold-3/ep101\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/cort/fold-4/ep101\",\n    ])],\n    # astro\n    [models.CellposeModel(\n        gpu=True,\n        pretrained_model=[\n            # cyto-1 fold models\n            \"../input/sartoriuscellposemodels/cyto-1/astro/fold-0/ep499\",\n            \"../input/sartoriuscellposemodels/cyto-1/astro/fold-1/ep499\",\n            \"../input/sartoriuscellposemodels/cyto-1/astro/fold-2/ep499\",\n            \"../input/sartoriuscellposemodels/cyto-1/astro/fold-3/ep499\",\n            \"../input/sartoriuscellposemodels/cyto-1/astro/fold-4/ep499\",\n            \n            # cyto-2\n            \"../input/sartoriuscellposemodels/cyto-2/astro/fold-4/ep299\",\n            \n            # cyto-2-pseudo\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/astro/fold-0/ep299\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/astro/fold-1/ep299\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/astro/fold-2/ep299\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/astro/fold-3/ep299\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/astro/fold-4/ep299\",\n            \n            # pseudo-2\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/astro/fold-0/ep101\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/astro/fold-1/ep101\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/astro/fold-2/ep101\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/astro/fold-3/ep101\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/astro/fold-4/ep101\",\n            \n    ])],\n    # shsy5y\n    [models.CellposeModel(\n        gpu=True,\n        pretrained_model=[\n            # cyto-1\n            \"../input/sartoriuscellposemodels/cyto-1/shsy5y/fold-0/ep159\",\n            \"../input/sartoriuscellposemodels/cyto-1/shsy5y/fold-1/ep159\",\n            \"../input/sartoriuscellposemodels/cyto-1/shsy5y/fold-2/ep159\",\n            \"../input/sartoriuscellposemodels/cyto-1/shsy5y/fold-3/ep159\",\n            \"../input/sartoriuscellposemodels/cyto-1/shsy5y/fold-4/ep159\",\n            \n            # cyto-2\n            \"../input/sartoriuscellposemodels/cyto-2/shsy5y/fold-4/ep159\",\n\n            # cyto-2-pseudo\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/shsy5y/fold-0/ep159\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/shsy5y/fold-1/ep159\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/shsy5y/fold-2/ep159\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/shsy5y/fold-3/ep159\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo/shsy5y/fold-4/ep159\",\n            \n            # pseudo-2\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/shsy5y/fold-0/ep121\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/shsy5y/fold-1/ep121\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/shsy5y/fold-2/ep121\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/shsy5y/fold-3/ep121\",\n            \"../input/sartoriuscellposemodels/cyto-2-pseudo-2/shsy5y/fold-4/ep121\",\n    ])]\n]\n\nids, masks = [],[]\nfor fn in test_files:\n    \n    # get class index\n    fn_basename = os.path.basename(fn).replace('.png','').replace('.tif','').replace('.tiff','')\n    _cl = id2class[fn_basename]\n    print(f'{fn_basename} - class {CLASS_NAMES[_cl]}')\n    flow_th = CELLPOSE_FLOW_THS[_cl]\n    \n    rle_list = []\n    for i, cp_model in enumerate(cellpose_models[_cl]):\n        \n        # Normal image\n        preds, flows, _ = cp_model.eval(\n            io.imread(str(fn)), \n            diameter=CELLPOSE_DIAMETERS[_cl][i],\n            flow_threshold=flow_th,\n            channels=[0,0],\n            augment=True,\n            mask_threshold=CELLPOSE_MASK_TH[_cl][i],\n            resample=True)\n        print(f'CP model {i} predicted {preds.max()} cells.')\n        for j in range (1, preds.max() + 1):\n            rle_list.append(rle_encode(preds == j))\n            \n        # HFlipped image\n        preds, flows, _ = cp_model.eval(\n            cv2.flip(io.imread(str(fn)), 0), \n            diameter=CELLPOSE_DIAMETERS[_cl][i], \n            flow_threshold=flow_th,\n            channels=[0,0],\n            augment=True,\n            mask_threshold=CELLPOSE_MASK_TH[_cl][i],\n            resample=True)\n        preds = cv2.flip(preds, 0)\n        \n        print(f'HFlip - CP model {i} predicted {preds.max()} cells.')\n        for j in range (1, preds.max() + 1):\n            rle_list.append(rle_encode(preds == j))\n            \n        # Transposed image\n        preds, flows, _ = cp_model.eval(\n            np.transpose(io.imread(str(fn)), axes=(1,0)), \n            diameter=CELLPOSE_DIAMETERS[_cl][i],\n            flow_threshold=flow_th,\n            channels=[0,0],\n            augment=True,\n            mask_threshold=CELLPOSE_MASK_TH[_cl][i],\n            resample=True)\n        preds = np.transpose(preds, axes=(1,0))\n        print(f'Transposed - CP model {i} predicted {preds.max()} cells.')\n        for j in range (1, preds.max() + 1):\n            rle_list.append(rle_encode(preds == j))\n            \n        # Transposed HFlipped image\n        preds, flows, _ = cp_model.eval(\n            cv2.flip(np.transpose(io.imread(str(fn)), axes=(1,0)), 0), \n            diameter=CELLPOSE_DIAMETERS[_cl][i],\n            flow_threshold=flow_th,\n            channels=[0,0],\n            augment=True,\n            mask_threshold=CELLPOSE_MASK_TH[_cl][i],\n            resample=True)\n        preds = cv2.flip(np.transpose(preds, axes=(1,0)), 0)\n        print(f'Transposed + HFlip - CP model {i} predicted {preds.max()} cells.')\n        for j in range (1, preds.max() + 1):\n            rle_list.append(rle_encode(preds == j))\n            \n    \n    # 4xTTA ensemble\n    # allow only preds that have min_overlaps replicates\n    filtered_rles = ensemble_rles(rle_list, low_iou_th=0.5, min_overlaps=3)\n    filtered_rles = remove_overlap_in_rles(filtered_rles, min_size_left=0.5)\n    print(f'NMS left {len(filtered_rles)} cells.')\n    #filtered_rles = rle_list\n    \n    for rle in filtered_rles:\n        ids.append(fn.stem)\n        masks.append(rle)\n        \npd.DataFrame({'id':ids, 'predicted':masks}).to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:50:22.361017Z","iopub.execute_input":"2021-12-30T11:50:22.361430Z","iopub.status.idle":"2021-12-30T11:50:22.380668Z","shell.execute_reply.started":"2021-12-30T11:50:22.361391Z","shell.execute_reply":"2021-12-30T11:50:22.379622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python cellpose_inference.py","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:51:08.414379Z","iopub.execute_input":"2021-12-30T11:51:08.414945Z","iopub.status.idle":"2021-12-30T11:53:42.012481Z","shell.execute_reply.started":"2021-12-30T11:51:08.414907Z","shell.execute_reply":"2021-12-30T11:53:42.011604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Detectron2 Patch","metadata":{}},{"cell_type":"code","source":"def paste_masks_in_image(masks, boxes, image_shape, threshold=0.5):\n\n    assert masks.shape[-1] == masks.shape[-2], \"Only square mask predictions are supported\"\n    N = len(masks)\n    if N == 0:\n        return masks.new_empty((0,) + image_shape, dtype=torch.uint8)\n    if not isinstance(boxes, torch.Tensor):\n        boxes = boxes.tensor\n    device = boxes.device\n    assert len(boxes) == N, boxes.shape\n\n    img_h, img_w = image_shape\n\n    # The actual implementation split the input into chunks,\n    # and paste them chunk by chunk.\n    if device.type == \"cpu\":\n        # CPU is most efficient when they are pasted one by one with skip_empty=True\n        # so that it performs minimal number of operations.\n        num_chunks = N\n    else:\n        # GPU benefits from parallelism for larger chunks, but may have memory issue\n        num_chunks = int(np.ceil(N * img_h * img_w * BYTES_PER_FLOAT / GPU_MEM_LIMIT))\n        assert (\n            num_chunks <= N\n        ), \"Default GPU_MEM_LIMIT in mask_ops.py is too small; try increasing it\"\n    chunks = torch.chunk(torch.arange(N, device=device), num_chunks)\n\n    img_masks = torch.zeros(\n        N, img_h, img_w, device=device, dtype=torch.float32\n    )\n    for inds in chunks:\n        masks_chunk, spatial_inds = _do_paste_mask(\n            masks[inds, None, :, :], boxes[inds], img_h, img_w, skip_empty=device.type == \"cpu\"\n        )\n        img_masks[(inds,) + spatial_inds] = masks_chunk\n\n    return img_masks\n\n\ndef BitMasks__init__(self, tensor: Union[torch.Tensor, np.ndarray]):\n\n    device = tensor.device if isinstance(tensor, torch.Tensor) else torch.device(\"cpu\")\n    tensor = torch.as_tensor(tensor, dtype=torch.float32, device=device)\n    assert tensor.dim() == 3, tensor.size()\n    self.image_size = tensor.shape[1:]\n    self.tensor = tensor\n\n\nPATCH = False\n\nif PATCH:\n    detectron2.layers.mask_ops.paste_masks_in_image.__code__ = paste_masks_in_image.__code__\n    print('detectron2.layers.mask_ops.paste_masks_in_image is patched.')\n    detectron2.structures.masks.BitMasks.__init__.__code__ = BitMasks__init__.__code__\n    print('detectron2.structures.masks.BitMasks.init is patched.')\n","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:55:46.921356Z","iopub.execute_input":"2021-12-30T11:55:46.921861Z","iopub.status.idle":"2021-12-30T11:55:46.938033Z","shell.execute_reply.started":"2021-12-30T11:55:46.921814Z","shell.execute_reply":"2021-12-30T11:55:46.936880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Detectron2 Models","metadata":{}},{"cell_type":"code","source":"def load_detectron2_models(model_directory, folds_to_use):\n\n    \"\"\"\n    Load detectron models from the given directory\n    \n    Parameters\n    ----------\n    model_directory (str): Directory of models, trainer_config and detectron_config\n    folds_to_use (list): List of folds to load\n    \n    Returns\n    -------\n    models (dict): Dictionary of models\n    \"\"\"\n\n    print(f'\\nLoading Detectron2 models from {model_directory}')\n    models = {}\n    model_names = sorted(glob(f'{model_directory}/*.pth'))\n    trainer_config = yaml.load(open(f'{model_directory}/trainer_config.yaml', 'r'), Loader=yaml.FullLoader)\n\n    for fold, weights_path in enumerate(model_names, start=1):\n\n        if fold in folds_to_use:\n\n            detectron_config = get_cfg()\n            detectron_config.merge_from_file(model_zoo.get_config_file(trainer_config['MODEL']['model_zoo_path']))\n            detectron_config.MODEL.WEIGHTS = weights_path\n            detectron_config.merge_from_file(f'{model_directory}/detectron_config.yaml')\n\n            # Disable NMS and score thresholds so it can be done class-wise\n            detectron_config.MODEL.ROI_HEADS.NMS_THRESH_TEST = 1.0\n            detectron_config.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.0\n            detectron_config.TEST.DETECTIONS_PER_IMAGE = 1000\n\n            model = DefaultPredictor(detectron_config)\n            models[fold] = model\n            print(f'Loaded model {weights_path} into memory')\n\n    return models, detectron_config","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:55:49.007431Z","iopub.execute_input":"2021-12-30T11:55:49.008018Z","iopub.status.idle":"2021-12-30T11:55:49.021185Z","shell.execute_reply.started":"2021-12-30T11:55:49.007979Z","shell.execute_reply":"2021-12-30T11:55:49.020194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LOAD_MASK_RCNN2_MODELS = True\nif LOAD_MASK_RCNN2_MODELS:\n    detectron2_mask_rcnn_models2, detectron2_config2 = load_detectron2_models(\n        f'../input/sartorius-cell-instance-segmentation-dataset/detectron2_mask_rcnn2',\n        folds_to_use=[1, 2, 3, 4, 5, 6]\n    )\n    \nLOAD_MASK_RCNN3_MODELS = False\nif LOAD_MASK_RCNN3_MODELS:\n    detectron2_mask_rcnn_models3, detectron2_config3 = load_detectron2_models(\n        f'../input/sartorius-cell-instance-segmentation-dataset/detectron2_mask_rcnn3',\n        folds_to_use=[1, 2, 3, 4, 5, 6]\n    )\n\nLOAD_MASK_RCNN4_MODELS = False\nif LOAD_MASK_RCNN4_MODELS:\n    detectron2_mask_rcnn_models4, detectron2_config4 = load_detectron2_models(\n        f'../input/sartorius-cell-instance-segmentation-dataset/detectron2_mask_rcnn4',\n        folds_to_use=[1, 2, 3, 4, 5, 6]\n    )","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:55:54.516015Z","iopub.execute_input":"2021-12-30T11:55:54.516282Z","iopub.status.idle":"2021-12-30T11:56:36.144281Z","shell.execute_reply.started":"2021-12-30T11:55:54.516252Z","shell.execute_reply":"2021-12-30T11:56:36.143515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Post-processing","metadata":{}},{"cell_type":"code","source":"def prepare_boxes(boxes, scores, labels, masks=None):\n    result_boxes = boxes.copy()\n\n    cond = (result_boxes < 0)\n    cond_sum = cond.astype(np.int32).sum()\n    if cond_sum > 0:\n        print('Warning. Fixed {} boxes coordinates < 0'.format(cond_sum))\n        result_boxes[cond] = 0\n\n    cond = (result_boxes > 1)\n    cond_sum = cond.astype(np.int32).sum()\n    if cond_sum > 0:\n        print('Warning. Fixed {} boxes coordinates > 1. Check that your boxes was normalized at [0, 1]'.format(cond_sum))\n        result_boxes[cond] = 1\n\n    boxes1 = result_boxes.copy()\n    result_boxes[:, 0] = np.min(boxes1[:, [0, 2]], axis=1)\n    result_boxes[:, 2] = np.max(boxes1[:, [0, 2]], axis=1)\n    result_boxes[:, 1] = np.min(boxes1[:, [1, 3]], axis=1)\n    result_boxes[:, 3] = np.max(boxes1[:, [1, 3]], axis=1)\n\n    area = (result_boxes[:, 2] - result_boxes[:, 0]) * (result_boxes[:, 3] - result_boxes[:, 1])\n    cond = (area == 0)\n    cond_sum = cond.astype(np.int32).sum()\n    if cond_sum > 0:\n        print('Warning. Removed {} boxes with zero area!'.format(cond_sum))\n        result_boxes = result_boxes[area > 0]\n        scores = scores[area > 0]\n        labels = labels[area > 0]\n        if masks is not None:\n            masks = masks[area > 0]\n\n    return result_boxes, scores, labels, masks\n\n\ndef cpu_soft_nms_float(dets, sc, Nt, sigma, thresh, method):\n\n    # indexes concatenate boxes with the last column\n    N = dets.shape[0]\n    indexes = np.array([np.arange(N)])\n    dets = np.concatenate((dets, indexes.T), axis=1)\n\n    # the order of boxes coordinate is [y1, x1, y2, x2]\n    y1 = dets[:, 1]\n    x1 = dets[:, 0]\n    y2 = dets[:, 3]\n    x2 = dets[:, 2]\n    scores = sc\n    areas = (x2 - x1) * (y2 - y1)\n\n    for i in range(N):\n        # intermediate parameters for later parameters exchange\n        tBD = dets[i, :].copy()\n        tscore = scores[i].copy()\n        tarea = areas[i].copy()\n        pos = i + 1\n\n        #\n        if i != N - 1:\n            maxscore = np.max(scores[pos:], axis=0)\n            maxpos = np.argmax(scores[pos:], axis=0)\n        else:\n            maxscore = scores[-1]\n            maxpos = 0\n        if tscore < maxscore:\n            dets[i, :] = dets[maxpos + i + 1, :]\n            dets[maxpos + i + 1, :] = tBD\n            tBD = dets[i, :]\n\n            scores[i] = scores[maxpos + i + 1]\n            scores[maxpos + i + 1] = tscore\n            tscore = scores[i]\n\n            areas[i] = areas[maxpos + i + 1]\n            areas[maxpos + i + 1] = tarea\n            tarea = areas[i]\n\n        # IoU calculate\n        xx1 = np.maximum(dets[i, 1], dets[pos:, 1])\n        yy1 = np.maximum(dets[i, 0], dets[pos:, 0])\n        xx2 = np.minimum(dets[i, 3], dets[pos:, 3])\n        yy2 = np.minimum(dets[i, 2], dets[pos:, 2])\n\n        w = np.maximum(0.0, xx2 - xx1)\n        h = np.maximum(0.0, yy2 - yy1)\n        inter = w * h\n        ovr = inter / (areas[i] + areas[pos:] - inter)\n\n        # Three methods: 1.linear 2.gaussian 3.original NMS\n        if method == 1:  # linear\n            weight = np.ones(ovr.shape)\n            weight[ovr > Nt] = weight[ovr > Nt] - ovr[ovr > Nt]\n        elif method == 2:  # gaussian\n            weight = np.exp(-(ovr * ovr) / sigma)\n        else:  # original NMS\n            weight = np.ones(ovr.shape)\n            weight[ovr > Nt] = 0\n\n        scores[pos:] = weight * scores[pos:]\n\n    # select the boxes and keep the corresponding indexes\n    inds = dets[:, 4][scores > thresh]\n    keep = inds.astype(int)\n    return keep\n\n\n@jit(nopython=True)\ndef nms_float_fast(dets, scores, thresh):\n\n    x1 = dets[:, 0]\n    y1 = dets[:, 1]\n    x2 = dets[:, 2]\n    y2 = dets[:, 3]\n\n    areas = (x2 - x1) * (y2 - y1)\n    order = scores.argsort()[::-1]\n\n    keep = []\n    while order.size > 0:\n        i = order[0]\n        keep.append(i)\n        xx1 = np.maximum(x1[i], x1[order[1:]])\n        yy1 = np.maximum(y1[i], y1[order[1:]])\n        xx2 = np.minimum(x2[i], x2[order[1:]])\n        yy2 = np.minimum(y2[i], y2[order[1:]])\n\n        w = np.maximum(0.0, xx2 - xx1)\n        h = np.maximum(0.0, yy2 - yy1)\n        inter = w * h\n        ovr = inter / (areas[i] + areas[order[1:]] - inter)\n        inds = np.where(ovr <= thresh)[0]\n        order = order[inds + 1]\n\n    return keep\n\n\ndef nms_method(boxes, scores, labels, masks=None, method=3, iou_thr=0.5, sigma=0.5, thresh=0.001, weights=None):\n\n    # If weights are specified\n    if weights is not None:\n        if len(boxes) != len(weights):\n            print('Incorrect number of weights: {}. Must be: {}. Skip it'.format(len(weights), len(boxes)))\n        else:\n            weights = np.array(weights)\n            for i in range(len(weights)):\n                scores[i] = (np.array(scores[i]) * weights[i]) / weights.sum()\n\n    # We concatenate everything\n    boxes = np.concatenate(boxes)\n    scores = np.concatenate(scores)\n    labels = np.concatenate(labels)\n    if masks is not None:\n        masks = np.concatenate(masks)\n\n    # Fix coordinates and removed zero area boxes\n    boxes, scores, labels, masks = prepare_boxes(boxes, scores, labels, masks)\n\n    # Run NMS independently for each label\n    unique_labels = np.unique(labels)\n    final_boxes = []\n    final_scores = []\n    final_labels = []\n    if masks is not None:\n        final_masks = []\n\n    for l in unique_labels:\n        condition = (labels == l)\n        boxes_by_label = boxes[condition]\n        scores_by_label = scores[condition]\n        labels_by_label = np.array([l] * len(boxes_by_label))\n        if masks is not None:\n            masks_by_label = masks[condition]\n\n        if method != 3:\n            keep = cpu_soft_nms_float(boxes_by_label.copy(), scores_by_label.copy(), Nt=iou_thr, sigma=sigma, thresh=thresh, method=method)\n        else:\n            # Use faster function\n            keep = nms_float_fast(boxes_by_label, scores_by_label, thresh=iou_thr)\n\n        final_boxes.append(boxes_by_label[keep])\n        final_scores.append(scores_by_label[keep])\n        final_labels.append(labels_by_label[keep])\n        if masks is not None:\n            final_masks.append(masks_by_label[keep])\n    final_boxes = np.concatenate(final_boxes)\n    final_scores = np.concatenate(final_scores)\n    final_labels = np.concatenate(final_labels)\n    if masks is not None:\n        final_masks = np.concatenate(final_masks)\n\n    return final_boxes, final_scores, final_labels, final_masks\n\n\ndef nms(boxes, scores, labels, masks=None, iou_thr=0.5, weights=None):\n\n    return nms_method(boxes, scores, labels, masks, method=3, iou_thr=iou_thr, weights=weights)\n\n\ndef soft_nms(boxes, scores, labels, masks=None, method=2, iou_thr=0.5, sigma=0.5, thresh=0.001, weights=None):\n\n    return nms_method(boxes, scores, labels, masks, method=method, iou_thr=iou_thr, sigma=sigma, thresh=thresh, weights=weights)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-30T11:57:01.343272Z","iopub.execute_input":"2021-12-30T11:57:01.343607Z","iopub.status.idle":"2021-12-30T11:57:01.423506Z","shell.execute_reply.started":"2021-12-30T11:57:01.343573Z","shell.execute_reply":"2021-12-30T11:57:01.422814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fix_overlaps(masks, area_threshold, mask_area_order='descending'):\n\n    \"\"\"\n    Remove overlapping regions of the given masks\n    \n    Parameters\n    ----------\n    masks [numpy.ndarray of shape (n_objects, height, width)]: 2d binary masks\n    area_threshold (int): Threshold for dropping small islands after removing overlapping regions\n    mask_area_order (str): Whether to sort masks by their area in descending or ascending order\n    \n    Returns\n    -------\n    non_overlapping_masks [numpy.ndarray of shape (n_objects, height, width)]: 2d binary masks with no overlapping regions\n    \"\"\"\n    \n    # Sort masks by their areas in descending or ascending order\n    # This will give importance to larger or smaller masks\n    mask_areas = np.sum(masks, axis=(1, 2))\n    mask_areas = mask_areas.astype(np.uint16)\n    if mask_area_order == 'descending':\n        masks = masks[np.argsort(mask_areas)[::-1], :, :]\n    else:\n        masks = masks[np.argsort(mask_areas), :, :]\n\n    non_overlapping_masks = []\n    used_pixels = np.zeros(masks.shape[1:], dtype=np.uint16)\n\n    for mask in masks:\n        mask = mask * (1 - used_pixels)\n        # Filter out objects smaller than area_threshold after removing overlapping regions\n        if np.sum(mask) >= area_threshold:\n            used_pixels += mask\n            non_overlapping_masks.append(mask)\n    \n    non_overlapping_masks = np.stack(non_overlapping_masks).astype(bool)\n    return non_overlapping_masks\n\n\ndef filter_predictions(predictions, box_height_scale, box_width_scale, iou_threshold=None, nms_weights=None, score_threshold=None, verbose=False):\n\n    \"\"\"\n    Filter predictions with NMS and scores\n    \n    Parameters\n    ----------\n    prediction (list): List of one or multiple dictionaries of predicted boxes, labels, scores and masks as numpy arrays\n    box_height_scale (int): Height of the image\n    box_width_scale (int): Width of the image\n    iou_threshold (float): Supress boxes and masks with NMS with this threshold (0 <= iou_threshold <= 1)\n    nms_weights (list): List of weights of predictions (nms_weights must have same length with predictions)\n    score_threshold (float): Remove boxes and masks based on their scores with this threshold (0 <= score_threshold <= 1)\n    verbose (str): Verbosity flag\n    \n    Returns\n    -------\n    prediction (dict): Dictionary of predicted boxes, labels, scores and masks as numpy arrays\n    \"\"\"\n\n    boxes_list = []\n    scores_list = []\n    labels_list = []\n    masks_list = []\n\n    # Storing predictions of multiple models into lists\n    for prediction in predictions:\n        # Scale box coordinates between 0 and 1\n        prediction['boxes'][:, 0] /= box_width_scale\n        prediction['boxes'][:, 1] /= box_height_scale\n        prediction['boxes'][:, 2] /= box_width_scale\n        prediction['boxes'][:, 3] /= box_height_scale\n\n        boxes_list.append(prediction['boxes'].tolist())\n        scores_list.append(prediction['scores'].tolist())\n        labels_list.append(prediction['labels'].tolist())\n        masks_list.append(prediction['masks'])\n\n        if verbose:\n            print(f'{len(prediction[\"scores\"])} objects are predicted with {np.mean(prediction[\"scores\"]):.4f} average score')\n            \n    del predictions\n\n    # Supress overlapping boxes with NMS\n    boxes, scores, labels, masks = nms(\n        boxes=boxes_list,\n        scores=scores_list,\n        labels=labels_list,\n        masks=masks_list,\n        iou_thr=iou_threshold,\n        weights=nms_weights\n    )\n    \n    del boxes_list, scores_list, labels_list, masks_list\n    if verbose:\n        print(f'{len(scores)} objects are kept after applying {iou_threshold} nms iou threshold with {np.mean(scores):.4f} average score')\n\n    # Rescale box coordinates between image height and width\n    boxes[:, 0] *= box_width_scale\n    boxes[:, 1] *= box_height_scale\n    boxes[:, 2] *= box_width_scale\n    boxes[:, 3] *= box_height_scale\n\n    # Filter out boxes based on scores\n    score_condition = scores >= score_threshold\n    boxes = boxes[score_condition]\n    scores = scores[score_condition]\n    masks = masks[score_condition]\n    labels = labels[score_condition]\n\n    if verbose:\n        print(f'{len(scores)} objects are kept after applying {score_threshold} score threshold with {np.mean(scores):.4f} average score')\n\n    return boxes, scores, labels, masks\n\n\ndef get_iou_matrix_from_boxes(bounding_boxes1, bounding_boxes2):\n\n    \"\"\"\n    Calculate IoU matrix between two sets of bounding boxes\n    \n    Parameters\n    ----------\n    bounding_boxes1 [numpy.ndarray of shape (n_objects, 4)]: Bounding boxes\n    bounding_boxes2 [numpy.ndarray of shape (m_objects, 4)]: Bounding boxes\n    \n    Returns\n    -------\n    iou_matrix [numpy.ndarray of shape (n_objects, m_objects)]: IoU matrix between two sets of bounding boxes\n    \"\"\"\n\n    bounding_boxes1_x1, bounding_boxes1_y1, bounding_boxes1_x2, bounding_boxes1_y2 = np.split(bounding_boxes1, 4, axis=1)\n    bounding_boxes2_x1, bounding_boxes2_y1, bounding_boxes2_x2, bounding_boxes2_y2 = np.split(bounding_boxes2, 4, axis=1)\n\n    xa = np.maximum(bounding_boxes1_x1, np.transpose(bounding_boxes2_x1))\n    ya = np.maximum(bounding_boxes1_y1, np.transpose(bounding_boxes2_y1))\n    xb = np.minimum(bounding_boxes1_x2, np.transpose(bounding_boxes2_x2))\n    yb = np.minimum(bounding_boxes1_y2, np.transpose(bounding_boxes2_y2))\n\n    inter_area = np.maximum((xb - xa + 1), 0) * np.maximum((yb - ya + 1), 0)\n    box_a_area = (bounding_boxes1_x2 - bounding_boxes1_x1 + 1) * (bounding_boxes1_y2 - bounding_boxes1_y1 + 1)\n    box_b_area = (bounding_boxes2_x2 - bounding_boxes2_x1 + 1) * (bounding_boxes2_y2 - bounding_boxes2_y1 + 1)\n    iou_matrix = inter_area / (box_a_area + np.transpose(box_b_area) - inter_area)\n\n    return iou_matrix\n\n\ndef get_iou_matrix_from_masks(masks1, masks2):\n    \n    \"\"\"\n    Calculate IOU matrix between two sets of masks\n    \n    Parameters\n    ----------\n    masks1 [numpy.ndarray of shape (n_objects, height, width)]: 2d binary masks\n    masks2 [numpy.ndarray of shape (m_objects, height, width)]: 2d binary masks\n    \n    Returns\n    -------\n    iou_matrix [numpy.ndarray of shape (n_objects, m_objects)]: IoU matrix between two sets of masks\n    \"\"\"\n    \n    if len(list(masks1)) == 0 or len(list(masks2)) == 0:\n        print(f'empty predictions - masks1 len {len(list(masks1))}, masks2 len {len(list(masks2))}')\n        return np.array([[]])\n    \n    enc_masks1 = [mask_util.encode(np.asarray(p, order='F')) for p in (masks1 > 0.5).astype(np.uint8)]\n    enc_masks2 = [mask_util.encode(np.asarray(p, order='F')) for p in (masks2 > 0.5).astype(np.uint8)]\n    iou_matrix = mask_util.iou(enc_masks1, enc_masks2, [0] * len(enc_masks1))\n    \n    return iou_matrix\n\n\n\ndef blend_masks(prediction_boxes, prediction_masks, iou_threshold=0.9, label_threshold=0.5, iou_method='boxes', drop_single_components=True):\n\n    \"\"\"\n    Blend prediction masks of multiple models based on IoU\n    \n    Parameters\n    ----------\n    prediction_boxes [list of shape (n_models)]: Bounding box predictions of multiple models\n    prediction_masks [list of shape (n_models)]: Mask predictions of multiple models\n    iou_threshold (int): IoU threshold for blending masks (0 <= iou_threshold <= 1)\n    label_threshold (int): Label threshold for converting soft predictions to labels (0 <= iou_threshold <= 1)\n    drop_single_components (bool): Whether to discard predictions without connections or not\n    \n    Returns\n    -------\n    blended_masks [numpy.ndarray of shape (n_objects, height, width)]: Blended binary masks\n    \"\"\"\n\n    iou_matrices = {}\n\n    # Create all combinations of IoU matrices from given predictions\n    for i in range(len(prediction_masks)):\n        for j in range(i, len(prediction_masks)):\n            if i == j:\n                continue\n            \n            if iou_method == 'boxes':\n                iou_matrix = get_iou_matrix_from_boxes(prediction_boxes[i], prediction_boxes[j])\n            elif iou_method == 'masks':\n                iou_matrix = get_iou_matrix_from_masks(prediction_masks[i], prediction_masks[j])\n            \n            iou_matrices[f'{i + 1}_{j + 1}'] = iou_matrix\n\n    # Create a graph to store connected bounding boxes\n    bounding_box_graph = nx.Graph()\n\n    # Add all masks from all models as nodes\n    for model_idx, boxes in enumerate(prediction_masks, start=1):\n        nodes = [f'model{model_idx}_box{box_idx}' for box_idx in np.arange(len(boxes))]\n        bounding_box_graph.add_nodes_from(nodes)\n        \n    del prediction_boxes\n\n    # Add edges between nodes with IoU >= iou_threshold\n    for model_combination, iou_matrix in iou_matrices.items():\n        matching_boxes_idx = np.where(iou_matrix >= iou_threshold)\n        model1_idx, model2_idx = model_combination.split('_')\n        edges = [(f'model{model1_idx}_box{box1}', f'model{model2_idx}_box{box2}') for box1, box2 in zip(*matching_boxes_idx)]\n        bounding_box_graph.add_edges_from(edges)\n\n    del iou_matrices\n    blended_masks = []\n\n    for connections in nx.connected_components(bounding_box_graph):\n        if len(connections) == 1:\n            # Skip mask if its bounding isn't connected to any other bounding box\n            if drop_single_components:\n                continue\n            else:\n                # Append mask directly if its bounding box isn't connected to any other bounding box\n                model_idx, box_idx = list(connections)[0].split('_')\n                model_idx = int(model_idx.replace('model', ''))\n                box_idx = int(box_idx.replace('box', ''))\n                blended_masks.append(prediction_masks[model_idx - 1][box_idx])\n        else:\n            # Blend mask with its connections and append\n            blended_mask = np.zeros((520, 704), dtype=np.float32)\n            for connection in connections:\n                model_idx, box_idx = connection.split('_')\n                model_idx = int(model_idx.replace('model', ''))\n                box_idx = int(box_idx.replace('box', ''))\n                # Divide soft predictions with number of connections and accumulate on blended_mask\n                blended_mask += (prediction_masks[model_idx - 1][box_idx] / len(connections))\n            blended_masks.append(blended_mask)\n            \n    del prediction_masks, bounding_box_graph\n    blended_masks = np.stack(blended_masks)\n    # Convert soft predictions to binary labels\n    blended_masks = np.uint8(blended_masks >= label_threshold)\n\n    return blended_masks","metadata":{"execution":{"iopub.status.busy":"2021-12-30T12:01:21.734839Z","iopub.execute_input":"2021-12-30T12:01:21.735128Z","iopub.status.idle":"2021-12-30T12:01:21.772341Z","shell.execute_reply.started":"2021-12-30T12:01:21.735095Z","shell.execute_reply":"2021-12-30T12:01:21.771589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Detectron2 Inference","metadata":{}},{"cell_type":"code","source":"def predict_single_image(image, model):\n\n    \"\"\"\n    Predict given image with given model and move predictions to cpu\n    \n    Parameters\n    ----------\n    image [numpy.ndarray of shape (height, width, channel)]: Image (BGR)\n    model (torch.nn.Module): Detectron2 Model\n    \n    Returns\n    -------\n    prediction (dict): Dictionary of predicted boxes, labels, scores and masks as numpy arrays\n    \"\"\"\n\n    prediction = model(image)\n    prediction = {\n        'boxes': prediction['instances'].pred_boxes.tensor.cpu().numpy(),\n        'labels': prediction['instances'].pred_classes.cpu().numpy(),\n        'scores': prediction['instances'].scores.cpu().numpy(),\n        'masks': prediction['instances'].pred_masks.cpu().numpy()\n    }\n\n    return prediction","metadata":{"execution":{"iopub.status.busy":"2021-12-30T12:01:22.488926Z","iopub.execute_input":"2021-12-30T12:01:22.489597Z","iopub.status.idle":"2021-12-30T12:01:22.495366Z","shell.execute_reply.started":"2021-12-30T12:01:22.489552Z","shell.execute_reply":"2021-12-30T12:01:22.494605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"detectron2_mask_rcnn_post_processing_parameters = {\n    'nms_iou_thresholds': {\n        0: 0.3,\n        1: 0.3,\n        2: 0.3\n    },\n    'score_thresholds': {\n        0: 0.8,\n        1: 0.45,\n        2: 0.45\n    },\n    'area_thresholds': {\n        0: 60,\n        1: 60,\n        2: 150\n    },\n    'blend_iou_thresholds': {\n        0: 0.8,\n        1: 0.8,\n        2: 0.8\n    },\n    'pixel_label_thresholds': {\n        0: 0.5,\n        1: 0.5,\n        2: 0.5\n    }\n}\n\ntest_cell_types = []\n\n\ndef detectron2_inference(test_images, models, post_processing_parameters):\n    \n    df = pd.DataFrame(columns=['id', 'predicted'])\n    \n    for file_name in tqdm(test_images):\n        \n        all_prediction_boxes = []\n        all_prediction_masks = []\n        cell_types = []\n        \n        image = cv2.imread(f'../input/sartorius-cell-instance-segmentation/test/{file_name}')\n        for fold, model in models.items():\n            prediction = predict_single_image(image=image, model=model)\n            # Select cell type as the most predicted label\n            cell_type = mode(prediction['labels'])[0][0]\n            prediction_boxes, prediction_scores, prediction_labels, prediction_masks = filter_predictions(\n                predictions=[prediction],\n                box_height_scale=image.shape[0],\n                box_width_scale=image.shape[1],\n                iou_threshold=post_processing_parameters['nms_iou_thresholds'][cell_type],\n                nms_weights=None,\n                score_threshold=post_processing_parameters['score_thresholds'][cell_type],\n                verbose=False\n            )\n\n            all_prediction_boxes.append(prediction_boxes)\n            all_prediction_masks.append(prediction_masks)\n            cell_types.append(cell_type)\n                \n        cell_type = mode(cell_types)[0][0]\n        test_cell_types.append(cell_type)\n        \n        # Blend prediction masks of multiple models based on IoU \n        blended_prediction_masks = blend_masks(\n            prediction_boxes=all_prediction_boxes,\n            prediction_masks=all_prediction_masks,\n            iou_threshold=post_processing_parameters['blend_iou_thresholds'][cell_type],\n            iou_method='masks',\n            label_threshold=post_processing_parameters['pixel_label_thresholds'][cell_type],\n            drop_single_components=True\n        )\n        \n        for prediction_mask in blended_prediction_masks:\n            rle_encoded_mask = encode_rle_mask(prediction_mask)\n            df = df.append({'id': file_name.split('.')[0], 'predicted': rle_encoded_mask}, ignore_index=True)\n                \n    return df\n","metadata":{"execution":{"iopub.status.busy":"2021-12-30T12:57:53.181625Z","iopub.execute_input":"2021-12-30T12:57:53.181913Z","iopub.status.idle":"2021-12-30T12:57:53.194297Z","shell.execute_reply.started":"2021-12-30T12:57:53.181882Z","shell.execute_reply":"2021-12-30T12:57:53.193587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_images = os.listdir('../input/sartorius-cell-instance-segmentation/test')\n\ndf_mask_rcnn2_predictions = detectron2_inference(\n    test_images=test_images,\n    models=detectron2_mask_rcnn_models2,\n    post_processing_parameters=detectron2_mask_rcnn_post_processing_parameters\n)\nprint(f'Mask R-CNN 2 - {df_mask_rcnn2_predictions.shape[0]} objects are predicted on {df_mask_rcnn2_predictions[\"id\"].nunique()} images')\ndel detectron2_mask_rcnn_models2\n\ndf_cellpose_predictions = pd.read_csv('submission.csv')\nprint(f'Cellpose - {df_cellpose_predictions.shape[0]} objects are predicted on {df_cellpose_predictions[\"id\"].nunique()} images')","metadata":{"execution":{"iopub.status.busy":"2021-12-30T12:57:55.711577Z","iopub.execute_input":"2021-12-30T12:57:55.712003Z","iopub.status.idle":"2021-12-30T12:58:55.357475Z","shell.execute_reply.started":"2021-12-30T12:57:55.711961Z","shell.execute_reply":"2021-12-30T12:58:55.356693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission = pd.DataFrame(columns=['id', 'predicted'])\n\nfor idx, file_name in enumerate(test_images):\n    \n    image = cv2.imread(f'../input/sartorius-cell-instance-segmentation/test/{file_name}')\n    \n    mask_rcnn2_masks = df_mask_rcnn2_predictions.loc[df_mask_rcnn2_predictions['id'] == file_name.split('.')[0], 'predicted']\n    mask_rcnn2_masks = np.stack([decode_rle_mask(rle_mask, shape=(image.shape[:2])) for rle_mask in mask_rcnn2_masks])\n    mask_rcnn2_boxes = np.stack([mask_to_bounding_box(mask) for mask in mask_rcnn2_masks])\n    \n    cellpose_masks = df_cellpose_predictions.loc[df_cellpose_predictions['id'] == file_name.split('.')[0], 'predicted']\n    cellpose_masks = np.stack([decode_rle_mask(rle_mask, shape=image.shape[:2]) for rle_mask in cellpose_masks])\n    cellpose_boxes = np.stack([mask_to_bounding_box(mask) for mask in cellpose_masks])\n    \n    blended_masks = blend_masks(\n        prediction_boxes=[mask_rcnn2_boxes, cellpose_boxes],\n        prediction_masks=[mask_rcnn2_masks, cellpose_masks],\n        iou_threshold=0.8,\n        iou_method='masks',\n        label_threshold=0.5,\n        drop_single_components=False\n    )\n    \n    blended_masks = fix_overlaps(\n        blended_masks,\n        area_threshold=detectron2_mask_rcnn_post_processing_parameters['area_thresholds'][test_cell_types[idx]],\n        mask_area_order='ascending'\n    )\n    \n    for mask in blended_masks:\n        rle_encoded_mask = encode_rle_mask(mask)\n        df_submission = df_submission.append({'id': file_name.split('.')[0], 'predicted': rle_encoded_mask}, ignore_index=True)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:01:29.781577Z","iopub.execute_input":"2021-12-30T13:01:29.781881Z","iopub.status.idle":"2021-12-30T13:01:33.522420Z","shell.execute_reply.started":"2021-12-30T13:01:29.781848Z","shell.execute_reply":"2021-12-30T13:01:33.521698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission.to_csv('submission.csv', index=False)\ndf_submission.head(15)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:01:33.524513Z","iopub.execute_input":"2021-12-30T13:01:33.525374Z","iopub.status.idle":"2021-12-30T13:01:33.547666Z","shell.execute_reply.started":"2021-12-30T13:01:33.525333Z","shell.execute_reply":"2021-12-30T13:01:33.546926Z"},"trusted":true},"execution_count":null,"outputs":[]}]}