{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Sartorius cell instance segmentation.\n\nThis notebook does the following.\n- Install detectron2\n- preprocess training data\n- Fine tune mask r-cnn model\n- Generate prediction on test dataset\n- Write submission file\n\nv2 ","metadata":{}},{"cell_type":"markdown","source":"# Install detectron2 ","metadata":{"execution":{"iopub.status.busy":"2021-10-26T12:24:52.11885Z","iopub.execute_input":"2021-10-26T12:24:52.119255Z","iopub.status.idle":"2021-10-26T12:24:52.12471Z","shell.execute_reply.started":"2021-10-26T12:24:52.119197Z","shell.execute_reply":"2021-10-26T12:24:52.123895Z"}}},{"cell_type":"code","source":"!pip install ../input/detectron-05/whls/pycocotools-2.0.2/dist/pycocotools-2.0.2.tar --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/fvcore-0.1.5.post20211019/fvcore-0.1.5.post20211019 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/antlr4-python3-runtime-4.8/antlr4-python3-runtime-4.8 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/detectron2-0.5/detectron2 --no-index --find-links ../input/detectron-05/whls ","metadata":{"execution":{"iopub.status.busy":"2021-10-27T12:32:09.784031Z","iopub.execute_input":"2021-10-27T12:32:09.784385Z","iopub.status.idle":"2021-10-27T12:35:29.89049Z","shell.execute_reply.started":"2021-10-27T12:32:09.784291Z","shell.execute_reply":"2021-10-27T12:35:29.889717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load training data","metadata":{}},{"cell_type":"code","source":"import torch\nimport pandas as pd\nimport numpy as np\nimport glob\nimport cv2\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\n\n# Some basic setup:\n# Setup detectron2 logger\nimport detectron2\nfrom detectron2.utils.logger import setup_logger\nsetup_logger()\n\n# import some common libraries\nimport numpy as np\nimport os, json, cv2, random\n#from google.colab.patches import cv2_imshow\n\n# import some common detectron2 utilities\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.data import MetadataCatalog, DatasetCatalog\nimport pycocotools\nimport skimage.measure\nfrom fastcore.all import *\n\n\nDATA_PATH = '../input/sartorius-cell-instance-segmentation'\ntrain_info = pd.read_csv(DATA_PATH + '/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-10-27T13:13:50.386403Z","iopub.execute_input":"2021-10-27T13:13:50.386676Z","iopub.status.idle":"2021-10-27T13:13:50.716404Z","shell.execute_reply.started":"2021-10-27T13:13:50.386646Z","shell.execute_reply":"2021-10-27T13:13:50.715658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define training labels\n\n\ncell_type_to_cat = {}\ncat_to_cell_type = {}\ncat_list = []\nfor i, ct in enumerate(train_info['cell_type'].unique()):\n    cell_type_to_cat[ct] = i\n    cat_to_cell_type[i] = ct\n    cat_list.append(ct)\ncat_list_three = cat_list\nprint(cell_type_to_cat)\nprint(cat_list_three)\n\ncat_one_type = {}\nfor t in cell_type_to_cat:\n    cat_one_type[t] = 0\ncat_list_one = [\"cell\"]\nprint(cat_one_type)\nprint(cat_list_one)\n\nONE_TYPE = False  # True: only has \"cell\" label. False: three cell type labels  \nINCLUDE_CELL_TYPE = cat_list_three # whether to include a cell type in training. Could be any combination of ['cort', 'shsy5y', 'astro']\n\nif ONE_TYPE:\n    cat_dict = cat_one_type\n    cat_list = cat_list_one\nelse:\n    cat_dict = cell_type_to_cat\n    cat_list = cat_list_three\n","metadata":{"execution":{"iopub.status.busy":"2021-10-27T12:35:32.484705Z","iopub.execute_input":"2021-10-27T12:35:32.48498Z","iopub.status.idle":"2021-10-27T12:35:32.507309Z","shell.execute_reply.started":"2021-10-27T12:35:32.484946Z","shell.execute_reply":"2021-10-27T12:35:32.50646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mask_to_bbox(mask):\n    #return boudning box of a mask\n    maskx = np.where(mask.sum(1))[0]\n    masky = np.where(mask.sum(0))[0]\n    xmin = maskx[0]\n    xmax = maskx[-1]\n    ymin = masky[0]\n    ymax = masky[-1]\n    return (ymin, xmin, ymax, xmax)\n\n\ndef rle_to_polygon(mask_rle, shape):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height, width) of array to return \n    Returns numpy array (mask)\n    '''\n    # convert rle to bitmask\n    s = mask_rle.split()\n    starts = list(map(lambda x: int(x) - 1, s[0::2]))\n    lengths = list(map(int, s[1::2]))\n    ends = [x + y for x, y in zip(starts, lengths)]\n    img = np.zeros((shape[0] * shape[1]), dtype=np.uint8)    \n    for start, end in zip(starts, ends):\n        img[start : end] = 1\n    img = img.reshape(shape)\n    \n    # bounding box\n    bbox = mask_to_bbox(img)\n    \n    contours = skimage.measure.find_contours(img, 0.5)\n    polygon = []\n    for contour in contours:\n        contour = np.flip(contour, axis=1)\n        segmentation = contour.ravel().tolist()\n        polygon.append(segmentation)    \n    return bbox, polygon, contours\n\n\nfrom detectron2.structures import BoxMode\ndef get_cell_dicts(img_dir, val_split = 0.02):\n    train_info = pd.read_csv(img_dir + '/train.csv')\n\n    train_dataset = []\n    val_dataset = []\n    #too_short = False\n    \n    for filename in tqdm(sorted(glob.glob(img_dir +  '/train/*.png'))):\n        record = {}\n        \n        height, width = cv2.imread(filename).shape[:2]\n        \n        record[\"file_name\"] = filename\n        record[\"image_id\"] = filename.split('/')[-1].strip('.png')\n        record[\"height\"] = height\n        record[\"width\"] = width\n        \n        objs = []\n        idx = train_info[\"id\"] == record[\"image_id\"]\n        annotations = train_info[idx][\"annotation\"].tolist()\n        cell_types = train_info[idx][\"cell_type\"].tolist()\n        \n        for annot, ct in zip(annotations, cell_types):\n            if ct not in INCLUDE_CELL_TYPE:\n                continue\n            #py,px = rle_to_xy(annot, shape=(height, width))\n#             poly = [(x + 0.5, y + 0.5) for x, y in zip(px, py)]\n#             poly = [p for x in poly for p in x]\n            #mask = rle_decode(annot, (width, height, 1))\n            #bitmask=pycocotools.mask.encode(np.asarray(mask, order=\"F\", dtype='uint8'))\n            bbox, polygon,_ = rle_to_polygon(annot, shape = (520, 704))\n            #polygon, bbox = rle_to_polygon_and_bbox(annot, shape = (520, 704))\n            \n            valid_polygons = True\n            for p in polygon:\n                if len(p) < 6:\n                    print(record[\"image_id\"] + ': polygon list too short%s. Annotation skipped.'%str(p))\n                    valid_polygons = False\n            if not valid_polygons:\n                continue\n\n            obj = {\n                #\"bbox\": [np.min(px), np.min(py), np.max(px), np.max(py)],\n                \"bbox\": bbox,\n                \"bbox_mode\": BoxMode.XYXY_ABS,\n                \"segmentation\": polygon,\n                #\"segmentation\": bitmask,\n                \"category_id\": cat_dict[ct],\n            }            \n            objs.append(obj)\n        record[\"annotations\"] = objs\n        if random.random() <= val_split:\n            val_dataset.append(record)\n        else:\n            train_dataset.append(record)\n            \n#         if too_short:\n#             break\n    return train_dataset, val_dataset\n\ntrain_dataset, val_dataset = get_cell_dicts(DATA_PATH)","metadata":{"execution":{"iopub.status.busy":"2021-10-27T12:35:32.509398Z","iopub.execute_input":"2021-10-27T12:35:32.510133Z","iopub.status.idle":"2021-10-27T12:42:39.950641Z","shell.execute_reply.started":"2021-10-27T12:35:32.510095Z","shell.execute_reply":"2021-10-27T12:42:39.949913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DatasetCatalog.register(\"cell_train\", lambda : train_dataset)\nMetadataCatalog.get(\"cell_train\").set(thing_classes=cat_list)\nDatasetCatalog.register(\"cell_val\", lambda : val_dataset)\nMetadataCatalog.get(\"cell_val\").set(thing_classes=cat_list)\n\n\ncell_metadata = MetadataCatalog.get(\"cell_train\")","metadata":{"execution":{"iopub.status.busy":"2021-10-27T12:42:39.952127Z","iopub.execute_input":"2021-10-27T12:42:39.952581Z","iopub.status.idle":"2021-10-27T12:42:39.9583Z","shell.execute_reply.started":"2021-10-27T12:42:39.952545Z","shell.execute_reply":"2021-10-27T12:42:39.957485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Visualize training image and mask","metadata":{}},{"cell_type":"code","source":"#dataset_dicts = get_cell_dicts(DATA_PATH)\nfor d in random.sample(train_dataset, 1):\n    print(d['image_id'])\n    img = cv2.imread(d[\"file_name\"])\n    visualizer = Visualizer(img[:, :, ::-1], metadata=cell_metadata, scale=2)\n    out = visualizer.draw_dataset_dict(d)\n    plt.figure(figsize=(12, 16))\n    plt.imshow(out.get_image()[:, :, ::-1])\n    plt.title('labeled')\n    plt.figure(figsize=(12, 16))\n    plt.imshow(img)\n    plt.title('original')","metadata":{"execution":{"iopub.status.busy":"2021-10-27T12:42:39.95942Z","iopub.execute_input":"2021-10-27T12:42:39.959744Z","iopub.status.idle":"2021-10-27T12:42:41.750585Z","shell.execute_reply.started":"2021-10-27T12:42:39.959708Z","shell.execute_reply":"2021-10-27T12:42:41.74977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"# train mask rcnn\nfrom detectron2.engine import DefaultTrainer\n# !rm ./output/*\ncfg = get_cfg()\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\ncfg.DATASETS.TRAIN = (\"cell_train\",)\ncfg.DATASETS.TEST = (\"cell_val\",)\ncfg.DATALOADER.NUM_WORKERS = 2\ncfg.MODEL.WEIGHTS = '../input/detectron-retinanet/FR-CNN_101.pkl'  # Let training initialize from model zoo\ncfg.SOLVER.IMS_PER_BATCH = 2\ncfg.SOLVER.BASE_LR = 0.001  # pick a good LR\ncfg.SOLVER.MAX_ITER = 12000\ncfg.SOLVER.STEPS = []        # do not decay learning rate\ncfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128   \ncfg.MODEL.ROI_HEADS.NUM_CLASSES = len(cat_list) \ncfg.OUTPUT_DIR = './output'\n# NOTE: this config means the number of classes, but a few popular unofficial tutorials incorrect uses num_classes+1 here.\n#print(cfg)\nos.makedirs(cfg.OUTPUT_DIR, exist_ok=True)\nwith open(cfg.OUTPUT_DIR+\"/config.yaml\", \"w\") as file:\n    file.write(cfg.dump())\ntrainer = DefaultTrainer(cfg) \ntrainer.resume_or_load(resume=False)\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2021-10-27T12:42:41.752068Z","iopub.execute_input":"2021-10-27T12:42:41.752341Z","iopub.status.idle":"2021-10-27T12:59:51.887496Z","shell.execute_reply.started":"2021-10-27T12:42:41.752302Z","shell.execute_reply":"2021-10-27T12:59:51.886749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# validation","metadata":{}},{"cell_type":"code","source":"# setup \ncfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, \"model_final.pth\")  # path to the model we just trained\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.4   # set a custom testing threshold\npredictor = DefaultPredictor(cfg)\n\nval_metadata = MetadataCatalog.get(\"cell_val\")\nval_metadata","metadata":{"execution":{"iopub.status.busy":"2021-10-27T13:12:27.521259Z","iopub.execute_input":"2021-10-27T13:12:27.521527Z","iopub.status.idle":"2021-10-27T13:12:28.312126Z","shell.execute_reply.started":"2021-10-27T13:12:27.521498Z","shell.execute_reply":"2021-10-27T13:12:28.31146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize prediction and truth\nfrom detectron2.utils.visualizer import ColorMode\nfor d in random.sample(val_dataset, 1):    \n    im = cv2.imread(d[\"file_name\"])\n    outputs = predictor(im)  # format is documented at https://detectron2.readthedocs.io/tutorials/models.html#model-output-format\n    v = Visualizer(im[:, :, ::-1],\n                   metadata=cell_metadata, \n                   scale=1, \n#                   instance_mode=ColorMode.IMAGE_BW   # remove the colors of unsegmented pixels. This option is only available for segmentation models\n    )\n    \n    op = outputs[\"instances\"].to(\"cpu\")\n    out = v.draw_instance_predictions(op)\n    plt.figure(figsize=(12,16))\n    plt.imshow(out.get_image()[:, :, ::-1])\n    \n    \n    # plot truth\n    plt.figure(figsize=(12,16))\n    print(train_info['cell_type'][d['image_id']==train_info['id']].unique())\n    visualizer = Visualizer(im[:, :, ::-1], metadata=val_metadata, scale=.5)\n    out = visualizer.draw_dataset_dict(d)\n    plt.figure(figsize=(12, 16))\n    plt.imshow(out.get_image()[:, :, ::-1])    ","metadata":{"execution":{"iopub.status.busy":"2021-10-27T13:12:32.175103Z","iopub.execute_input":"2021-10-27T13:12:32.175353Z","iopub.status.idle":"2021-10-27T13:12:35.152705Z","shell.execute_reply.started":"2021-10-27T13:12:32.175325Z","shell.execute_reply":"2021-10-27T13:12:35.152091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from detectron2.evaluation import COCOEvaluator, inference_on_dataset\nfrom detectron2.data import build_detection_test_loader\nevaluator = COCOEvaluator(\"cell_val\", output_dir=cfg.OUTPUT_DIR)\nval_loader = build_detection_test_loader(cfg, \"cell_val\")\nprint(inference_on_dataset(predictor.model, val_loader, evaluator))\n# another equivalent way to evaluate the model is to use `trainer.test`","metadata":{"execution":{"iopub.status.busy":"2021-10-27T12:59:55.320638Z","iopub.execute_input":"2021-10-27T12:59:55.321061Z","iopub.status.idle":"2021-10-27T12:59:57.045122Z","shell.execute_reply.started":"2021-10-27T12:59:55.32102Z","shell.execute_reply":"2021-10-27T12:59:57.044319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Process and predict on Test dataset","metadata":{}},{"cell_type":"code","source":"def get_cell_test_dicts(img_dir):\n    test_dataset = []\n    for filename in tqdm(sorted(glob.glob(img_dir +  '/test/*.png'))):\n        record = {}\n        \n        height, width = cv2.imread(filename).shape[:2]\n        \n        record[\"file_name\"] = filename\n        record[\"image_id\"] = filename.split('/')[-1].strip('.png')\n        record[\"height\"] = height\n        record[\"width\"] = width\n        test_dataset.append(record)\n    return test_dataset\n\n\ntest_dataset = get_cell_test_dicts(DATA_PATH)","metadata":{"execution":{"iopub.status.busy":"2021-10-27T12:59:57.046762Z","iopub.execute_input":"2021-10-27T12:59:57.04746Z","iopub.status.idle":"2021-10-27T12:59:57.118424Z","shell.execute_reply.started":"2021-10-27T12:59:57.04742Z","shell.execute_reply":"2021-10-27T12:59:57.117308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_DIR = cfg.OUTPUT_DIR\n\nprint(len(test_dataset))\ntry:\n    DatasetCatalog.remove('cell_test')\n    MetadataCatalog.remove('cell_test')\n    \nexcept:\n    pass\n\n\nDatasetCatalog.register(\"cell_test\", lambda : train_dataset)\nMetadataCatalog.get(\"cell_test\").set(thing_classes=cat_list)\ntest_metadata = MetadataCatalog.get(\"cell_test\")\n\ncfg = get_cfg()\ncfg.merge_from_file(MODEL_DIR + \"/config.yaml\")\ncfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, \"model_final.pth\")  # path to the model we just trained\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.4   # set a custom testing threshold\npredictor = DefaultPredictor(cfg)","metadata":{"execution":{"iopub.status.busy":"2021-10-27T12:59:57.120166Z","iopub.execute_input":"2021-10-27T12:59:57.121091Z","iopub.status.idle":"2021-10-27T12:59:58.122172Z","shell.execute_reply.started":"2021-10-27T12:59:57.121034Z","shell.execute_reply":"2021-10-27T12:59:58.121407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize prediction results\nfrom detectron2.utils.visualizer import ColorMode\nfor d in random.sample(test_dataset, 1):    \n    im = cv2.imread(d[\"file_name\"])\n    outputs = predictor(im)  # format is documented at https://detectron2.readthedocs.io/tutorials/models.html#model-output-format\n    v = Visualizer(im[:, :, ::-1],\n                   metadata=test_metadata, \n                   scale=1, \n#                   instance_mode=ColorMode.IMAGE_BW   # remove the colors of unsegmented pixels. This option is only available for segmentation models\n    )\n    \n    op = outputs[\"instances\"].to(\"cpu\")\n    out = v.draw_instance_predictions(op)\n    plt.figure(figsize=(12,16))\n    plt.imshow(out.get_image()[:, :, ::-1])","metadata":{"execution":{"iopub.status.busy":"2021-10-27T12:59:58.123776Z","iopub.execute_input":"2021-10-27T12:59:58.12402Z","iopub.status.idle":"2021-10-27T12:59:59.282082Z","shell.execute_reply.started":"2021-10-27T12:59:58.123987Z","shell.execute_reply":"2021-10-27T12:59:59.28145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Write submission file","metadata":{}},{"cell_type":"code","source":"ids, masks=[],[]\ndataDir=Path('../input/sartorius-cell-instance-segmentation')\ntest_names = (dataDir/'test').ls()\n# From https://www.kaggle.com/stainsby/fast-tested-rle\ndef rle_decode(mask_rle, shape=(520, 704)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape)  # Needed to align to RLE direction\n\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef get_masks(fn, predictor):\n    im = cv2.imread(str(fn))\n    outputs = predictor(im)\n    pred_masks = outputs['instances'].pred_masks.cpu().numpy()\n    res = []\n    used = np.zeros(im.shape[:2], dtype=int) \n    for mask in pred_masks:\n        mask = mask * (1-used)\n        used += mask\n        res.append(rle_encode(mask))\n    return res\n\n\nfor fn in test_names:\n    encoded_masks = get_masks(fn, predictor)\n    for enc in encoded_masks:\n        ids.append(fn.stem)\n        masks.append(enc)\n\npd.DataFrame({'id':ids, 'predicted':masks}).to_csv('submission.csv', index=False)\npd.read_csv('submission.csv').head()","metadata":{"execution":{"iopub.status.busy":"2021-10-27T13:14:12.436052Z","iopub.execute_input":"2021-10-27T13:14:12.436382Z","iopub.status.idle":"2021-10-27T13:14:13.256489Z","shell.execute_reply.started":"2021-10-27T13:14:12.436327Z","shell.execute_reply":"2021-10-27T13:14:13.255842Z"},"trusted":true},"execution_count":null,"outputs":[]}]}