{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### Version history\n* V1 - Inference with basic model trained 1000 iterations\n* V2 - Same training procedure but for more iterations. Validation score .267, LB score .286\n* V3 - Same model weights as previously, but score thresholds set for each class individually. LB score .293 *\n* V4 - Another small inference time improvement - throw away predictions with area smaller than a threshold (set per cell type). LB score .294\n\n\\* *Initally that version had score .287 but that was due to a bug in the code*","metadata":{}},{"cell_type":"markdown","source":"## Inference and submission\nAfter [part one](https://www.kaggle.com/slawekbiel/positive-score-with-detectron-1-3-input-data/) and [part two](https://www.kaggle.com/slawekbiel/positive-score-with-detectron-2-3-training) we have a trained model. I'm attaching it to this notebook through a dataset. Now all that's left is to run all the test files through it.\n\nThere are two minor details we need to handle:\n- The submission notebooks don't have access to the internet, in order to install detectron2 I needed to download dependecies with `pip download`, put them into a dataset and attach it to the notebook: https://www.kaggle.com/slawekbiel/detectron-05\n- The masks we submit can't overlap, see [the discussion](https://www.kaggle.com/c/sartorius-cell-instance-segmentation/discussion/279790#1550666). So I'm manually clipping the output returned from the model) I'm processing the masks ordereded by score, so in the case of conflict the more confident one remaines whole and the other one gets clipped.","metadata":{}},{"cell_type":"code","source":"!pip install ../input/detectron-05/whls/pycocotools-2.0.2/dist/pycocotools-2.0.2.tar --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/fvcore-0.1.5.post20211019/fvcore-0.1.5.post20211019 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/antlr4-python3-runtime-4.8/antlr4-python3-runtime-4.8 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/detectron2-0.5/detectron2 --no-index --find-links ../input/detectron-05/whls ","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-01T11:44:53.028053Z","iopub.execute_input":"2021-11-01T11:44:53.028863Z","iopub.status.idle":"2021-11-01T11:48:15.830354Z","shell.execute_reply.started":"2021-11-01T11:44:53.028754Z","shell.execute_reply":"2021-11-01T11:48:15.829489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import detectron2\nimport torch\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom PIL import Image\nimport cv2\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom fastcore.all import *","metadata":{"execution":{"iopub.status.busy":"2021-11-01T11:48:15.835645Z","iopub.execute_input":"2021-11-01T11:48:15.837643Z","iopub.status.idle":"2021-11-01T11:48:17.112308Z","shell.execute_reply.started":"2021-11-01T11:48:15.837604Z","shell.execute_reply":"2021-11-01T11:48:17.11157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/sartorius-cell-instance-segmentation/train.csv')\ndf_instances = df_train.groupby(['id']).agg({'annotation': 'count', 'cell_type': 'first'})\ndf_instances = df_instances.groupby(\"cell_type\")[['annotation']]\\\n                               .describe(percentiles=[0.1, 0.25, 0.75, 0.8, 0.85, 0.9, 0.95, 0.99]).astype(int)\\\n                               .T.droplevel(level=0).T.drop(['count', '50%', 'std'], axis=1)\ndf_instances\ndf_train['n_pixels'] = df_train.annotation.apply(lambda x: np.sum([int(e) for e in x.split()[1:][::2]]))\ndf_pixels = df_train.groupby(\"cell_type\")[['n_pixels']].describe(percentiles=[0.02, 0.05, 0.1, 0.9, 0.95, 0.98])\\\n                    .astype(int).T.droplevel(level=0).T.drop(['count', '50%', 'std'], axis=1)\ndf_pixels","metadata":{"execution":{"iopub.status.busy":"2021-11-01T11:48:17.113618Z","iopub.execute_input":"2021-11-01T11:48:17.113908Z","iopub.status.idle":"2021-11-01T11:48:19.270889Z","shell.execute_reply.started":"2021-11-01T11:48:17.113871Z","shell.execute_reply":"2021-11-01T11:48:19.270104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataDir=Path('../input/sartorius-cell-instance-segmentation')","metadata":{"execution":{"iopub.status.busy":"2021-11-01T11:48:19.272344Z","iopub.execute_input":"2021-11-01T11:48:19.272601Z","iopub.status.idle":"2021-11-01T11:48:19.276484Z","shell.execute_reply.started":"2021-11-01T11:48:19.272566Z","shell.execute_reply":"2021-11-01T11:48:19.275682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchvision.models import resnet34\nfrom torch import nn\nDEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\nm = resnet34(False)\nm.fc = nn.Linear(512, 3)\n\nclassifier = torch.load('../input/sartorius-resnet-34-classifier-finetuned/resnet34-finetuned.bin', map_location='cpu')\nclassifier.to(DEVICE)\nclassifier.eval();","metadata":{"execution":{"iopub.status.busy":"2021-11-01T11:48:19.279416Z","iopub.execute_input":"2021-11-01T11:48:19.279758Z","iopub.status.idle":"2021-11-01T11:48:24.343724Z","shell.execute_reply.started":"2021-11-01T11:48:19.27972Z","shell.execute_reply":"2021-11-01T11:48:24.342985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# From https://www.kaggle.com/stainsby/fast-tested-rle\ndef rle_decode(mask_rle, shape=(520, 704)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape)  # Needed to align to RLE direction\n\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef get_masks(fn, predictor):\n    im = cv2.imread(str(fn))\n    \n    cell_type = get_image_cell_type(classifier, str(fn))\n    \n    pred = predictor(im)\n    pred_class = torch.mode(pred['instances'].pred_classes)[0]\n    take = pred['instances'].scores >= THRESHOLDS[pred_class]\n    pred_masks = pred['instances'].pred_masks[take]\n    pred_masks = pred_masks.cpu().numpy()\n    res = []\n    used = np.zeros(im.shape[:2], dtype=int) \n    previous_masks = []\n    for mask in pred_masks:\n        mask = mask * (1-used)\n        mask = refine_mask(mask, df_pixels, cell_type)\n        mask = remove_overlapping_pixels(mask, previous_masks)\n        previous_masks.append(mask)\n        if mask.sum() >= MIN_PIXELS[pred_class]: # skip predictions with small area\n            used += mask\n            res.append(rle_encode(mask))\n    return res","metadata":{"execution":{"iopub.status.busy":"2021-11-01T11:53:43.913766Z","iopub.execute_input":"2021-11-01T11:53:43.914054Z","iopub.status.idle":"2021-11-01T11:53:43.926889Z","shell.execute_reply.started":"2021-11-01T11:53:43.914015Z","shell.execute_reply":"2021-11-01T11:53:43.92574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_PATH = '../input/sartorius-cell-instance-segmentation/test'\ndef get_image_for_classifier(image_id):\n    image_path = image_id\n    print(image_path)\n    transforms = A.Compose([A.Resize(224, 224), \n                       A.Normalize(mean=RESNET_MEAN, std=RESNET_STD, p=1), \n                       ToTensorV2()])\n    image = transforms(image=cv2.imread(image_path))['image']\n    return image.unsqueeze(0).to(DEVICE)\n\n# Assess the image_id cell_type with the classifier\ndef get_image_cell_type(classifier, image_id):\n    img = get_image_for_classifier(image_id)\n    with torch.no_grad():\n        logits = classifier(img)[0]\n        cell_type_idx = torch.argmax(logits).item()\n    print(cell_type_idx)\n    return CELL_TYPES[cell_type_idx]\n\n\ndef refine_mask(mask, df_pixels, cell_type):\n    \n    \n    # Minimum number of pixels:\n    # The percentile 0.02 of the cell_type in the train set\n    min_pixels = df_pixels.loc[cell_type, '2%']\n    # Max number of pixels\n    # The percentile 0.95 of the cell_type in the train set\n    max_pixels = df_pixels.loc[cell_type, '98%']\n    \n    binary_mask = mask > MASK_THRESHOLD\n    \n    # If the mask is too small, make the condition less strict\n    # increasing its size until it reaches a minimum number of pixels\n    if binary_mask.sum() < min_pixels:\n        for t in range(25):\n            binary_mask = mask > (MASK_THRESHOLD - t * 0.02)\n            if binary_mask.sum() > min_pixels:\n                break\n    \n    # If the mask is too large, make the condition more strict\n    # reducing its size until it has less than certain amount of pixels\n    if binary_mask.sum() > max_pixels:\n        for t in range(25):\n            binary_mask = mask > (MASK_THRESHOLD + t * 0.02)\n            if binary_mask.sum() < max_pixels:\n                break\n\n    return binary_mask\n\n\ndef remove_overlapping_pixels(mask, other_masks):\n    for other_mask in other_masks:\n        if np.sum(np.logical_and(mask, other_mask)) > 0:\n            mask[np.logical_and(mask, other_mask)] = 0\n    return mask\n","metadata":{"execution":{"iopub.status.busy":"2021-11-01T11:54:34.149616Z","iopub.execute_input":"2021-11-01T11:54:34.149913Z","iopub.status.idle":"2021-11-01T11:54:34.163808Z","shell.execute_reply.started":"2021-11-01T11:54:34.149875Z","shell.execute_reply":"2021-11-01T11:54:34.163106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids, masks=[],[]\ntest_names = (dataDir/'test').ls()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T11:52:52.828228Z","iopub.execute_input":"2021-11-01T11:52:52.828517Z","iopub.status.idle":"2021-11-01T11:52:52.835601Z","shell.execute_reply.started":"2021-11-01T11:52:52.828483Z","shell.execute_reply":"2021-11-01T11:52:52.83478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Initiate a Predictor from our trained model","metadata":{}},{"cell_type":"code","source":"cfg = get_cfg()\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\ncfg.INPUT.MASK_FORMAT='bitmask'\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = 3 \ncfg.MODEL.WEIGHTS = '../input/positive-score-with-detectron-2-3-training/output/model_final.pth'#os.path.join('../input/sartorius-models', \"model_2696.pth\")  \ncfg.TEST.DETECTIONS_PER_IMAGE = 1000\npredictor = DefaultPredictor(cfg)\nTHRESHOLDS = [.15, .35, .55]\nMIN_PIXELS = [75, 150, 75]","metadata":{"execution":{"iopub.status.busy":"2021-11-01T11:52:53.31669Z","iopub.execute_input":"2021-11-01T11:52:53.31694Z","iopub.status.idle":"2021-11-01T11:52:54.170864Z","shell.execute_reply.started":"2021-11-01T11:52:53.316912Z","shell.execute_reply":"2021-11-01T11:52:54.170066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Look at the outputs on a sample test file to sanity check\nI'm encoding here in the competition format and decoding back to bit mask just to make sure everything is fine","metadata":{}},{"cell_type":"code","source":"import cv2\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nRESNET_MEAN = (0.485, 0.456, 0.406)\nRESNET_STD = (0.229, 0.224, 0.225)\nCELL_TYPES  = {0: 'shsy5y', 1: 'astro', 2: 'cort'}\n\nMIN_SCORE = 0.59\n\nMASK_THRESHOLD = 0.5\n\n# The maximum possible amount of predictions\n# 539 is the 90% percentile of the cell_type with more instances per image\nBOX_DETECTIONS_PER_IMG = 559\nencoded_masks = get_masks(test_names[0], predictor)\n\n_, axs = plt.subplots(1,2, figsize=(40,15))\naxs[1].imshow(cv2.imread(str(test_names[0])))\nfor enc in encoded_masks:\n    dec = rle_decode(enc)\n    axs[0].imshow(np.ma.masked_where(dec==0, dec))","metadata":{"execution":{"iopub.status.busy":"2021-11-01T11:54:37.20415Z","iopub.execute_input":"2021-11-01T11:54:37.204721Z","iopub.status.idle":"2021-11-01T11:55:14.827611Z","shell.execute_reply.started":"2021-11-01T11:54:37.204679Z","shell.execute_reply":"2021-11-01T11:55:14.825307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Looks good, so lets generate masks for all the files and create a submission","metadata":{}},{"cell_type":"code","source":"for fn in test_names:\n    encoded_masks = get_masks(fn, predictor)\n    for enc in encoded_masks:\n        ids.append(fn.stem)\n        masks.append(enc)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T11:55:40.521201Z","iopub.execute_input":"2021-11-01T11:55:40.52188Z","iopub.status.idle":"2021-11-01T11:55:50.061706Z","shell.execute_reply.started":"2021-11-01T11:55:40.521838Z","shell.execute_reply":"2021-11-01T11:55:50.060809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame({'id':ids, 'predicted':masks}).to_csv('submission.csv', index=False)\npd.read_csv('submission.csv').head()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T11:48:27.528211Z","iopub.status.idle":"2021-11-01T11:48:27.528828Z","shell.execute_reply.started":"2021-11-01T11:48:27.528578Z","shell.execute_reply":"2021-11-01T11:48:27.528605Z"},"trusted":true},"execution_count":null,"outputs":[]}]}