{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🦠 Sartorius - Torch Mask R-CNN\n### A self-contained, Torch Mask R-CNN implementation\n\nAdapted from https://www.kaggle.com/julian3833/sartorius-starter-torch-mask-r-cnn-lb-0-202\n\nMain differences to Julian's notebook: \n - use 3 classes for model training\n - use different thresholds for each class\n - use IOUmAP score to select best model\n\n### Changelog\n\n\n| Version | Comments | Validation | LB |\n| --- | --- | --- | --- |\n|51| use CV2 for image processing, set random state in train_test_split | 0.275 | 0.278 |\n|48| fix combine_masks mistake | 0.267 | 0.291 |\n|46| revert cutoffs to V43 | 0.247 | 0.288 |\n|45| update cutoffs | 0.242 | 0.281 |\n|43| update cutoffs | 0.249 | 0.29 |\n|42| BOX_DETECTIONS_PER_IMG = 540 (from Julians notebook) | 0.245 | 0.281 |\n|40| BOX_DETECTIONS_PER_IMG = 450 | 0.245 | 0.28 |\n|39| use different thresholds for each class | 0.242 | 0.279|\n|37| use cell_type as class labels, use best validation epoch using IOU score | 0.241 | 0.274 |\n|28| use cell_type as class labels, use best validation epoch | | 0.265 |\n|26| same as V 16, select correct best model (best_epoch+1) | | 0.274 |\n|16| with `MIN_SCORE=0.5`, use best validation epoch (19) | | 0.263 |\n|11| 30 epochs, use best validation (17) | | 0.203 |\n|5| 10 epochs, Adam optimizer | | 0.135 | \n|1| 8 epochs. With Scheduler. | | 0.197 | \n\n[Julian's](https://www.kaggle.com/julian3833/sartorius-starter-torch-mask-r-cnn-lb-0-202) log:\n\n|| Version | Comments | LB |\n|---|  --- | --- | --- |\n||30| Version 18 with `MIN_SCORE=0.5`. Remove validation. | `0.273` |\n||28| V27 but pick best epoch using mask-only validation loss. 18 epochs. | `0.205` |\n||27| V18 + 7.5% validation (`PCT_IMAGES_VALIDATION`) w/best epoch for pred. Added `BOX_DETECTIONS_PER_IMG` and `MIN_SCORE` but not used yet. | `0.178` |\n||24| 8 epochs. With Scheduler. | `0.195` |\n||23| 8 epochs. Mask loss only. | `0.036` |\n||22| 8 epochs. Normalize. (7 epochs = `0.189`) | `0.202`|\n||19| 3 epochs size 25%. 3 epochs size 50%. 6 epochs full sized| `0.178` |\n||18| 8 epochs. Full sized. Tidied-up code.|  `0.202` |\n||15| 12 -> 15 epochs. Setup classification head with classes. Bugfix in `analyze_train_sample`|  `0.172` |\n|| *14* | *12 epochs. Full sized* |`0.173` |\n|| 8 | 12 epochs. Resize to (256, 256) |`0.057` |\n\n","metadata":{"id":"AJ_abxrh0zRR","papermill":{"duration":0.042875,"end_time":"2021-11-16T19:15:44.870272","exception":false,"start_time":"2021-11-16T19:15:44.827397","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Imports","metadata":{"id":"1NZ-_x8E0zRW","papermill":{"duration":0.03399,"end_time":"2021-11-16T19:15:44.943314","exception":false,"start_time":"2021-11-16T19:15:44.909324","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# The notebooks is self-contained\n# It has very few imports\n# No external dependencies (only the model weights)\n# No train - inference notebooks\n# We only rely on Pytorch\nimport os\nimport random\nimport time\nimport collections\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.notebook import tqdm\nfrom albumentations.pytorch import ToTensorV2\nimport albumentations as A\n\nimport torch\nimport torchvision\nfrom torchvision.transforms import ToPILImage\nfrom torchvision.transforms import functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection.mask_rcnn import MaskRCNNPredictor","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","id":"oQYSI0Y00zRX","papermill":{"duration":2.65723,"end_time":"2021-11-16T19:15:47.624915","exception":false,"start_time":"2021-11-16T19:15:44.967685","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:11.857333Z","iopub.execute_input":"2021-11-27T14:19:11.857601Z","iopub.status.idle":"2021-11-27T14:19:11.863649Z","shell.execute_reply.started":"2021-11-27T14:19:11.857570Z","shell.execute_reply":"2021-11-27T14:19:11.862973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fix randomness\n\ndef fix_all_seeds(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n    \nfix_all_seeds(2021)","metadata":{"id":"Y7fwE02H0zRY","papermill":{"duration":0.07734,"end_time":"2021-11-16T19:15:47.727488","exception":false,"start_time":"2021-11-16T19:15:47.650148","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:11.869356Z","iopub.execute_input":"2021-11-27T14:19:11.869984Z","iopub.status.idle":"2021-11-27T14:19:11.877956Z","shell.execute_reply.started":"2021-11-27T14:19:11.869946Z","shell.execute_reply":"2021-11-27T14:19:11.877240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configuration","metadata":{"id":"NqZ4eNVK0zRZ","papermill":{"duration":0.02465,"end_time":"2021-11-16T19:15:47.776995","exception":false,"start_time":"2021-11-16T19:15:47.752345","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Reduced the train dataset to 5000 rows\nTEST = False\n\nif os.path.exists(\"../input/sartorius-cell-instance-segmentation\"):\n    # running on kaggle\n    data_directory = '../input/sartorius-cell-instance-segmentation'\n    DEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n    BATCH_SIZE = 2\n    NUM_EPOCHS = 30\n\nelif 'google.colab' in str(get_ipython()):\n    # running on CoLab\n    from google.colab import drive\n    drive.mount('/content/drive')\n    data_directory = '/content/drive/MyDrive/input'\n    DEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n    BATCH_SIZE = 1\n    NUM_EPOCHS = 5\n    \nelse:\n    data_directory = 'input'\n    DEVICE = torch.device('cpu')\n    BATCH_SIZE = 2\n    NUM_EPOCHS = 1\n    TEST = True\n\nTRAIN_CSV = f\"{data_directory}/train.csv\"\nTRAIN_PATH = f\"{data_directory}/train\"\nTEST_PATH = f\"{data_directory}/test\"\n\nWIDTH = 704\nHEIGHT = 520\n\nresize_factor = False # 0.5\n\n# Normalize to resnet mean and std if True.\nNORMALIZE = False\nRESNET_MEAN = (0.485, 0.456, 0.406)\nRESNET_STD = (0.229, 0.224, 0.225)\n\n# No changes tried with the optimizer yet.\nMOMENTUM = 0.9\nLEARNING_RATE = 0.001\nWEIGHT_DECAY = 0.0005\n\n# Changes the confidence required for a pixel to be kept for a mask. \n# Only used 0.5 till now.\n# MASK_THRESHOLD = 0.5\n# MIN_SCORE = 0.5\n# cell type specific thresholds\ncell_type_dict = {\"astro\": 1, \"cort\": 2, \"shsy5y\": 3}\nmask_threshold_dict = {1: 0.55, 2: 0.75, 3:  0.6}\nmin_score_dict = {1: 0.55, 2: 0.75, 3: 0.5}\n\n# Use a StepLR scheduler if True. \nUSE_SCHEDULER = False\n\nPCT_IMAGES_VALIDATION = 0.075\n\nBOX_DETECTIONS_PER_IMG = 540","metadata":{"id":"VSPe6quz0zRZ","lines_to_next_cell":1,"outputId":"e2cca0e2-0ada-471b-e688-33ee16049407","papermill":{"duration":0.038948,"end_time":"2021-11-16T19:15:47.84119","exception":false,"start_time":"2021-11-16T19:15:47.802242","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:11.882932Z","iopub.execute_input":"2021-11-27T14:19:11.883131Z","iopub.status.idle":"2021-11-27T14:19:11.895310Z","shell.execute_reply.started":"2021-11-27T14:19:11.883109Z","shell.execute_reply":"2021-11-27T14:19:11.894641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utilities","metadata":{"id":"iIpvad7y0zRb","papermill":{"duration":0.024895,"end_time":"2021-11-16T19:15:47.89054","exception":false,"start_time":"2021-11-16T19:15:47.865645","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ref: https://www.kaggle.com/inversion/run-length-decoding-quick-start\ndef rle_decode(mask_rle, shape, color=1):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height, width, channels) of array to return\n    color: color for the mask\n    Returns numpy array (mask)\n\n    '''\n    s = mask_rle.split()\n\n    starts = list(map(lambda x: int(x) - 1, s[0::2]))\n    lengths = list(map(int, s[1::2]))\n    ends = [x + y for x, y in zip(starts, lengths)]\n    if len(shape)==3:\n        img = np.zeros((shape[0] * shape[1], shape[2]), dtype=np.float32)\n    else:\n        img = np.zeros(shape[0] * shape[1], dtype=np.float32)\n    for start, end in zip(starts, ends):\n        img[start : end] = color\n\n    return img.reshape(shape)\n\n\ndef rle_encoding(x):\n    dots = np.where(x.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return ' '.join(map(str, run_lengths))\n\n\ndef remove_overlapping_pixels(mask, other_masks):\n    for other_mask in other_masks:\n        if np.sum(np.logical_and(mask, other_mask)) > 0:\n            mask[np.logical_and(mask, other_mask)] = 0\n    return mask\n\ndef combine_masks(masks, mask_threshold):\n    \"\"\"\n    combine masks into one image\n    \"\"\"\n    maskimg = np.zeros((HEIGHT, WIDTH))\n    # print(len(masks.shape), masks.shape)\n    for m, mask in enumerate(masks,1):\n        maskimg[mask>mask_threshold] = m\n    return maskimg\n\n\ndef get_filtered_masks(pred):\n    \"\"\"\n    filter masks using MIN_SCORE for mask and MAX_THRESHOLD for pixels\n    \"\"\"\n    use_masks = []   \n    for i, mask in enumerate(pred[\"masks\"]):\n\n        # Filter-out low-scoring results. Not tried yet.\n        scr = pred[\"scores\"][i].cpu().item()\n        label = pred[\"labels\"][i].cpu().item()\n        if scr > min_score_dict[label]:\n            mask = mask.cpu().numpy().squeeze()\n            # Keep only highly likely pixels\n            binary_mask = mask > mask_threshold_dict[label]\n            binary_mask = remove_overlapping_pixels(binary_mask, use_masks)\n            use_masks.append(binary_mask)\n\n    return use_masks\n","metadata":{"papermill":{"duration":0.045044,"end_time":"2021-11-16T19:15:47.96066","exception":false,"start_time":"2021-11-16T19:15:47.915616","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:11.901032Z","iopub.execute_input":"2021-11-27T14:19:11.901295Z","iopub.status.idle":"2021-11-27T14:19:11.917656Z","shell.execute_reply.started":"2021-11-27T14:19:11.901271Z","shell.execute_reply":"2021-11-27T14:19:11.916958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Metric: mean of the precision values at each IoU threshold\n\nRef: https://www.kaggle.com/theoviel/competition-metric-map-iou","metadata":{"papermill":{"duration":0.024425,"end_time":"2021-11-16T19:15:48.009197","exception":false,"start_time":"2021-11-16T19:15:47.984772","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def compute_iou(labels, y_pred, verbose=0):\n    \"\"\"\n    Computes the IoU for instance labels and predictions.\n\n    Args:\n        labels (np array): Labels.\n        y_pred (np array): predictions\n\n    Returns:\n        np array: IoU matrix, of size true_objects x pred_objects.\n    \"\"\"\n\n    true_objects = len(np.unique(labels))\n    pred_objects = len(np.unique(y_pred))\n\n    if verbose:\n        print(\"Number of true objects: {}\".format(true_objects))\n        print(\"Number of predicted objects: {}\".format(pred_objects))\n\n    # Compute intersection between all objects\n    intersection = np.histogram2d(\n        labels.flatten(), y_pred.flatten(), bins=(true_objects, pred_objects)\n    )[0]\n\n    # Compute areas (needed for finding the union between all objects)\n    area_true = np.histogram(labels, bins=true_objects)[0]\n    area_pred = np.histogram(y_pred, bins=pred_objects)[0]\n    area_true = np.expand_dims(area_true, -1)\n    area_pred = np.expand_dims(area_pred, 0)\n\n    # Compute union\n    union = area_true + area_pred - intersection\n    intersection = intersection[1:, 1:] # exclude background\n    union = union[1:, 1:]\n    union[union == 0] = 1e-9\n    iou = intersection / union\n    \n    return iou  \n\ndef precision_at(threshold, iou):\n    \"\"\"\n    Computes the precision at a given threshold.\n\n    Args:\n        threshold (float): Threshold.\n        iou (np array): IoU matrix.\n\n    Returns:\n        int: Number of true positives,\n        int: Number of false positives,\n        int: Number of false negatives.\n    \"\"\"\n    matches = iou > threshold\n    true_positives = np.sum(matches, axis=1) == 1  # Correct objects\n    false_positives = np.sum(matches, axis=0) == 0  # Missed objects\n    false_negatives = np.sum(matches, axis=1) == 0  # Extra objects\n    tp, fp, fn = (\n        np.sum(true_positives),\n        np.sum(false_positives),\n        np.sum(false_negatives),\n    )\n    return tp, fp, fn\n\ndef iou_map(truths, preds, verbose=0):\n    \"\"\"\n    Computes the metric for the competition.\n    Masks contain the segmented pixels where each object has one value associated,\n    and 0 is the background.\n\n    Args:\n        truths (list of masks): Ground truths.\n        preds (list of masks): Predictions.\n        verbose (int, optional): Whether to print infos. Defaults to 0.\n\n    Returns:\n        float: mAP.\n    \"\"\"\n    ious = [compute_iou(truth, pred, verbose) for truth, pred in zip(truths, preds)]\n\n    if verbose:\n        print(\"Thresh\\tTP\\tFP\\tFN\\tPrec.\")\n\n    prec = []\n    for t in np.arange(0.5, 1.0, 0.05):\n        tps, fps, fns = 0, 0, 0\n        for iou in ious:\n            tp, fp, fn = precision_at(t, iou)\n            tps += tp\n            fps += fp\n            fns += fn\n\n        p = tps / (tps + fps + fns)\n        prec.append(p)\n\n        if verbose:\n            print(\"{:1.3f}\\t{}\\t{}\\t{}\\t{:1.3f}\".format(t, tps, fps, fns, p))\n\n    if verbose:\n        print(\"AP\\t-\\t-\\t-\\t{:1.3f}\".format(np.mean(prec)))\n\n    return np.mean(prec)\n\n\ndef get_score(ds, mdl):\n    \"\"\"\n    Get average IOU mAP score for a dataset\n    \"\"\"\n    mdl.eval()\n    iouscore = 0\n    for i in tqdm(range(len(ds))):\n        img, targets = ds[i]\n        with torch.no_grad():\n            result = mdl([img.to(DEVICE)])[0]\n            \n        masks = combine_masks(targets['masks'], 0.5)\n        labels = pd.Series(result['labels'].cpu().numpy()).value_counts()\n\n        mask_threshold = mask_threshold_dict[labels.sort_values().index[-1]]\n        pred_masks = combine_masks(get_filtered_masks(result), mask_threshold)\n        iouscore += iou_map([masks],[pred_masks])\n    return iouscore / len(ds)\n","metadata":{"papermill":{"duration":0.045718,"end_time":"2021-11-16T19:15:48.078982","exception":false,"start_time":"2021-11-16T19:15:48.033264","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:11.931241Z","iopub.execute_input":"2021-11-27T14:19:11.931533Z","iopub.status.idle":"2021-11-27T14:19:11.948806Z","shell.execute_reply.started":"2021-11-27T14:19:11.931495Z","shell.execute_reply":"2021-11-27T14:19:11.948080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transformations\nJust Horizontal and Vertical Flip for now.\n\nNormalization to Resnet's mean and std can be performed using the parameter `NORMALIZE` in the top cell.\n\nThe first 3 transformations come from [this](https://www.kaggle.com/abhishek/maskrcnn-utils) utils package by Abishek, `VerticalFlip` is my adaption of HorizontalFlip, and `Normalize` is of my own.","metadata":{"papermill":{"duration":0.02436,"end_time":"2021-11-16T19:15:48.128156","exception":false,"start_time":"2021-11-16T19:15:48.103796","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# These are slight redefinitions of torch.transformation classes\n# The difference is that they handle the target and the mask\n# Copied from Abishek, added new ones\nclass Compose:\n    def __init__(self, transforms):\n        self.transforms = transforms\n\n    def __call__(self, image, target):\n        for t in self.transforms:\n            image, target = t(image, target)\n        return image, target\n\nclass VerticalFlip:\n    def __init__(self, prob):\n        self.prob = prob\n\n    def __call__(self, image, target):\n        if random.random() < self.prob:\n            height, width = image.shape[-2:]\n            image = image.flip(-2)\n            bbox = target[\"boxes\"]\n            bbox[:, [1, 3]] = height - bbox[:, [3, 1]]\n            target[\"boxes\"] = bbox\n            target[\"masks\"] = target[\"masks\"].flip(-2)\n        return image, target\n\nclass HorizontalFlip:\n    def __init__(self, prob):\n        self.prob = prob\n\n    def __call__(self, image, target):\n        if random.random() < self.prob:\n            height, width = image.shape[-2:]\n            image = image.flip(-1)\n            bbox = target[\"boxes\"]\n            bbox[:, [0, 2]] = width - bbox[:, [2, 0]]\n            target[\"boxes\"] = bbox\n            target[\"masks\"] = target[\"masks\"].flip(-1)\n        return image, target\n\nclass Normalize:\n    def __call__(self, image, target):\n        image = F.normalize(image, RESNET_MEAN, RESNET_STD)\n        return image, target\n\nclass ToTensor:\n    def __call__(self, image, target):\n        image = F.to_tensor(image)\n        return image, target\n    \n\ndef get_transforms():\n    transforms = A.Compose([A.RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n                               A.CLAHE(p=1)\n                               ])\n    #augmentation = transforms(image = img)\n    #img = augmentation['image']\n    tensor_transform = Compose([ToTensor()])\n    #img = tensor_transform(img)\n    return transforms, tensor_transform\n\n","metadata":{"papermill":{"duration":0.041677,"end_time":"2021-11-16T19:15:48.194207","exception":false,"start_time":"2021-11-16T19:15:48.15253","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:11.953836Z","iopub.execute_input":"2021-11-27T14:19:11.954153Z","iopub.status.idle":"2021-11-27T14:19:11.967766Z","shell.execute_reply.started":"2021-11-27T14:19:11.954122Z","shell.execute_reply":"2021-11-27T14:19:11.967073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentation","metadata":{}},{"cell_type":"code","source":"# transforms = A.Compose([\n#         #A.RandomResizedCrop(450,450),\n#         #A.Transpose(p=0.5),\n#         #A.HorizontalFlip(p=1),\n#         #A.VerticalFlip(p=0.8),\n#         #A.ShiftScaleRotate(p=0.5),\n#         #A.GaussNoise(),\n#         #A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n#         A.RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n#         #A.RandomSizedCrop((MAX_SIZE-100, MAX_SIZE), HEIGHT//2, WIDTH//2, w2h_ratio=1.0, \n#                                         #interpolation=cv2.INTER_LINEAR, always_apply=False, p=0.5),  \n#         #A.Resize(HEIGHT//2, WIDTH//2, interpolation=cv2.INTER_LINEAR, p=1), \n#         A.CLAHE(p=1),\n#         #A.OneOf([A.MotionBlur(p=0.2),\n#          #        A.MedianBlur(blur_limit=3, p=0.1),\n#           #       A.Blur(blur_limit=3, p=0.1),\n#            #     ], p=0.5),\n#         #A.Normalize(),\n#         ToTensorV2()\n        \n#         ], bbox_params=A.BboxParams(format='pascal_voc', min_area=0, label_fields= ['labels']) \n#                     , p=1)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:19:11.969049Z","iopub.execute_input":"2021-11-27T14:19:11.969658Z","iopub.status.idle":"2021-11-27T14:19:11.979414Z","shell.execute_reply.started":"2021-11-27T14:19:11.969623Z","shell.execute_reply":"2021-11-27T14:19:11.978688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training Dataset and DataLoader","metadata":{"id":"hHT_aovU0zRd","papermill":{"duration":0.023997,"end_time":"2021-11-16T19:15:48.242236","exception":false,"start_time":"2021-11-16T19:15:48.218239","status":"completed"},"tags":[]}},{"cell_type":"code","source":"cell_type_dict = {\"astro\": 1, \"cort\": 2, \"shsy5y\": 3}\n\nclass CellDataset(Dataset):\n    def __init__(self, image_dir, df, transforms=None, resize=False):\n        self.transforms = transforms\n        self.image_dir = image_dir\n        self.df = df\n        \n        self.should_resize = resize is not False\n        if self.should_resize:\n            self.height = int(HEIGHT * resize)\n            self.width = int(WIDTH * resize)\n            print(\"image size used:\", self.height, self.width)\n        else:\n            self.height = HEIGHT\n            self.width = WIDTH\n        \n        self.image_info = collections.defaultdict(dict)\n        temp_df = self.df.groupby([\"id\", \"cell_type\"])['annotation'].agg(lambda x: list(x)).reset_index()\n        for index, row in temp_df.iterrows():\n            self.image_info[index] = {\n                    'image_id': row['id'],\n                    'image_path': os.path.join(self.image_dir, row['id'] + '.png'),\n                    'annotations': list(row[\"annotation\"]),\n                    'cell_type': cell_type_dict[row[\"cell_type\"]]\n                    }\n            \n    def get_box(self, a_mask):\n        ''' Get the bounding box of a given mask '''\n        pos = np.where(a_mask)\n        xmin = np.min(pos[1])\n        xmax = np.max(pos[1])\n        ymin = np.min(pos[0])\n        ymax = np.max(pos[0])\n        return [xmin, ymin, xmax, ymax]\n\n    def __getitem__(self, idx):\n        ''' Get the image and the target'''\n        \n        img_path = self.image_info[idx][\"image_path\"]\n        img = cv2.imread(img_path, cv2.IMREAD_COLOR)\n        \n        if self.should_resize:\n            img = cv2.resize(img, (self.width, self.height))\n\n        info = self.image_info[idx]\n\n        n_objects = len(info['annotations'])\n        masks = np.zeros((len(info['annotations']), self.height, self.width), dtype=np.uint8)\n        boxes = []\n        labels = []\n        for i, annotation in enumerate(info['annotations']):\n            a_mask = rle_decode(annotation, (HEIGHT, WIDTH))\n            \n            if self.should_resize:\n                a_mask = cv2.resize(a_mask, (self.width, self.height))\n            \n            a_mask = np.array(a_mask) > 0\n            masks[i, :, :] = a_mask\n            \n            boxes.append(self.get_box(a_mask))\n\n        # labels\n        labels = [int(info[\"cell_type\"]) for _ in range(n_objects)]\n        #labels = [1 for _ in range(n_objects)]\n        \n        \n        boxes = torch.as_tensor(boxes, dtype=torch.float32)\n        labels = torch.as_tensor(labels, dtype=torch.int64)\n        masks = torch.as_tensor(masks, dtype=torch.uint8)\n\n        image_id = torch.tensor([idx])\n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n        iscrowd = torch.zeros((n_objects,), dtype=torch.int64)\n\n        # This is the required target for the Mask R-CNN\n        target = {\n            'boxes': boxes,\n            'labels': labels,\n            'masks': masks,\n            'image_id': image_id,\n            'area': area,\n            'iscrowd': iscrowd\n        }\n\n        if self.transforms is not None:\n            augmentation, totensor = self.transforms\n            aug = augmentation(image = img)\n            img = aug['image']\n            img, target = totensor(img, target)\n            #augmentation = self.transforms(image = img)\n            #img = augmentation['image']\n            #img = self.transforms(img)\n        #transforms = Compose([ToTensor()])\n        #img = transforms(img)\n        \n        return img, target\n\n    def __len__(self):\n        return len(self.image_info)","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:19:11.993177Z","iopub.execute_input":"2021-11-27T14:19:11.993548Z","iopub.status.idle":"2021-11-27T14:19:12.014554Z","shell.execute_reply.started":"2021-11-27T14:19:11.993521Z","shell.execute_reply":"2021-11-27T14:19:12.013855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cell_type_dict = {\"astro\": 1, \"cort\": 2, \"shsy5y\": 3}\n\n# class CellDataset(Dataset):\n#     def __init__(self, image_dir, df, transforms=None, resize=False):\n#         self.transforms = transforms\n#         self.image_dir = image_dir\n#         self.df = df\n        \n#         self.should_resize = resize is not False\n#         if self.should_resize:\n#             self.height = int(HEIGHT * resize)\n#             self.width = int(WIDTH * resize)\n#             print(\"image size used:\", self.height, self.width)\n#         else:\n#             self.height = HEIGHT\n#             self.width = WIDTH\n        \n#         self.image_info = collections.defaultdict(dict)\n#         temp_df = self.df.groupby([\"id\", \"cell_type\"])['annotation'].agg(lambda x: list(x)).reset_index()\n#         for index, row in temp_df.iterrows():\n#             self.image_info[index] = {\n#                     'image_id': row['id'],\n#                     'image_path': os.path.join(self.image_dir, row['id'] + '.png'),\n#                     'annotations': list(row[\"annotation\"]),\n#                     'cell_type': cell_type_dict[row[\"cell_type\"]]\n#                     }\n            \n#     def get_box(self, a_mask):\n#         ''' Get the bounding box of a given mask '''\n#         pos = np.where(a_mask)\n#         xmin = np.min(pos[1])\n#         xmax = np.max(pos[1])\n#         ymin = np.min(pos[0])\n#         ymax = np.max(pos[0])\n#         return [xmin, ymin, xmax, ymax]\n\n#     def __getitem__(self, idx):\n#         ''' Get the image and the target'''\n        \n#         img_path = self.image_info[idx][\"image_path\"]\n#         img = cv2.imread(img_path, cv2.IMREAD_COLOR)\n       \n#         if self.should_resize:\n#             img = cv2.resize(img, (self.width, self.height))\n\n#         info = self.image_info[idx]\n\n#         n_objects = len(info['annotations'])\n#         masks = np.zeros((len(info['annotations']), self.height, self.width), dtype=np.uint8)\n#         boxes = []\n#         labels = []\n#         for i, annotation in enumerate(info['annotations']):\n#             a_mask = rle_decode(annotation, (HEIGHT, WIDTH))\n            \n#             if self.should_resize:\n#                 a_mask = cv2.resize(a_mask, (self.width, self.height))\n            \n#             a_mask = np.array(a_mask) > 0\n#             masks[i, :, :] = a_mask\n            \n#             boxes.append(self.get_box(a_mask))\n\n#         # labels\n#         labels = [int(info[\"cell_type\"]) for _ in range(n_objects)]\n#         #labels = [1 for _ in range(n_objects)]\n#         boxes = np.array(boxes, dtype=np.float32)\n#         labels = np.array(labels, dtype=np.int64)\n#         masks = np.array(masks, dtype=np.uint8)\n        \n#         image_id = torch.tensor([idx])\n#         area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n#         iscrowd = torch.zeros((n_objects,), dtype=torch.int64)\n#         area_as_tensor = torch.as_tensor(area, dtype=torch.uint8)\n\n#         # This is the required target for the Mask R-CNN\n#         if self.transforms is not None :\n    \n#             augmentation = self.transforms(image=img, mask=masks, bboxes=boxes, labels=labels)     \n#             img_as_tensor = augmentation['image']\n#             masks_as_tensor = augmentation['mask']\n#             boxes_as_list = augmentation['bboxes']\n#             lables_as_list = augmentation['labels']\n#             boxes_as_tensor = torch.as_tensor(boxes_as_list, dtype=torch.float32)\n#             labels_as_tensor = torch.as_tensor(lables_as_list, dtype=torch.int64)\n        \n#         else :\n#             img_as_tensor = torch.as_tensor(img, dtype=torch.float32)\n#             boxes_as_tensor = torch.as_tensor(boxes, dtype=torch.float32)\n#             labels_as_tensor = torch.as_tensor(labels, dtype=torch.int64)\n#             masks_as_tensor = torch.as_tensor(masks, dtype=torch.uint8)\n            \n            \n#         target = {\n#             'boxes': boxes_as_tensor,\n#             'labels': labels_as_tensor,\n#             'masks': masks_as_tensor,\n#             'image_id': image_id,\n#             'area': area_as_tensor,\n#             'iscrowd': iscrowd\n#         }\n\n#         #if self.transforms is not None:\n#          #   img, target = self.transforms(img, target)\n\n#         return img_as_tensor, target\n\n#     def __len__(self):\n#         return len(self.image_info)","metadata":{"id":"C9Y03YgA0zRd","papermill":{"duration":0.047664,"end_time":"2021-11-16T19:15:48.313898","exception":false,"start_time":"2021-11-16T19:15:48.266234","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:12.029991Z","iopub.execute_input":"2021-11-27T14:19:12.030279Z","iopub.status.idle":"2021-11-27T14:19:12.036508Z","shell.execute_reply.started":"2021-11-27T14:19:12.030252Z","shell.execute_reply":"2021-11-27T14:19:12.035837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base = pd.read_csv(TRAIN_CSV, nrows=5000 if TEST else None)","metadata":{"id":"tmCw3DTL0zRe","papermill":{"duration":0.534642,"end_time":"2021-11-16T19:15:48.872797","exception":false,"start_time":"2021-11-16T19:15:48.338155","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:12.037841Z","iopub.execute_input":"2021-11-27T14:19:12.038463Z","iopub.status.idle":"2021-11-27T14:19:12.311411Z","shell.execute_reply.started":"2021-11-27T14:19:12.038421Z","shell.execute_reply":"2021-11-27T14:19:12.310608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_images = df_base.groupby([\"id\", \"cell_type\"]).agg({'annotation': 'count'}).sort_values(\"annotation\", ascending=False).reset_index()\n\nfor ct in cell_type_dict:\n    ctdf = df_images[df_images[\"cell_type\"]==ct].copy()\n    if len(ctdf)>0:\n        ctdf['quantiles'] = pd.qcut(ctdf['annotation'], 5)\n        display(ctdf.head())","metadata":{"id":"pQXGwdnL0zRe","outputId":"08e892c7-aa0c-4b96-de86-7eb8a4234f16","papermill":{"duration":0.114291,"end_time":"2021-11-16T19:15:49.012377","exception":false,"start_time":"2021-11-16T19:15:48.898086","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:12.313266Z","iopub.execute_input":"2021-11-27T14:19:12.313531Z","iopub.status.idle":"2021-11-27T14:19:12.375134Z","shell.execute_reply.started":"2021-11-27T14:19:12.313496Z","shell.execute_reply":"2021-11-27T14:19:12.374352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_images.groupby(\"cell_type\").annotation.describe().astype(int)","metadata":{"id":"DQeF1Zi00zRf","outputId":"af89f7a7-245f-4583-85e3-4f4872c62171","papermill":{"duration":0.049592,"end_time":"2021-11-16T19:15:49.088235","exception":false,"start_time":"2021-11-16T19:15:49.038643","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:12.376516Z","iopub.execute_input":"2021-11-27T14:19:12.376790Z","iopub.status.idle":"2021-11-27T14:19:12.397276Z","shell.execute_reply.started":"2021-11-27T14:19:12.376755Z","shell.execute_reply":"2021-11-27T14:19:12.396590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We used this as a reference to fill BOX_DETECTIONS_PER_IMG=140\ndf_images[['annotation']].describe().astype(int)","metadata":{"id":"oCRxcK2f0zRf","outputId":"ea5a6673-5758-4964-cdde-03235c5c8215","papermill":{"duration":0.046519,"end_time":"2021-11-16T19:15:49.161404","exception":false,"start_time":"2021-11-16T19:15:49.114885","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:12.399530Z","iopub.execute_input":"2021-11-27T14:19:12.400052Z","iopub.status.idle":"2021-11-27T14:19:12.413245Z","shell.execute_reply.started":"2021-11-27T14:19:12.400016Z","shell.execute_reply":"2021-11-27T14:19:12.412306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use the quantiles of amoount of annotations to stratify\ndf_images_train, df_images_val = train_test_split(df_images, stratify=df_images['cell_type'], \n                                                  test_size=PCT_IMAGES_VALIDATION,\n                                                  random_state=1234)\ndf_train = df_base[df_base['id'].isin(df_images_train['id'])]\ndf_val = df_base[df_base['id'].isin(df_images_val['id'])]\nprint(f\"Images in train set:           {len(df_images_train)}\")\nprint(f\"Annotations in train set:      {len(df_train)}\")\nprint(f\"Images in validation set:      {len(df_images_val)}\")\nprint(f\"Annotations in validation set: {len(df_val)}\")","metadata":{"id":"2v7VvtTp0zRf","outputId":"82690b02-dd4b-4c1d-ed5d-78e30bf084a2","papermill":{"duration":0.060789,"end_time":"2021-11-16T19:15:49.249546","exception":false,"start_time":"2021-11-16T19:15:49.188757","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:12.414541Z","iopub.execute_input":"2021-11-27T14:19:12.415022Z","iopub.status.idle":"2021-11-27T14:19:12.445050Z","shell.execute_reply.started":"2021-11-27T14:19:12.414974Z","shell.execute_reply":"2021-11-27T14:19:12.444404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_train = CellDataset(TRAIN_PATH, df_train, resize=resize_factor, transforms = get_transforms())\ndl_train = DataLoader(ds_train, batch_size=BATCH_SIZE, shuffle=True, pin_memory=True,\n                      num_workers=2, collate_fn=lambda x: tuple(zip(*x)))\n\n\n\nds_val = CellDataset(TRAIN_PATH, df_val, resize=resize_factor, transforms = get_transforms())\ndl_val = DataLoader(ds_val, batch_size=BATCH_SIZE, shuffle=True, pin_memory=True,\n                    num_workers=2, collate_fn=lambda x: tuple(zip(*x)))","metadata":{"id":"kUcpAbdO0zRg","papermill":{"duration":0.124209,"end_time":"2021-11-16T19:15:49.402795","exception":false,"start_time":"2021-11-16T19:15:49.278586","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:12.446088Z","iopub.execute_input":"2021-11-27T14:19:12.446395Z","iopub.status.idle":"2021-11-27T14:19:12.535593Z","shell.execute_reply.started":"2021-11-27T14:19:12.446359Z","shell.execute_reply":"2021-11-27T14:19:12.534881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img, target = next(iter(dl_train))\nlen(img)\nimg[0].shape\n#img = img[0].reshape((-1,img[0].shape[0], img[0].shape[1]))\n#img.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:19:12.536980Z","iopub.execute_input":"2021-11-27T14:19:12.537312Z","iopub.status.idle":"2021-11-27T14:19:14.373539Z","shell.execute_reply.started":"2021-11-27T14:19:12.537271Z","shell.execute_reply":"2021-11-27T14:19:14.372697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img[0].dtype\n#target[0]\nlen(target)","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:19:14.376235Z","iopub.execute_input":"2021-11-27T14:19:14.376704Z","iopub.status.idle":"2021-11-27T14:19:14.384516Z","shell.execute_reply.started":"2021-11-27T14:19:14.376662Z","shell.execute_reply":"2021-11-27T14:19:14.383499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#target = [{k: v.to(DEVICE) for k, v in t.items()} for t in targets]\n#for t in target :\n#    for k, v in t.items() :\n#        v.to(DEVICE)\n#        print('ok')\n#image = img[0].type(torch.FloatTensor)\n#type(image)\n#model(img, target)","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:19:14.386548Z","iopub.execute_input":"2021-11-27T14:19:14.387179Z","iopub.status.idle":"2021-11-27T14:19:14.390827Z","shell.execute_reply.started":"2021-11-27T14:19:14.387125Z","shell.execute_reply":"2021-11-27T14:19:14.390167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train model","metadata":{"id":"y8JNMn770zRg","papermill":{"duration":0.027207,"end_time":"2021-11-16T19:15:49.457545","exception":false,"start_time":"2021-11-16T19:15:49.430338","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## setup model","metadata":{"papermill":{"duration":0.027206,"end_time":"2021-11-16T19:15:49.511797","exception":false,"start_time":"2021-11-16T19:15:49.484591","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Override pythorch checkpoint with an \"offline\" version of the file\n!mkdir -p /root/.cache/torch/hub/checkpoints/\n!cp ../input/cocopre/maskrcnn_resnet50_fpn_coco-bf2d0c1e.pth /root/.cache/torch/hub/checkpoints/maskrcnn_resnet50_fpn_coco-bf2d0c1e.pth","metadata":{"id":"VMaqdcNa0zRg","outputId":"c5ca31a5-6d8d-4639-e547-f44e5772c725","papermill":{"duration":4.974966,"end_time":"2021-11-16T19:15:54.513706","exception":false,"start_time":"2021-11-16T19:15:49.53874","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:14.394600Z","iopub.execute_input":"2021-11-27T14:19:14.395255Z","iopub.status.idle":"2021-11-27T14:19:16.508558Z","shell.execute_reply.started":"2021-11-27T14:19:14.395218Z","shell.execute_reply":"2021-11-27T14:19:16.507619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(num_classes, model_chkpt=None):\n    # This is just a dummy value for the classification head\n    \n    if NORMALIZE:\n        model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=True,\n                                                                   box_detections_per_img=BOX_DETECTIONS_PER_IMG,\n                                                                   image_mean=RESNET_MEAN,\n                                                                   image_std=RESNET_STD)\n    else:\n        model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=True,\n                                                                   box_detections_per_img=BOX_DETECTIONS_PER_IMG)\n\n    # get the number of input features for the classifier\n    in_features = model.roi_heads.box_predictor.cls_score.in_features\n    # replace the pre-trained head with a new one\n    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes+1)\n\n    # now get the number of input features for the mask classifier\n    in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels\n    hidden_layer = 256\n    # and replace the mask predictor with a new one\n    model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask, hidden_layer, num_classes+1)\n    \n    if model_chkpt:\n        model.load_state_dict(torch.load(model_chkpt, map_location=DEVICE))\n    return model\n\n# Get the Mask R-CNN model\n# The model does classification, bounding boxes and MASKs for individuals, all at the same time\n# We only care about MASKS\nmodel = get_model(len(cell_type_dict))\nmodel.to(DEVICE)\n\n# TODO: try removing this for\n#for param in model.parameters():\n#    param.requires_grad = True\n    \n#model.train();","metadata":{"id":"3Ds5dHex0zRh","papermill":{"duration":3.945677,"end_time":"2021-11-16T19:15:58.48714","exception":false,"start_time":"2021-11-16T19:15:54.541463","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:16.513533Z","iopub.execute_input":"2021-11-27T14:19:16.515546Z","iopub.status.idle":"2021-11-27T14:19:17.400286Z","shell.execute_reply.started":"2021-11-27T14:19:16.515503Z","shell.execute_reply":"2021-11-27T14:19:17.399540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training loop!","metadata":{"id":"RvawgUM30zRh","papermill":{"duration":0.027235,"end_time":"2021-11-16T19:15:58.542089","exception":false,"start_time":"2021-11-16T19:15:58.514854","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# params = [p for p in model.parameters() if p.requires_grad]\n# optimizer = torch.optim.SGD(params, lr=LEARNING_RATE, momentum=MOMENTUM, weight_decay=WEIGHT_DECAY)\n# #optimizer = torch.optim.Adam(params, lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n\n# lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1)\n\n# n_batches, n_batches_val = len(dl_train), len(dl_val)\n\n# validation_mask_losses = []\n\n# for epoch in range(1, NUM_EPOCHS + 1):\n#     print(f\"Starting epoch {epoch} of {NUM_EPOCHS}\")\n\n#     time_start = time.time()\n#     loss_accum = 0.0\n#     loss_mask_accum = 0.0\n#     loss_classifier_accum = 0.0\n#     for batch_idx, (images, targets) in enumerate(dl_train, 1):\n    \n#         # Predict\n#         images = list(image.type(torch.FloatTensor).to(DEVICE) for image in images)\n#         targets = [{k: v.to(DEVICE) for k, v in t.items()} for t in targets]\n\n#         loss_dict = model(images, targets)\n#         loss = sum(loss for loss in loss_dict.values())\n        \n#         # Backprop\n#         optimizer.zero_grad()\n#         loss.backward()\n#         optimizer.step()\n        \n#         # Logging\n#         loss_mask = loss_dict['loss_mask'].item()\n#         loss_accum += loss.item()\n#         loss_mask_accum += loss_mask\n#         loss_classifier_accum += loss_dict['loss_classifier'].item()\n        \n#         if batch_idx % 500 == 0:\n#             print(f\"    [Batch {batch_idx:3d} / {n_batches:3d}] Batch train loss: {loss.item():7.3f}. Mask-only loss: {loss_mask:7.3f}.\")\n                        \n#     if USE_SCHEDULER:\n#         lr_scheduler.step()\n\n#     # Train losses\n#     train_loss = loss_accum / n_batches\n#     train_loss_mask = loss_mask_accum / n_batches\n#     train_loss_classifier = loss_classifier_accum / n_batches\n\n#     # Validation\n#     val_loss_accum = 0\n#     val_loss_mask_accum = 0\n#     val_loss_classifier_accum = 0\n    \n#     with torch.no_grad():\n#         for batch_idx, (images, targets) in enumerate(dl_val, 1):\n#             images = list(image.type(torch.FloatTensor).to(DEVICE) for image in images)\n#             targets = [{k: v.to(DEVICE) for k, v in t.items()} for t in targets]\n\n#             val_loss_dict = model(images, targets)\n#             val_batch_loss = sum(loss for loss in val_loss_dict.values())\n#             val_loss_accum += val_batch_loss.item()\n#             val_loss_mask_accum += val_loss_dict['loss_mask'].item()\n#             val_loss_classifier_accum += val_loss_dict['loss_classifier'].item()\n\n#     # Validation losses\n#     val_loss = val_loss_accum / n_batches_val\n#     val_loss_mask = val_loss_mask_accum / n_batches_val\n#     val_loss_classifier = val_loss_classifier_accum / n_batches_val\n#     elapsed = time.time() - time_start\n\n#     validation_mask_losses.append(val_loss_mask)\n\n#     torch.save(model.state_dict(), f\"pytorch_model-e{epoch}.bin\")\n#     prefix = f\"[Epoch {epoch:2d} / {NUM_EPOCHS:2d}]\"\n#     print(prefix)\n#     print(f\"{prefix} Train mask-only loss: {train_loss_mask:7.3f}, classifier loss {train_loss_classifier:7.3f}\")\n#     print(f\"{prefix} Val mask-only loss  : {val_loss_mask:7.3f}, classifier loss {val_loss_classifier:7.3f}\")\n#     print(prefix)\n#     print(f\"{prefix} Train loss: {train_loss:7.3f}. Val loss: {val_loss:7.3f} [{elapsed:.0f} secs]\")\n#     print(prefix)","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:19:17.402910Z","iopub.execute_input":"2021-11-27T14:19:17.403188Z","iopub.status.idle":"2021-11-27T14:19:17.409430Z","shell.execute_reply.started":"2021-11-27T14:19:17.403150Z","shell.execute_reply":"2021-11-27T14:19:17.408562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# params = [p for p in model.parameters() if p.requires_grad]\n# optimizer = torch.optim.SGD(params, lr=LEARNING_RATE, momentum=MOMENTUM, weight_decay=WEIGHT_DECAY)\n# #optimizer = torch.optim.Adam(params, lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n\n# lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1)\n\n# n_batches, n_batches_val = len(dl_train), len(dl_val)\n\n# validation_mask_losses = []\n\n# for epoch in range(1, NUM_EPOCHS + 1):\n#     print(f\"Starting epoch {epoch} of {NUM_EPOCHS}\")\n\n#     time_start = time.time()\n#     loss_accum = 0.0\n#     loss_mask_accum = 0.0\n#     loss_classifier_accum = 0.0\n#     for batch_idx, (images, targets) in enumerate(dl_train, 1):\n        \n# #         # Predict\n#         images = [image.reshape((-1,img[0].shape[0], img[0].shape[1])) for image in images]\n#         images = list(image.type(torch.FloatTensor).to(DEVICE) for image in images)\n#         targets = [{k: v.to(DEVICE) for k, v in t.items()} for t in targets]\n#         #print(images[0].shape)\n#         loss_dict = model(images, targets)\n#         loss = sum(loss for loss in loss_dict.values())\n        \n# #         # Backprop\n#         optimizer.zero_grad()\n#         loss.backward()\n#         optimizer.step()\n        \n# #         # Logging\n#         loss_mask = loss_dict['loss_mask'].item()\n#         loss_accum += loss.item()\n#         loss_mask_accum += loss_mask\n#         loss_classifier_accum += loss_dict['loss_classifier'].item()\n#         if batch_idx == 20 :\n#             break\n#         if batch_idx % 500 == 0:\n#             print(f\"    [Batch {batch_idx:3d} / {n_batches:3d}] Batch train loss: {loss.item():7.3f}. Mask-only loss: {loss_mask:7.3f}.\")\n        \n#     if USE_SCHEDULER:\n#         lr_scheduler.step()\n\n#     # Train losses\n#     train_loss = loss_accum / n_batches\n#     train_loss_mask = loss_mask_accum / n_batches\n#     train_loss_classifier = loss_classifier_accum / n_batches\n\n# #     # Validation\n#     val_loss_accum = 0\n#     val_loss_mask_accum = 0\n#     val_loss_classifier_accum = 0\n    \n#     with torch.no_grad():\n#         for batch_idx, (images, targets) in enumerate(dl_val, 1):\n#             images = [image.reshape((-1,img[0].shape[0], img[0].shape[1])) for image in images]\n#             images = list(image.type(torch.FloatTensor).to(DEVICE) for image in images)\n#             #print(images[0].shape)\n#             targets = [{k: v.to(DEVICE) for k, v in t.items()} for t in targets]\n#             #images = images.tran\n#             val_loss_dict = model(images, targets)\n#             val_batch_loss = sum(loss for loss in val_loss_dict.values())\n#             val_loss_accum += val_batch_loss.item()\n#             val_loss_mask_accum += val_loss_dict['loss_mask'].item()\n#             val_loss_classifier_accum += val_loss_dict['loss_classifier'].item()\n\n# #     # Validation losses\n#     val_loss = val_loss_accum / n_batches_val\n#     val_loss_mask = val_loss_mask_accum / n_batches_val\n#     val_loss_classifier = val_loss_classifier_accum / n_batches_val\n#     elapsed = time.time() - time_start\n\n#     validation_mask_losses.append(val_loss_mask)\n\n#     torch.save(model.state_dict(), f\"pytorch_model-e{epoch}.bin\")\n#     prefix = f\"[Epoch {epoch:2d} / {NUM_EPOCHS:2d}]\"\n#     print(prefix)\n#     print(f\"{prefix} Train mask-only loss: {train_loss_mask:7.3f}, classifier loss {train_loss_classifier:7.3f}\")\n#     print(f\"{prefix} Val mask-only loss  : {val_loss_mask:7.3f}, classifier loss {val_loss_classifier:7.3f}\")\n#     print(prefix)\n#     print(f\"{prefix} Train loss: {train_loss:7.3f}. Val loss: {val_loss:7.3f} [{elapsed:.0f} secs]\")\n#     print(prefix)","metadata":{"id":"52B16JCW0zRh","outputId":"9b9c5ad9-58c1-4d50-dd7c-79b18b1e57b7","papermill":{"duration":0.03779,"end_time":"2021-11-16T19:15:58.607371","exception":false,"start_time":"2021-11-16T19:15:58.569581","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:17.411020Z","iopub.execute_input":"2021-11-27T14:19:17.411438Z","iopub.status.idle":"2021-11-27T14:19:17.422400Z","shell.execute_reply.started":"2021-11-27T14:19:17.411403Z","shell.execute_reply":"2021-11-27T14:19:17.421667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analyze prediction results for train set","metadata":{"id":"MspyyJlP0zRh","papermill":{"duration":0.026848,"end_time":"2021-11-16T19:15:58.661643","exception":false,"start_time":"2021-11-16T19:15:58.634795","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Plots: the image, The image + the ground truth mask, The image + the predicted mask\n\ndef analyze_train_sample(model, ds_train, sample_index):\n    \n    img, targets = ds_train[sample_index]\n    #print(img.shape)\n    l = np.unique(targets[\"labels\"])\n    ig, ax = plt.subplots(nrows=1, ncols=3, figsize=(20,60), facecolor=\"#fefefe\")\n    ax[0].imshow(img.numpy().transpose((1,2,0)))\n    ax[0].set_title(f\"cell type {l}\")\n    ax[0].axis(\"off\")\n    \n    masks = combine_masks(targets['masks'], 0.5)\n    #plt.imshow(img.numpy().transpose((1,2,0)))\n    ax[1].imshow(masks)\n    ax[1].set_title(f\"Ground truth, {len(targets['masks'])} cells\")\n    ax[1].axis(\"off\")\n    \n    model.eval()\n    with torch.no_grad():\n        preds = model([img.to(DEVICE)])[0]\n        print(preds['labels'][0])\n    l = pd.Series(preds['labels'].cpu().numpy()).value_counts()\n    lstr = \"\"\n    for i in l.index:\n        lstr += f\"{l[i]}x{i} \"\n    #print(l, l.sort_values().index[-1])\n    #plt.imshow(img.cpu().numpy().transpose((1,2,0)))\n    mask_threshold = mask_threshold_dict[l.sort_values().index[-1]]\n    #print(mask_threshold)\n    pred_masks = combine_masks(get_filtered_masks(preds), mask_threshold)\n    ax[2].imshow(pred_masks)\n    ax[2].set_title(f\"Predictions, labels: {lstr}\")\n    ax[2].axis(\"off\")\n    plt.show() \n    print(pred_masks)\n    #print(masks.shape, pred_masks.shape)\n    score = iou_map([masks],[pred_masks])\n    print(\"Score:\", score)    \n    \n    \n# NOTE: It puts the model in eval mode!! Revert for re-training\nanalyze_train_sample(model, ds_train, 20)","metadata":{"id":"dJOE3B0u0zRi","papermill":{"duration":6.44458,"end_time":"2021-11-16T19:16:05.13417","exception":false,"start_time":"2021-11-16T19:15:58.68959","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:17.425587Z","iopub.execute_input":"2021-11-27T14:19:17.425813Z","iopub.status.idle":"2021-11-27T14:19:18.408470Z","shell.execute_reply.started":"2021-11-27T14:19:17.425783Z","shell.execute_reply":"2021-11-27T14:19:18.407593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"analyze_train_sample(model, ds_train, 102)","metadata":{"id":"ZRx9K5n60zRi","outputId":"3ff2b8d0-8e4d-46a0-b3cb-7f2fad862808","papermill":{"duration":0.896035,"end_time":"2021-11-16T19:16:06.063052","exception":false,"start_time":"2021-11-16T19:16:05.167017","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:18.409605Z","iopub.execute_input":"2021-11-27T14:19:18.409819Z","iopub.status.idle":"2021-11-27T14:19:19.238770Z","shell.execute_reply.started":"2021-11-27T14:19:18.409793Z","shell.execute_reply":"2021-11-27T14:19:19.237755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"analyze_train_sample(model, ds_train, 7)","metadata":{"id":"Rn6YeGVZ0zRi","outputId":"3b0b0c2b-1f0b-40b0-9529-275d24c3afa3","papermill":{"duration":1.271094,"end_time":"2021-11-16T19:16:07.371504","exception":false,"start_time":"2021-11-16T19:16:06.10041","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:19.240401Z","iopub.execute_input":"2021-11-27T14:19:19.240665Z","iopub.status.idle":"2021-11-27T14:19:20.660946Z","shell.execute_reply.started":"2021-11-27T14:19:19.240629Z","shell.execute_reply":"2021-11-27T14:19:20.660175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get the model from the best epoch","metadata":{"id":"Jon4MSmk0zRj","papermill":{"duration":0.041534,"end_time":"2021-11-16T19:16:07.506405","exception":false,"start_time":"2021-11-16T19:16:07.464871","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Epochs with their losses and IOU scores\n\n# val_scores = pd.DataFrame()\n# for e, val_loss in enumerate(validation_mask_losses):\n#     model_chk = f\"pytorch_model-e{e+1}.bin\"\n#     print(\"Loading:\", model_chk)\n#     model = get_model(len(cell_type_dict), model_chk)\n#     model.load_state_dict(torch.load(model_chk))\n#     model = model.to(DEVICE)\n#     val_scores.loc[e,\"mask_loss\"] = val_loss\n#     val_scores.loc[e,\"score\"] = get_score(ds_val, model)\n    \n    \n# display(val_scores.sort_values(\"score\", ascending=False))\n\n# best_epoch = np.argmax(val_scores[\"score\"])\n# print(best_epoch+1)","metadata":{"id":"O0ejRcer0zRj","outputId":"0806ad69-abcf-440a-c9ad-d5a2a454abe4","papermill":{"duration":0.049237,"end_time":"2021-11-16T19:16:07.597721","exception":false,"start_time":"2021-11-16T19:16:07.548484","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:20.662344Z","iopub.execute_input":"2021-11-27T14:19:20.662772Z","iopub.status.idle":"2021-11-27T14:19:20.667452Z","shell.execute_reply.started":"2021-11-27T14:19:20.662719Z","shell.execute_reply":"2021-11-27T14:19:20.666623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{"id":"bTNGfMuQ0zRi","papermill":{"duration":0.043152,"end_time":"2021-11-16T19:16:07.683516","exception":false,"start_time":"2021-11-16T19:16:07.640364","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Test Dataset and DataLoader","metadata":{"id":"tRSo-FPt0zRi","papermill":{"duration":0.040244,"end_time":"2021-11-16T19:16:07.76467","exception":false,"start_time":"2021-11-16T19:16:07.724426","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CellTestDataset(Dataset):\n    def __init__(self, image_dir, transforms=None, resize=False):\n        self.transforms = transforms\n        self.image_dir = image_dir\n        self.image_ids = [f[:-4]for f in os.listdir(self.image_dir)]\n        self.should_resize = resize is not False\n        if self.should_resize:\n            self.height = int(HEIGHT * resize)\n            self.width = int(WIDTH * resize)\n            print(\"image size used:\", self.height, self.width)\n            \n    def __getitem__(self, idx):\n        image_id = self.image_ids[idx]\n        image_path = os.path.join(self.image_dir, image_id + '.png')\n        image = cv2.imread(image_path, cv2.IMREAD_COLOR)\n        if self.should_resize:\n            image = cv2.resize(image, (self.width, self.height))\n\n        if self.transforms is not None:\n            image, _ = self.transforms(image=image, target=None)\n        return {'image': image, 'image_id': image_id}\n\n    def __len__(self):\n        return len(self.image_ids)","metadata":{"id":"ijZzdcHB0zRj","papermill":{"duration":0.053315,"end_time":"2021-11-16T19:16:07.858612","exception":false,"start_time":"2021-11-16T19:16:07.805297","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:20.668791Z","iopub.execute_input":"2021-11-27T14:19:20.669103Z","iopub.status.idle":"2021-11-27T14:19:20.681411Z","shell.execute_reply.started":"2021-11-27T14:19:20.669067Z","shell.execute_reply":"2021-11-27T14:19:20.680704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_transforms = Compose([ToTensor()])","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:19:20.682724Z","iopub.execute_input":"2021-11-27T14:19:20.683512Z","iopub.status.idle":"2021-11-27T14:19:20.691197Z","shell.execute_reply.started":"2021-11-27T14:19:20.683475Z","shell.execute_reply":"2021-11-27T14:19:20.690469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%cd /kaggle/working","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:19:20.692268Z","iopub.execute_input":"2021-11-27T14:19:20.692727Z","iopub.status.idle":"2021-11-27T14:19:20.699645Z","shell.execute_reply.started":"2021-11-27T14:19:20.692692Z","shell.execute_reply":"2021-11-27T14:19:20.699027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from IPython.display import FileLink\n# FileLink(r'./pytorch_model-e12.bin')\n\n","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:19:20.700876Z","iopub.execute_input":"2021-11-27T14:19:20.701306Z","iopub.status.idle":"2021-11-27T14:19:20.709127Z","shell.execute_reply.started":"2021-11-27T14:19:20.701254Z","shell.execute_reply":"2021-11-27T14:19:20.708319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Override pythorch checkpoint with an \"offline\" version of the file\n# !mkdir -p /root/.cache/torch/hub/checkpoints/\n#!cp ./pytorch_model-e27.bin ../input/sartorius-cell-instance-segmentation","metadata":{"papermill":{"duration":1.020145,"end_time":"2021-11-16T19:16:08.946029","exception":false,"start_time":"2021-11-16T19:16:07.925884","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:20.712349Z","iopub.execute_input":"2021-11-27T14:19:20.712930Z","iopub.status.idle":"2021-11-27T14:19:20.717578Z","shell.execute_reply.started":"2021-11-27T14:19:20.712902Z","shell.execute_reply":"2021-11-27T14:19:20.716824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_test = CellTestDataset(TEST_PATH, transforms = test_transforms)","metadata":{"id":"WbciaVrJ0zRj","outputId":"3ec69aa1-4136-41a1-b853-67d38654ef9a","papermill":{"duration":0.138386,"end_time":"2021-11-16T19:16:09.227981","exception":false,"start_time":"2021-11-16T19:16:09.089595","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:20.719276Z","iopub.execute_input":"2021-11-27T14:19:20.719491Z","iopub.status.idle":"2021-11-27T14:19:20.727260Z","shell.execute_reply.started":"2021-11-27T14:19:20.719466Z","shell.execute_reply":"2021-11-27T14:19:20.726389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_chk = f\"pytorch_model-e{best_epoch+1}.bin\"\nmodel_chk=\"../input/best-model/pytorch_model-e12.bin\"\nprint(\"Loading:\", model_chk)\nmodel = get_model(len(cell_type_dict))\nmodel.load_state_dict(torch.load(model_chk))\nmodel = model.to(DEVICE)\n\nfor param in model.parameters():\n    param.requires_grad = False\n\nmodel.eval();\n\nsubmission = []\nfor sample in ds_test:\n    img = sample['image']\n    image_id = sample['image_id']\n    with torch.no_grad():\n        result = model([img.to(DEVICE)])[0]\n    \n    previous_masks = []\n    for i, mask in enumerate(result[\"masks\"]):\n\n        # Filter-out low-scoring results.\n        score = result[\"scores\"][i].cpu().item()\n        label = result[\"labels\"][i].cpu().item()\n        if score > min_score_dict[label]:\n            mask = mask.cpu().numpy()\n            # Keep only highly likely pixels\n            binary_mask = mask > mask_threshold_dict[label]\n            binary_mask = remove_overlapping_pixels(binary_mask, previous_masks)\n            previous_masks.append(binary_mask)\n            rle = rle_encoding(binary_mask)\n            submission.append((image_id, rle))\n\n    # Add empty prediction if no RLE was generated for this image\n    all_images_ids = [image_id for image_id, rle in submission]\n    if image_id not in all_images_ids:\n        submission.append((image_id, \"\"))\n\ndf_sub = pd.DataFrame(submission, columns=['id', 'predicted'])\ndf_sub.to_csv(\"submission.csv\", index=False)\ndf_sub.head()","metadata":{"id":"dUrfObRF0zRk","outputId":"5f0432a3-8b87-4540-f49d-9d033b61bdbc","papermill":{"duration":8.818355,"end_time":"2021-11-16T19:16:18.114106","exception":false,"start_time":"2021-11-16T19:16:09.295751","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-27T14:19:20.728734Z","iopub.execute_input":"2021-11-27T14:19:20.729166Z","iopub.status.idle":"2021-11-27T14:19:27.337351Z","shell.execute_reply.started":"2021-11-27T14:19:20.729127Z","shell.execute_reply":"2021-11-27T14:19:27.336654Z"},"trusted":true},"execution_count":null,"outputs":[]}]}