{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Reference:\n\nhttps://www.kaggle.com/rluethy/sartorius-torch-mask-r-cnn\n","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport time\nimport collections\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nimport torchvision\nfrom torchvision.transforms import ToPILImage\nfrom torchvision.transforms import functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection.mask_rcnn import MaskRCNNPredictor\n\ndef seed_all(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n    \nseed_all(1)","metadata":{"id":"aAswznIORdXP","executionInfo":{"status":"ok","timestamp":1640900771241,"user_tz":-330,"elapsed":2750,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-21T15:42:17.129658Z","iopub.execute_input":"2022-06-21T15:42:17.130059Z","iopub.status.idle":"2022-06-21T15:42:17.140511Z","shell.execute_reply.started":"2022-06-21T15:42:17.130016Z","shell.execute_reply":"2022-06-21T15:42:17.139521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST = False\ndata_directory = './'\nDEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\nBATCH_SIZE = 1\nNUM_EPOCHS = 1\nWIDTH = 704\nHEIGHT = 520\nresize_factor = False\nNORMALIZE = False\nRESNET_MEAN = (0.485, 0.456, 0.406)\nRESNET_STD = (0.229, 0.224, 0.225)\ncell_type_dict = {\"astro\": 1, \"cort\": 2, \"shsy5y\": 3}\nmask_threshold_dict = {1: 0.50, 2: 0.65, 3:  0.6}\nmin_score_dict = {1: 0.50, 2: 0.75, 3: 0.5}\nBOX_DETECTIONS_PER_IMG = 540\nTRAIN_CSV = \"../input/sartorius-cell-instance-segmentation/train.csv\"\nTRAIN_PATH = \"../input/sartorius-cell-instance-segmentation/train\"\nTEST_PATH = \"../input/sartorius-cell-instance-segmentation/test\"","metadata":{"id":"ZMpNlHRKRdTn","executionInfo":{"status":"ok","timestamp":1640900771242,"user_tz":-330,"elapsed":7,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-21T15:43:29.571586Z","iopub.execute_input":"2022-06-21T15:43:29.571951Z","iopub.status.idle":"2022-06-21T15:43:29.579477Z","shell.execute_reply.started":"2022-06-21T15:43:29.571921Z","shell.execute_reply":"2022-06-21T15:43:29.578437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_decode(mask_rle, shape, color=1):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height, width, channels) of array to return\n    color: color for the mask\n    Returns numpy array (mask)\n\n    '''\n    s = mask_rle.split()\n\n    starts = list(map(lambda x: int(x) - 1, s[0::2]))\n    lengths = list(map(int, s[1::2]))\n    ends = [x + y for x, y in zip(starts, lengths)]\n    if len(shape)==3:\n        img = np.zeros((shape[0] * shape[1], shape[2]), dtype=np.float32)\n    else:\n        img = np.zeros(shape[0] * shape[1], dtype=np.float32)\n    for start, end in zip(starts, ends):\n        img[start : end] = color\n\n    return img.reshape(shape)\n\n\ndef rle_encoding(x):\n    dots = np.where(x.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return ' '.join(map(str, run_lengths))\n\n\ndef remove_overlapping_pixels(mask, other_masks):\n    for other_mask in other_masks:\n        if np.sum(np.logical_and(mask, other_mask)) > 0:\n            mask[np.logical_and(mask, other_mask)] = 0\n    return mask\n\ndef combine_masks(masks, mask_threshold):\n    \"\"\"\n    combine masks into one image\n    \"\"\"\n    maskimg = np.zeros((HEIGHT, WIDTH))\n\n    for m, mask in enumerate(masks,1):\n        maskimg[mask>mask_threshold] = m\n    return maskimg\n\n\ndef get_filtered_masks(pred):\n    \"\"\"\n    filter masks using MIN_SCORE for mask and MAX_THRESHOLD for pixels\n    \"\"\"\n    use_masks = []   \n    for i, mask in enumerate(pred[\"masks\"]):\n\n        # Filter-out low-scoring results. Not tried yet.\n        scr = pred[\"scores\"][i].cpu().item()\n        label = pred[\"labels\"][i].cpu().item()\n        if scr > min_score_dict[label]:\n            mask = mask.cpu().numpy().squeeze()\n            # Keep only highly likely pixels\n            binary_mask = mask > mask_threshold_dict[label]\n            binary_mask = remove_overlapping_pixels(binary_mask, use_masks)\n            use_masks.append(binary_mask)\n\n    return use_masks    ","metadata":{"id":"WY4qb8cfSXOH","executionInfo":{"status":"ok","timestamp":1640900771243,"user_tz":-330,"elapsed":7,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-21T15:43:32.245886Z","iopub.execute_input":"2022-06-21T15:43:32.246621Z","iopub.status.idle":"2022-06-21T15:43:32.264589Z","shell.execute_reply.started":"2022-06-21T15:43:32.246581Z","shell.execute_reply":"2022-06-21T15:43:32.263408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_iou(labels, y_pred, verbose=0):\n    \"\"\"\n    Computes the IoU for instance labels and predictions.\n\n    Args:\n        labels (np array): Labels.\n        y_pred (np array): predictions\n\n    Returns:\n        np array: IoU matrix, of size true_objects x pred_objects.\n    \"\"\"\n\n    true_objects = len(np.unique(labels))\n    pred_objects = len(np.unique(y_pred))\n\n    if verbose:\n        print(\"Number of true objects: {}\".format(true_objects))\n        print(\"Number of predicted objects: {}\".format(pred_objects))\n\n    # Compute intersection between all objects\n    intersection = np.histogram2d(\n        labels.flatten(), y_pred.flatten(), bins=(true_objects, pred_objects)\n    )[0]\n\n    # Compute areas (needed for finding the union between all objects)\n    area_true = np.histogram(labels, bins=true_objects)[0]\n    area_pred = np.histogram(y_pred, bins=pred_objects)[0]\n    area_true = np.expand_dims(area_true, -1)\n    area_pred = np.expand_dims(area_pred, 0)\n\n    # Compute union\n    union = area_true + area_pred - intersection\n    intersection = intersection[1:, 1:] # exclude background\n    union = union[1:, 1:]\n    union[union == 0] = 1e-9\n    iou = intersection / union\n    \n    return iou  \n\ndef precision_at(threshold, iou):\n    \"\"\"\n    Computes the precision at a given threshold.\n\n    Args:\n        threshold (float): Threshold.\n        iou (np array): IoU matrix.\n\n    Returns:\n        int: Number of true positives,\n        int: Number of false positives,\n        int: Number of false negatives.\n    \"\"\"\n    matches = iou > threshold\n    true_positives = np.sum(matches, axis=1) == 1  # Correct objects\n    false_positives = np.sum(matches, axis=0) == 0  # Missed objects\n    false_negatives = np.sum(matches, axis=1) == 0  # Extra objects\n    tp, fp, fn = (\n        np.sum(true_positives),\n        np.sum(false_positives),\n        np.sum(false_negatives),\n    )\n    return tp, fp, fn\n\ndef iou_map(truths, preds, verbose=0):\n    \"\"\"\n    Computes the metric for the competition.\n    Masks contain the segmented pixels where each object has one value associated,\n    and 0 is the background.\n\n    Args:\n        truths (list of masks): Ground truths.\n        preds (list of masks): Predictions.\n        verbose (int, optional): Whether to print infos. Defaults to 0.\n\n    Returns:\n        float: mAP.\n    \"\"\"\n    ious = [compute_iou(truth, pred, verbose) for truth, pred in zip(truths, preds)]\n\n    if verbose:\n        print(\"Thresh\\tTP\\tFP\\tFN\\tPrec.\")\n\n    prec = []\n    for t in np.arange(0.5, 1.0, 0.05):\n        tps, fps, fns = 0, 0, 0\n        for iou in ious:\n            tp, fp, fn = precision_at(t, iou)\n            tps += tp\n            fps += fp\n            fns += fn\n\n        p = tps / (tps + fps + fns)\n        prec.append(p)\n\n        if verbose:\n            print(\"{:1.3f}\\t{}\\t{}\\t{}\\t{:1.3f}\".format(t, tps, fps, fns, p))\n\n    if verbose:\n        print(\"AP\\t-\\t-\\t-\\t{:1.3f}\".format(np.mean(prec)))\n\n    return np.mean(prec)\n\n\ndef get_score(ds, mdl):\n    \"\"\"\n    Get average IOU mAP score for a dataset\n    \"\"\"\n    mdl.eval()\n    iouscore = 0\n    for i in tqdm(range(len(ds))):\n        img, targets = ds[i]\n        with torch.no_grad():\n            result = mdl([img.to(DEVICE)])[0]\n            \n        masks = combine_masks(targets['masks'], 0.5)\n        labels = pd.Series(result['labels'].cpu().numpy()).value_counts()\n\n        mask_threshold = mask_threshold_dict[labels.sort_values().index[-1]]\n        pred_masks = combine_masks(get_filtered_masks(result), mask_threshold)\n        iouscore += iou_map([masks],[pred_masks])\n    return iouscore / len(ds)","metadata":{"id":"fcAn1JJqSXK_","executionInfo":{"status":"ok","timestamp":1640900771243,"user_tz":-330,"elapsed":6,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-21T15:43:32.393194Z","iopub.execute_input":"2022-06-21T15:43:32.394132Z","iopub.status.idle":"2022-06-21T15:43:32.414963Z","shell.execute_reply.started":"2022-06-21T15:43:32.394088Z","shell.execute_reply":"2022-06-21T15:43:32.413782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transform(train):\n  if train:\n    if NORMALIZE:\n          transforms = torchvision.transforms.Compose([\n          torchvision.transforms.ToPILImage(),     \n          torchvision.transforms.RandomGrayscale(p=0.1),           \n          torchvision.transforms.ToTensor(),\n          torchvision.transforms.Normalize(RESNET_MEAN, RESNET_STD),\n          ])\n    else:\n      transforms = torchvision.transforms.Compose([\n          torchvision.transforms.ToPILImage(),     \n          torchvision.transforms.RandomGrayscale(p=0.1),           \n          torchvision.transforms.ToTensor(),\n          \n          ])\n  else:\n    if NORMALIZE:\n          transforms = torchvision.transforms.Compose([\n          torchvision.transforms.ToTensor(),\n          torchvision.transforms.Normalize(RESNET_MEAN, RESNET_STD),\n          ])\n    else:\n      transforms = torchvision.transforms.Compose([\n          torchvision.transforms.ToTensor(),\n          \n          ])\n       \n  return transforms","metadata":{"id":"Wbdmm-G4SmgH","executionInfo":{"status":"ok","timestamp":1640900771244,"user_tz":-330,"elapsed":7,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-21T15:43:32.631228Z","iopub.execute_input":"2022-06-21T15:43:32.631961Z","iopub.status.idle":"2022-06-21T15:43:32.641813Z","shell.execute_reply.started":"2022-06-21T15:43:32.631927Z","shell.execute_reply":"2022-06-21T15:43:32.640199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_type_dict = {\"astro\": 1, \"cort\": 2, \"shsy5y\": 3}\n\nclass CellDataset(Dataset):\n    def __init__(self, image_dir, df, transforms=None, resize=False):\n        self.transforms = transforms\n        self.image_dir = image_dir\n        self.df = df\n        \n        self.should_resize = resize is not False\n        if self.should_resize:\n            self.height = int(HEIGHT * resize)\n            self.width = int(WIDTH * resize)\n            print(\"image size used:\", self.height, self.width)\n        else:\n            self.height = HEIGHT\n            self.width = WIDTH\n        \n        self.image_info = collections.defaultdict(dict)\n        temp_df = self.df.groupby([\"id\", \"cell_type\"])['annotation'].agg(lambda x: list(x)).reset_index()\n        for index, row in temp_df.iterrows():\n            self.image_info[index] = {\n                    'image_id': row['id'],\n                    'image_path': os.path.join(self.image_dir, row['id'] + '.png'),\n                    'annotations': list(row[\"annotation\"]),\n                    'cell_type': cell_type_dict[row[\"cell_type\"]]\n                    }\n            \n    def get_box(self, a_mask):\n        ''' Get the bounding box of a given mask '''\n        pos = np.where(a_mask)\n        xmin = np.min(pos[1])\n        xmax = np.max(pos[1])\n        ymin = np.min(pos[0])\n        ymax = np.max(pos[0])\n        return [xmin, ymin, xmax, ymax]\n\n    def __getitem__(self, idx):\n        ''' Get the image and the target'''\n        \n        img_path = self.image_info[idx][\"image_path\"]\n        img = cv2.imread(img_path, cv2.IMREAD_COLOR)\n        \n        if self.should_resize:\n            img = cv2.resize(img, (self.width, self.height))\n\n        info = self.image_info[idx]\n\n        n_objects = len(info['annotations'])\n        masks = np.zeros((len(info['annotations']), self.height, self.width), dtype=np.uint8)\n        boxes = []\n        labels = []\n        for i, annotation in enumerate(info['annotations']):\n            a_mask = rle_decode(annotation, (HEIGHT, WIDTH))\n            \n            if self.should_resize:\n                a_mask = cv2.resize(a_mask, (self.width, self.height))\n            \n            a_mask = np.array(a_mask) > 0\n            masks[i, :, :] = a_mask\n            \n            boxes.append(self.get_box(a_mask))\n\n        # labels\n        labels = [int(info[\"cell_type\"]) for _ in range(n_objects)]\n        #labels = [1 for _ in range(n_objects)]\n        \n        \n        boxes = torch.as_tensor(boxes, dtype=torch.float32)\n        labels = torch.as_tensor(labels, dtype=torch.int64)\n        masks = torch.as_tensor(masks, dtype=torch.uint8)\n\n        image_id = torch.tensor([idx])\n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n        iscrowd = torch.zeros((n_objects,), dtype=torch.int64)\n\n        # This is the required target for the Mask R-CNN\n        target = {\n            'boxes': boxes,\n            'labels': labels,\n            'masks': masks,\n            'image_id': image_id,\n            'area': area,\n            'iscrowd': iscrowd\n        }\n\n        if self.transforms is not None:\n            img = self.transforms(img)\n\n        return img, target\n\n    def __len__(self):\n        return len(self.image_info)","metadata":{"id":"FZk11wuRSv5P","executionInfo":{"status":"ok","timestamp":1640900772393,"user_tz":-330,"elapsed":3,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-21T15:43:32.859650Z","iopub.execute_input":"2022-06-21T15:43:32.860275Z","iopub.status.idle":"2022-06-21T15:43:32.883718Z","shell.execute_reply.started":"2022-06-21T15:43:32.860238Z","shell.execute_reply":"2022-06-21T15:43:32.882588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base = pd.read_csv(TRAIN_CSV, nrows=5000 if TEST else None)\ndf_images = df_base.groupby([\"id\", \"cell_type\"]).agg({'annotation': 'count'}).sort_values(\"annotation\", ascending=False).reset_index()","metadata":{"id":"AGeBKAgQS5aX","executionInfo":{"status":"ok","timestamp":1640900774199,"user_tz":-330,"elapsed":1808,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-21T15:43:33.077037Z","iopub.execute_input":"2022-06-21T15:43:33.077416Z","iopub.status.idle":"2022-06-21T15:43:33.416414Z","shell.execute_reply.started":"2022-06-21T15:43:33.077386Z","shell.execute_reply":"2022-06-21T15:43:33.415358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_images.groupby(\"cell_type\").annotation.describe().astype(int)","metadata":{"id":"aALU082uTVAv","outputId":"c80aac61-96d2-492e-8372-692e44f75b32","executionInfo":{"status":"ok","timestamp":1640900774200,"user_tz":-330,"elapsed":53,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-21T15:43:33.823162Z","iopub.execute_input":"2022-06-21T15:43:33.823523Z","iopub.status.idle":"2022-06-21T15:43:33.846746Z","shell.execute_reply.started":"2022-06-21T15:43:33.823494Z","shell.execute_reply":"2022-06-21T15:43:33.845643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We used this as a reference to fill BOX_DETECTIONS_PER_IMG=140\ndf_images[['annotation']].describe().astype(int)","metadata":{"id":"UwYP6l6ETeeA","outputId":"054aeb46-ffdf-48d3-cfb0-8a9f65d08ed4","executionInfo":{"status":"ok","timestamp":1640900774200,"user_tz":-330,"elapsed":46,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-21T15:43:34.055686Z","iopub.execute_input":"2022-06-21T15:43:34.056713Z","iopub.status.idle":"2022-06-21T15:43:34.072216Z","shell.execute_reply.started":"2022-06-21T15:43:34.056658Z","shell.execute_reply":"2022-06-21T15:43:34.071265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_images_train = df_images.iloc[0:50]\ndf_images_val = df_images.iloc[50::].sample(n=50, random_state=1)\n\ndf_train = df_base[df_base['id'].isin(df_images_train['id'])]\ndf_val = df_base[df_base['id'].isin(df_images_val['id'])]\n\nprint(f\"Images in train set:           {df_train.id.nunique()}\")\nprint(f\"Annotations in train set:      {len(df_train)}\")\nprint(f\"Images in validation set:      {len(df_images_val)}\")\nprint(f\"Annotations in validation set: {len(df_val)}\")","metadata":{"id":"Qx7SfPMYReCS","outputId":"eee77407-c957-4009-ad35-895025642a9f","executionInfo":{"status":"ok","timestamp":1640900790694,"user_tz":-330,"elapsed":563,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-21T15:43:34.663671Z","iopub.execute_input":"2022-06-21T15:43:34.664752Z","iopub.status.idle":"2022-06-21T15:43:34.687154Z","shell.execute_reply.started":"2022-06-21T15:43:34.664703Z","shell.execute_reply":"2022-06-21T15:43:34.685917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_images_train.cell_type.value_counts()","metadata":{"id":"Czrrei7B7SqH","outputId":"c94990b6-eacb-497a-be4e-87c9d4847557","executionInfo":{"status":"ok","timestamp":1640900793003,"user_tz":-330,"elapsed":3,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-21T15:43:34.929749Z","iopub.execute_input":"2022-06-21T15:43:34.930562Z","iopub.status.idle":"2022-06-21T15:43:34.938904Z","shell.execute_reply.started":"2022-06-21T15:43:34.930523Z","shell.execute_reply":"2022-06-21T15:43:34.937672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_train = CellDataset(TRAIN_PATH, df_train, resize=resize_factor, transforms=get_transform(train=True))\ndl_train = DataLoader(ds_train, batch_size=BATCH_SIZE, shuffle=False, pin_memory=True,\n                      num_workers=4, \n                      collate_fn=lambda x: tuple(zip(*x))\n                      )\n\nds_val = CellDataset(TRAIN_PATH, df_val, resize=resize_factor, transforms=get_transform(train=False))\ndl_val = DataLoader(ds_val, batch_size=BATCH_SIZE, shuffle=False, pin_memory=True,\n                    num_workers=4, \n                    collate_fn=lambda x: tuple(zip(*x))\n                    )","metadata":{"id":"M9xkw8DxTtWv","outputId":"b89788df-1baf-4476-e51e-535f30e51990","executionInfo":{"status":"ok","timestamp":1640900795055,"user_tz":-330,"elapsed":20,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-21T15:43:35.268493Z","iopub.execute_input":"2022-06-21T15:43:35.269534Z","iopub.status.idle":"2022-06-21T15:43:35.309692Z","shell.execute_reply.started":"2022-06-21T15:43:35.269476Z","shell.execute_reply":"2022-06-21T15:43:35.308458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(num_classes, model_chkpt=None):\n    \n    if NORMALIZE:\n        model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=True,\n                                                                   pretrained_backbone=True,\n                                                                   box_detections_per_img=BOX_DETECTIONS_PER_IMG,\n                                                                   image_mean=RESNET_MEAN,\n                                                                   image_std=RESNET_STD, \n                                                                   )\n    else:\n        model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=True,\n                                                                   pretrained_backbone=True,\n                                                                   box_detections_per_img=BOX_DETECTIONS_PER_IMG, \n                                                                   )\n\n    # get the number of input features for the classifier\n    in_features = model.roi_heads.box_predictor.cls_score.in_features\n    # replace the pre-trained head with a new one\n    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes+1)\n\n    # now get the number of input features for the mask classifier\n    in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels\n    hidden_layer = 256\n    # and replace the mask predictor with a new one\n    model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask, hidden_layer, num_classes+1)\n    \n    if model_chkpt:\n        model.load_state_dict(torch.load(model_chkpt, map_location=DEVICE))\n    return model","metadata":{"id":"z-VwTytNTzeI","executionInfo":{"status":"ok","timestamp":1640900795057,"user_tz":-330,"elapsed":14,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-21T15:42:32.298735Z","iopub.execute_input":"2022-06-21T15:42:32.299824Z","iopub.status.idle":"2022-06-21T15:42:32.309455Z","shell.execute_reply.started":"2022-06-21T15:42:32.299782Z","shell.execute_reply":"2022-06-21T15:42:32.308311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MOMENTUM = 0.1\nLEARNING_RATE = 0.009\nWEIGHT_DECAY = 0.001\nUSE_SCHEDULER = False","metadata":{"id":"nt3-cUHhrlFd","executionInfo":{"status":"ok","timestamp":1640900795503,"user_tz":-330,"elapsed":4,"user":{"displayName":"vivek joshi","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14701461628611770995"}},"execution":{"iopub.status.busy":"2022-06-21T15:42:39.746939Z","iopub.execute_input":"2022-06-21T15:42:39.747564Z","iopub.status.idle":"2022-06-21T15:42:39.753311Z","shell.execute_reply.started":"2022-06-21T15:42:39.747525Z","shell.execute_reply":"2022-06-21T15:42:39.751918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model(len(cell_type_dict))\nmodel.to(DEVICE)\n\nfor param in model.parameters():\n    param.requires_grad = True\n\nparams = [p for p in model.parameters() if p.requires_grad]\n\noptimizer = torch.optim.SGD(params, lr=LEARNING_RATE,  \n                            momentum=MOMENTUM, \n                            )\n\n# optimizer = torch.optim.AdamW(params, lr=LEARNING_RATE, \n                            # betas=0.9, \n                            # eps=1e-8,\n                            # weight_decay=WEIGHT_DECAY,\n                            # amsgrad = False\n                            # )\n\n# lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1)\nlr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=1, factor=0.5)\n\nn_batches, n_batches_val = len(dl_train), len(dl_val)\n\nvalidation_mask_losses = []\n\nfor epoch in range(1, NUM_EPOCHS + 1):\n    print(f\"Starting epoch {epoch} of {NUM_EPOCHS}\")\n\n    time_start = time.time()\n    loss_accum = 0.0\n    loss_mask_accum = 0.0\n    loss_classifier_accum = 0.0\n    for batch_idx, (images, targets) in enumerate(dl_train, 1):\n    \n        # Predict\n        images = list(image.to(DEVICE) for image in images)\n        targets = [{k: v.to(DEVICE) for k, v in t.items()} for t in targets]\n\n        loss_dict = model(images, targets)\n        loss = sum(loss for loss in loss_dict.values())\n        \n        # Backprop\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n        # Logging\n        loss_mask = loss_dict['loss_mask'].item()\n        loss_accum += loss.item()\n        loss_mask_accum += loss_mask\n        loss_classifier_accum += loss_dict['loss_classifier'].item()\n        \n        if batch_idx % 500 == 0:\n            print(f\"    [Batch {batch_idx:3d} / {n_batches:3d}] Batch train loss: {loss.item():7.3f}. Mask-only loss: {loss_mask:7.3f}.\")\n                        \n    # Train losses\n    train_loss = loss_accum / n_batches\n    train_loss_mask = loss_mask_accum / n_batches\n    train_loss_classifier = loss_classifier_accum / n_batches\n\n    # Validation\n    val_loss_accum = 0\n    val_loss_mask_accum = 0\n    val_loss_classifier_accum = 0\n    \n    with torch.no_grad():\n        for batch_idx, (images, targets) in enumerate(dl_val, 1):\n            images = list(image.to(DEVICE) for image in images)\n            targets = [{k: v.to(DEVICE) for k, v in t.items()} for t in targets]\n            \n            val_loss_dict = model(images, targets)\n            val_batch_loss = sum(loss for loss in val_loss_dict.values())\n            val_loss_accum += val_batch_loss.item()\n            val_loss_mask_accum += val_loss_dict['loss_mask'].item()\n            val_loss_classifier_accum += val_loss_dict['loss_classifier'].item()\n\n    # Validation losses\n    val_loss = val_loss_accum / n_batches_val\n    val_loss_mask = val_loss_mask_accum / n_batches_val\n    val_loss_classifier = val_loss_classifier_accum / n_batches_val\n    elapsed = time.time() - time_start\n\n    validation_mask_losses.append(val_loss_mask)\n\n    if USE_SCHEDULER:\n        lr_scheduler.step(metrics=val_loss)\n\n    torch.save(model.state_dict(), f\"sartorius-e{epoch}.bin\")\n    prefix = f\"[Epoch {epoch:2d} / {NUM_EPOCHS:2d}]\"\n    print(prefix)\n    print(f\"{prefix} Train mask-only loss: {train_loss_mask:7.3f}, classifier loss {train_loss_classifier:7.3f}\")\n    print(f\"{prefix} Val mask-only loss  : {val_loss_mask:7.3f}, classifier loss {val_loss_classifier:7.3f}\")\n    print(prefix)\n    print(f\"{prefix} Train loss: {train_loss:7.3f}. Val loss: {val_loss:7.3f} [{elapsed:.0f} secs]\")\n    print(prefix)\n","metadata":{"id":"YwizxE5kYE74","outputId":"9febfa1a-ea88-4915-80c1-af569cc595e6","execution":{"iopub.status.busy":"2022-06-21T15:19:45.138814Z","iopub.execute_input":"2022-06-21T15:19:45.139642Z","iopub.status.idle":"2022-06-21T15:22:44.625428Z","shell.execute_reply.started":"2022-06-21T15:19:45.139596Z","shell.execute_reply":"2022-06-21T15:22:44.623996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def analyze_train_sample(model, ds_train, sample_index):\n    \n    img, targets = ds_train[sample_index]\n    #print(img.shape)\n    l = np.unique(targets[\"labels\"])\n    ig, ax = plt.subplots(nrows=1, ncols=3, figsize=(20,60), facecolor=\"#fefefe\")\n    ax[0].imshow(img.numpy().transpose((1,2,0)))\n    ax[0].set_title(f\"cell type {l}\")\n    ax[0].axis(\"off\")\n    \n    masks = combine_masks(targets['masks'], 0.5)\n    #plt.imshow(img.numpy().transpose((1,2,0)))\n    ax[1].imshow(masks)\n    ax[1].set_title(f\"Ground truth, {len(targets['masks'])} cells\")\n    ax[1].axis(\"off\")\n    \n    model.eval()\n    with torch.no_grad():\n        model_chk = \"../input/sartorius-data/sartorius.bin\"\n        model.load_state_dict(torch.load(model_chk))\n        preds = model([img.to(DEVICE)])[0]\n    \n    l = pd.Series(preds['labels'].cpu().numpy()).value_counts()\n    lstr = \"\"\n    for i in l.index:\n        lstr += f\"{l[i]}x{i} \"\n    #print(l, l.sort_values().index[-1])\n    #plt.imshow(img.cpu().numpy().transpose((1,2,0)))\n    mask_threshold = mask_threshold_dict[l.sort_values().index[-1]]\n    #print(mask_threshold)\n    pred_masks = combine_masks(get_filtered_masks(preds), mask_threshold)\n    ax[2].imshow(pred_masks)\n    ax[2].set_title(f\"Predictions, labels: {lstr}\")\n    ax[2].axis(\"off\")\n    plt.show() \n    \n    #print(masks.shape, pred_masks.shape)\n    score = iou_map([masks],[pred_masks])\n    print(\"Score:\", score)    \n    \n    \n# NOTE: It puts the model in eval mode!! Revert for re-training\nanalyze_train_sample(model, ds_train, 20)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T15:43:46.066798Z","iopub.execute_input":"2022-06-21T15:43:46.067477Z","iopub.status.idle":"2022-06-21T15:44:01.696802Z","shell.execute_reply.started":"2022-06-21T15:43:46.067440Z","shell.execute_reply":"2022-06-21T15:44:01.695665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{"id":"QAydhxHkZ-V4"}},{"cell_type":"code","source":"class CellTestDataset(Dataset):\n    def __init__(self, image_dir, transforms=None, resize=False):\n        self.transforms = transforms\n        self.image_dir = image_dir\n        self.image_ids = [f[:-4]for f in os.listdir(self.image_dir)]\n        self.should_resize = resize is not False\n        if self.should_resize:\n            self.height = int(HEIGHT * resize)\n            self.width = int(WIDTH * resize)\n            print(\"image size used:\", self.height, self.width)\n            \n    def __getitem__(self, idx):\n        image_id = self.image_ids[idx]\n        image_path = os.path.join(self.image_dir, image_id + '.png')\n        image = cv2.imread(image_path, cv2.IMREAD_COLOR)\n        if self.should_resize:\n            image = cv2.resize(image, (self.width, self.height))\n\n        if self.transforms is not None:\n            image, _ = self.transforms(image=image, target=None)\n        return {'image': image, 'image_id': image_id}\n\n    def __len__(self):\n        return len(self.image_ids)\n    \nclass Compose:\n    def __init__(self, transforms):\n        self.transforms = transforms\n\n    def __call__(self, image, target):\n        for t in self.transforms:\n            image, target = t(image, target)\n        return image, target\n\nclass Normalize:\n    def __call__(self, image, target):\n        image = F.normalize(image, RESNET_MEAN, RESNET_STD)\n        return image, target    \n\nclass ToTensor:\n    def __call__(self, image, target):\n        image = F.to_tensor(image)\n        return image, target\n    \ndef get_transform(train):\n    transforms = [ToTensor()]\n    if NORMALIZE:\n        transforms.append(Normalize())\n        \n    return Compose(transforms)    \n\nds_test = CellTestDataset(TEST_PATH, transforms=get_transform(train=False))\nmodel_chk = \"../input/sartorius-data/sartorius.bin\"\nprint(\"Loading:\", model_chk)\n\nmodel = get_model(len(cell_type_dict))\nmodel.load_state_dict(torch.load(model_chk))\nmodel = model.to(DEVICE)\n\nfor param in model.parameters():\n    param.requires_grad = False\n\nmodel.eval();\n\nsubmission = []\nfor sample in ds_test:\n    img = sample['image']\n    image_id = sample['image_id']\n    with torch.no_grad():\n        result = model([img.to(DEVICE)])[0]\n    \n    previous_masks = []\n    for i, mask in enumerate(result[\"masks\"]):\n\n        # Filter-out low-scoring results.\n        score = result[\"scores\"][i].cpu().item()\n        label = result[\"labels\"][i].cpu().item()\n        if score > min_score_dict[label]:\n            mask = mask.cpu().numpy()\n            # Keep only highly likely pixels\n            binary_mask = mask > mask_threshold_dict[label]\n            binary_mask = remove_overlapping_pixels(binary_mask, previous_masks)\n            previous_masks.append(binary_mask)\n            rle = rle_encoding(binary_mask)\n            submission.append((image_id, rle))\n\n    # Add empty prediction if no RLE was generated for this image\n    all_images_ids = [image_id for image_id, rle in submission]\n    if image_id not in all_images_ids:\n        submission.append((image_id, \"\"))\n\ndf_sub = pd.DataFrame(submission, columns=['id', 'predicted'])\ndf_sub.to_csv(\"submission.csv\", index=False)\ndf_sub.head()","metadata":{"id":"J2djAzAVaBiQ","execution":{"iopub.status.busy":"2022-06-21T15:33:58.939495Z","iopub.execute_input":"2022-06-21T15:33:58.940023Z","iopub.status.idle":"2022-06-21T15:34:08.452823Z","shell.execute_reply.started":"2022-06-21T15:33:58.939989Z","shell.execute_reply":"2022-06-21T15:34:08.451841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}