{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":30201,"databundleVersionId":2750748,"sourceType":"competition"},{"sourceId":2714594,"sourceType":"datasetVersion","datasetId":1654115},{"sourceId":2786089,"sourceType":"datasetVersion","datasetId":1701116},{"sourceId":2807989,"sourceType":"datasetVersion","datasetId":1703642}],"dockerImageVersionId":30140,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# [Sartorius - Cell Instance Segmentation](https://www.kaggle.com/c/petfinder-pawpularity-score)\n> Detect single neuronal cells in microscopy images\n\n","metadata":{}},{"cell_type":"markdown","source":"# ā½ Goal\nš The purpose of this notebook is to show how to achieve Good score even using **UNet**. \n\nš Even though the competition is about **Instance Segmentation** we can use **UNet** do **Semantic Segmentation** and then convert them to individual **Instances**.\n\nš Finally, we can use **UNet** with **Mask-RCNN** for Ensemble to further boost our score.\n\n![](\"https://i.stack.imgur.com/MEB9F.png\")
","metadata":{}},{"cell_type":"markdown","source":"# š© Version Info:\n* `v10`: aggregate `tta` masks first\n* `v7`: test-time-augmentation added","metadata":{}},{"cell_type":"markdown","source":"# š Notebooks\nš **UNet**:\n* Train: [[PyTorch] Sartorius: UNet Strikes Back [Train] š„](https://www.kaggle.com/awsaf49/pytorch-sartorius-unet-strikes-back-train/edit)\n* Infer: [[PyTorch] Sartorius: UNet Strikes Back [Infer] š„](https://www.kaggle.com/awsaf49/pytorch-sartorius-unet-strikes-back-infer/edit)\n\nš **Mask-RCNN**:\n* Train: [Sartorius: MMDetection [Train]](https://www.kaggle.com/awsaf49/sartorius-mmdetection-train)\n* Infer: [Sartorius: MMDetection [Infer]](https://www.kaggle.com/awsaf49/sartorius-mmdetection-infer)","metadata":{}},{"cell_type":"markdown","source":"## Please Upvote if you Find this Useful :)","metadata":{}},{"cell_type":"markdown","source":"# š Install Libraries","metadata":{}},{"cell_type":"code","source":"!pip install -q ../input/pytorch-segmentation-models-lib/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4\n!pip install -q ../input/pytorch-segmentation-models-lib/efficientnet_pytorch-0.6.3/efficientnet_pytorch-0.6.3\n!pip install -q ../input/pytorch-segmentation-models-lib/timm-0.4.12-py3-none-any.whl\n!pip install -q ../input/pytorch-segmentation-models-lib/segmentation_models_pytorch-0.2.0-py3-none-any.whl","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-10T11:53:51.166435Z","iopub.execute_input":"2021-11-10T11:53:51.166817Z","iopub.status.idle":"2021-11-10T11:55:46.053551Z","shell.execute_reply.started":"2021-11-10T11:53:51.166678Z","shell.execute_reply":"2021-11-10T11:55:46.052501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# š Import Libraries ","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\npd.options.plotting.backend = \"plotly\"\nimport random\nfrom glob import glob\nimport os, shutil\nfrom tqdm import tqdm\ntqdm.pandas()\nimport time\nimport copy\nimport joblib\nfrom collections import defaultdict\nimport gc\nfrom IPython import display as ipd\n\n# visualization\nimport cv2\nimport matplotlib.pyplot as plt\n\n# Sklearn\nfrom sklearn.model_selection import StratifiedKFold, KFold\n\n# PyTorch \nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda import amp\nimport torch.nn.functional as F\n\nimport timm\n\n# Albumentations for augmentations\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# For colored terminal text\nfrom colorama import Fore, Back, Style\nc_ = Fore.GREEN\nsr_ = Style.RESET_ALL\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# For descriptive error messages\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-10T11:55:46.055903Z","iopub.execute_input":"2021-11-10T11:55:46.056188Z","iopub.status.idle":"2021-11-10T11:55:54.131611Z","shell.execute_reply.started":"2021-11-10T11:55:46.056151Z","shell.execute_reply":"2021-11-10T11:55:54.130804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# āļø Configuration ","metadata":{}},{"cell_type":"code","source":"class CFG:\n seed = 42\n exp_name = 'Unet-resnet34-512x512'\n model_name = 'Unet'\n backbone = 'efficientnet-b2'\n train_bs = 24\n valid_bs = 48\n img_size = [512, 512]\n epochs = 50\n lr = 5e-3\n scheduler = 'CosineAnnealingLR'\n min_lr = 1e-6\n T_max = int(100*6*1.5)\n T_0 = 25\n warmup_epochs = 0\n wd = 1e-6\n n_accumulate = 32//train_bs\n n_fold = 10\n num_classes = 1\n device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n ttas = [0, 1, 2, 3, 4, 5]\n competition = 'sartorius'\n _wandb_kernel = 'awsaf49'","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-10T11:56:43.377998Z","iopub.execute_input":"2021-11-10T11:56:43.378262Z","iopub.status.idle":"2021-11-10T11:56:43.386386Z","shell.execute_reply.started":"2021-11-10T11:56:43.378234Z","shell.execute_reply":"2021-11-10T11:56:43.385636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ā Reproducibility","metadata":{}},{"cell_type":"code","source":"def set_seed(seed = 42):\n '''Sets the seed of the entire notebook so results are the same every time we run.\n This is for REPRODUCIBILITY.'''\n np.random.seed(seed)\n random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n # When running on the CuDNN backend, two further options must be set\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\n # Set a fixed value for the hash seed\n os.environ['PYTHONHASHSEED'] = str(seed)\n print('> SEEDING DONE')\n \nset_seed(CFG.seed)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-10T11:55:54.183333Z","iopub.execute_input":"2021-11-10T11:55:54.184074Z","iopub.status.idle":"2021-11-10T11:55:54.199421Z","shell.execute_reply.started":"2021-11-10T11:55:54.184016Z","shell.execute_reply":"2021-11-10T11:55:54.198446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# š Meta Data","metadata":{}},{"cell_type":"code","source":"BASE_PATH = '/kaggle/input/sartorius-cell-instance-segmentation'\nBASE_PATH2 = '/kaggle/input/sartorius-binary-mask-dataset'\nCKPT_DIR = '/kaggle/input/pytorch-sartorius-unet-strikes-back-ds'","metadata":{"execution":{"iopub.status.busy":"2021-11-10T11:55:54.200941Z","iopub.execute_input":"2021-11-10T11:55:54.202Z","iopub.status.idle":"2021-11-10T11:55:54.206231Z","shell.execute_reply.started":"2021-11-10T11:55:54.201959Z","shell.execute_reply":"2021-11-10T11:55:54.205193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Data\ndf = pd.read_csv(f'{BASE_PATH}/train.csv')\ndf['image_path'] = BASE_PATH + '/train/' + df['id'] + '.png'\ntmp_df = df.drop_duplicates(subset=[\"id\", \"image_path\"]).reset_index(drop=True)\ntmp_df[\"annotation\"] = df.groupby(\"id\")[\"annotation\"].agg(list).reset_index(drop=True)\ndf = tmp_df.copy()\ndf['mask_path'] = BASE_PATH2 + '/' + df['id'] + '.npy'\ndisplay(df.head(2))\n\n# Test Data\ntest_df = pd.DataFrame(glob(BASE_PATH+'/test/*'), columns=['image_path'])\ntest_df['id'] = test_df.image_path.map(lambda x: x.split('/')[-1].split('.')[0])\n\ndisplay(test_df.head(2))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-10T11:55:54.207675Z","iopub.execute_input":"2021-11-10T11:55:54.208463Z","iopub.status.idle":"2021-11-10T11:55:54.815748Z","shell.execute_reply.started":"2021-11-10T11:55:54.208426Z","shell.execute_reply":"2021-11-10T11:55:54.815027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# š Dataset","metadata":{}},{"cell_type":"code","source":"class BuildDataset(torch.utils.data.Dataset):\n def __init__(self, df, transforms=None):\n self.df = df\n self.img_paths = df['image_path'].values\n try: # if there is no mask then only send images --> test data\n self.msk_paths = df['mask_path'].values\n except:\n self.msk_paths = None\n self.transforms = transforms\n \n def __len__(self):\n return len(self.df)\n \n def __getitem__(self, index):\n img_path = self.img_paths[index]\n img = cv2.imread(img_path)\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n if self.msk_paths is not None:\n msk_path = self.msk_paths[index]\n msk = np.load(msk_path)\n if self.transforms:\n data = self.transforms(image=img, mask=msk)\n img = data['image']\n msk = data['mask']\n msk = np.expand_dims(msk, axis=0) # output_shape: (batch_size, 1, img_size, img_size)\n return img, msk\n else:\n if self.transforms:\n data = self.transforms(image=img)\n img = data['image']\n return img, img_path","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-10T11:55:54.817334Z","iopub.execute_input":"2021-11-10T11:55:54.817593Z","iopub.status.idle":"2021-11-10T11:55:54.827043Z","shell.execute_reply.started":"2021-11-10T11:55:54.817559Z","shell.execute_reply":"2021-11-10T11:55:54.826042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# š Augmentations","metadata":{}},{"cell_type":"code","source":"data_transforms = {\n \"train\": A.Compose([\n A.Resize(*CFG.img_size),\n# A.Normalize(\n# mean=[0.485, 0.456, 0.406], \n# std=[0.229, 0.224, 0.225], \n# max_pixel_value=255.0, \n# p=1.0,\n# ),\n A.CLAHE(p=0.35),\n A.ColorJitter(p=0.5),\n A.HorizontalFlip(p=0.5),\n A.VerticalFlip(p=0.5),\n A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=90, p=0.5),\n A.OneOf([\n A.GridDistortion(num_steps=5, distort_limit=0.05, p=1.0),\n# A.OpticalDistortion(distort_limit=0.05, shift_limit=0.05, p=1.0),\n A.ElasticTransform(alpha=1, sigma=50, alpha_affine=50, p=1.0)\n ], p=0.25),\n A.CoarseDropout(max_holes=8, max_height=CFG.img_size[0]//20, max_width=CFG.img_size[1]//20,\n min_holes=5, fill_value=0, mask_fill_value=0, p=0.5),\n ToTensorV2()], p=1.0),\n \n \"valid\": A.Compose([\n A.Resize(*CFG.img_size),\n# A.Normalize(\n# mean=[0.485, 0.456, 0.406], \n# std=[0.229, 0.224, 0.225], \n# max_pixel_value=255.0, \n# p=1.0\n# ),\n ToTensorV2()], p=1.0)\n}","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-10T11:55:54.828667Z","iopub.execute_input":"2021-11-10T11:55:54.828981Z","iopub.status.idle":"2021-11-10T11:55:54.841858Z","shell.execute_reply.started":"2021-11-10T11:55:54.828942Z","shell.execute_reply":"2021-11-10T11:55:54.840936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# š° DataLoader","metadata":{}},{"cell_type":"code","source":"test_dataset = BuildDataset(test_df, transforms=data_transforms['valid'])\ntest_loader = DataLoader(test_dataset, batch_size=3, \n num_workers=4, shuffle=False, pin_memory=True)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-10T11:55:54.843849Z","iopub.execute_input":"2021-11-10T11:55:54.844749Z","iopub.status.idle":"2021-11-10T11:55:54.852754Z","shell.execute_reply.started":"2021-11-10T11:55:54.844709Z","shell.execute_reply":"2021-11-10T11:55:54.851717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs, img_paths = next(iter(test_loader))\nimgs = imgs.permute((0, 2, 3, 1))\nimgs.size()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-10T11:55:54.856918Z","iopub.execute_input":"2021-11-10T11:55:54.857702Z","iopub.status.idle":"2021-11-10T11:55:59.606385Z","shell.execute_reply.started":"2021-11-10T11:55:54.857667Z","shell.execute_reply":"2021-11-10T11:55:59.605634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# š¦ Model\n","metadata":{}},{"cell_type":"markdown","source":"## UNet\n\n![](\"https://miro.medium.com/max/875/1*f7YOaE4TWubwaFF7Z1fzNw.png\")
\n\nš **Pros**:\n* Performs well even with smaller data\n* Can be used with `imagenet` pretrain models\n\nš **Cons**:\n* Struggles with **edge** cases\n* Semantic Difference in **Skip Connection**","metadata":{}},{"cell_type":"code","source":"import segmentation_models_pytorch as smp\n\ndef build_model():\n model = smp.Unet(\n encoder_name=CFG.backbone, # choose encoder, e.g. mobilenet_v2 or efficientnet-b7\n encoder_weights=None, # use `imagenet` pre-trained weights for encoder initialization\n in_channels=3, # model input channels (1 for gray-scale images, 3 for RGB, etc.)\n classes=CFG.num_classes, # model output channels (number of classes in your dataset)\n activation=None,\n )\n model.to(CFG.device)\n return model\n\ndef load_model(path):\n model = build_model()\n model.load_state_dict(torch.load(path))\n model.eval()\n return model","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-10T11:55:59.607831Z","iopub.execute_input":"2021-11-10T11:55:59.608317Z","iopub.status.idle":"2021-11-10T11:56:00.916892Z","shell.execute_reply.started":"2021-11-10T11:55:59.60828Z","shell.execute_reply":"2021-11-10T11:56:00.91602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test\nimg = torch.randn(1, 3, *CFG.img_size).to(CFG.device)\nimg = (img - img.min())/(img.max() - img.min())\nmodel = build_model()\n_ = model(img)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-10T11:56:00.918438Z","iopub.execute_input":"2021-11-10T11:56:00.918698Z","iopub.status.idle":"2021-11-10T11:56:06.283272Z","shell.execute_reply.started":"2021-11-10T11:56:00.918664Z","shell.execute_reply":"2021-11-10T11:56:06.28247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# šØ Helper","metadata":{}},{"cell_type":"code","source":"import cupy as cp\nimport skimage.morphology \n\ndef ins2rle(ins):\n '''\n img: numpy array, 1 - mask, 0 - background\n Returns run length as string formated\n '''\n ins = cp.array(ins)\n pixels = ins.flatten()\n pad = cp.array([0])\n pixels = cp.concatenate([pad, pixels, pad])\n runs = cp.where(pixels[1:] != pixels[:-1])[0] + 1\n runs[1::2] -= runs[::2]\n return ' '.join(str(x) for x in runs)\n\ndef mask2rle(mask, cutoff=0.5, min_object_size=1.0):\n \"\"\" Return run length encoding of mask. \n ref: https://www.kaggle.com/raoulma/nuclei-dsb-2018-tensorflow-u-net-score-0-352\n \"\"\"\n # segment image and label different objects\n lab_mask = skimage.morphology.label(mask > cutoff)\n \n # Keep only objects that are large enough.\n (mask_labels, mask_sizes) = np.unique(lab_mask, return_counts=True)\n if (mask_sizes < min_object_size).any():\n mask_labels = mask_labels[mask_sizes < min_object_size]\n for n in mask_labels:\n lab_mask[lab_mask == n] = 0\n lab_mask = skimage.morphology.label(lab_mask > cutoff) \n \n # Loop over each object excluding the background labeled by 0.\n for i in range(1, lab_mask.max() + 1):\n yield ins2rle(lab_mask == i)\n \ndef aug(img, axis=0):\n if axis == 1:\n return torch.flip(img,dims=(1,))\n elif axis == 2:\n return torch.flip(img,dims=(2,))\n elif axis == 3:\n return torch.flip(img,dims=(1,2))\n elif axis == 4:\n return torch.rot90(img, k=1, dims=(1,2))\n elif axis == 5:\n return torch.rot90(img, k=1, dims=(2,1))\n else:\n return img\n \ndef reverse_aug(img, axis=0):\n if axis == 1:\n return torch.flip(img,dims=(1,))\n elif axis == 2:\n return torch.flip(img,dims=(2,))\n elif axis == 3:\n return torch.flip(img,dims=(1,2))\n elif axis == 4:\n return torch.rot90(img, k=1, dims=(2,1))\n elif axis == 5:\n return torch.rot90(img, k=1, dims=(1,2))\n else:\n return img\n \ndef get_aug_img(img, ttas=CFG.ttas):\n \"\"\"\n Args:\n img : image\n ttas : tta modes ex [0, 1]\n Return:\n augmentated images shape (num_tta, dim0, dim1, channel)\n \"\"\"\n if len(ttas)==0:\n return img.unsqueeze(0)\n aug_img = []\n for idx, tta_mode in enumerate(ttas):\n aug_img.append(aug(img, axis=tta_mode))\n aug_img = torch.stack(aug_img, dim=0)\n return aug_img\n\ndef fix_aug_img(aug_pred, ttas=CFG.ttas):\n \"\"\"\n Args:\n aug_pred : prediction of augmented images\n ttas : tta modes ex [0, 1]\n Return:\n final image after ensemble\n \"\"\"\n if len(ttas)==0:\n return aug_pred\n fixed_pred = []\n for idx, tta_mode in enumerate(ttas):\n fixed_pred.append(reverse_aug(aug_pred[idx], axis=tta_mode))\n fixed_pred = torch.stack(fixed_pred, dim=0)\n fixed_pred = torch.mean(fixed_pred, dim=0)\n return fixed_pred","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-10T12:20:43.264992Z","iopub.execute_input":"2021-11-10T12:20:43.265255Z","iopub.status.idle":"2021-11-10T12:20:43.284908Z","shell.execute_reply.started":"2021-11-10T12:20:43.265224Z","shell.execute_reply":"2021-11-10T12:20:43.284074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# š Inference","metadata":{}},{"cell_type":"code","source":"@torch.no_grad()\ndef infer(model_paths, test_loader, num_log=3):\n pred_strings = []; pred_paths = []; msks = []; imgs = [];\n for idx, (img, img_path) in enumerate(tqdm(test_loader, total=len(test_loader), desc='Infer ')):\n img = img.to(CFG.device, dtype=torch.float).squeeze()\n img = get_aug_img(img, ttas=CFG.ttas)\n msk = []\n for path in model_paths:\n model = load_model(path)\n out = model(img).squeeze(0) # removing batch axis\n out = fix_aug_img(out,ttas=CFG.ttas)\n out = nn.Sigmoid()(out).squeeze(0) # removing channel axis\n msk.append(out)\n msk = torch.mean(torch.stack(msk, dim=0), dim=0)\n msk = F.interpolate(msk[None,None,], size=(520, 704), mode='nearest')[0,0]\n msk = msk.cpu().detach().numpy()\n img = F.interpolate(img[0:1,], size=(520, 704), mode='nearest')[0] # first dim is image w/o aug\n img = img.squeeze().permute((1,2,0)).cpu().detach().numpy()\n if idx