{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Choosing transforms for 🦠 Sartorius -  🦠 Cell Instance Segmentation 🦠 \n\nIts generally a practice to go for robust augmentation technqiues, and using as much as possible. \nIn this experiment, I am aiming to choose augmentation based on what might help model.\n\n\nIn this competition, since we have limited amount of trainig data, using augmentation widely might be key to winning this competition.","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\n\nimport random\nimport torch\nfrom pytorch_lightning import LightningDataModule\nfrom torch.utils.data import Dataset\nfrom torch.utils.data.dataloader import DataLoader\nimport albumentations as A\nfrom albumentations import Normalize\nfrom albumentations.pytorch import ToTensorV2","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:41:45.892461Z","iopub.execute_input":"2021-10-17T14:41:45.893153Z","iopub.status.idle":"2021-10-17T14:41:51.522363Z","shell.execute_reply.started":"2021-10-17T14:41:45.893041Z","shell.execute_reply":"2021-10-17T14:41:51.52154Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Crtieria to choose augmentation\n\n1. Augmentations doesn't harm the image. \n2. Augmentation helps model by creating potential edge cases missing in training dataset, but also, doesn't do a over-kill to produce cases that model might never see.\n3. Augmentation works fine for both Image and Mask.","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/sartorius-cell-instance-segmentation/train.csv\")\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:41:51.52392Z","iopub.execute_input":"2021-10-17T14:41:51.524128Z","iopub.status.idle":"2021-10-17T14:41:52.14424Z","shell.execute_reply.started":"2021-10-17T14:41:51.524103Z","shell.execute_reply":"2021-10-17T14:41:52.143724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ref: https://www.kaggle.com/inversion/run-length-decoding-quick-start\ndef rle_decode(mask_rle, shape, color=1):\n    '''mask_rle: run-length as string formated (start length)\n    shape: (height, width, channels) of array to return \n    color: color for the mask\n    Returns numpy array (mask)\n    '''\n    s = mask_rle.split()\n    \n    starts = list(map(lambda x: int(x) - 1, s[0::2]))\n    lengths = list(map(int, s[1::2]))\n    ends = [x + y for x, y in zip(starts, lengths)]\n    \n    img = np.zeros((shape[0] * shape[1], shape[2]), dtype=np.float32)\n            \n    for start, end in zip(starts, ends):\n        img[start : end] = color\n    \n    return img.reshape(shape)\n\n\nclass Segmentation_ImageDataset(Dataset):\n    def __init__(self, dataframe, path_column='file_path', label_column='target',\n                 transform=None , function = None) -> None:\n        \"\"\"dataframe - pandas dataframe with 2 columns - file_path and target.\"\"\"\n        super().__init__()\n        self.df = dataframe\n        self.transform = transform\n        self.paths = self.df[path_column]\n        self.function = function\n\n        #Classes - Unique pixel values in masks\n\n    def __getitem__(self, index):\n        \n        image_id = df_train['id'][index]\n        labels = df_train[df_train[\"id\"] == image_id][\"annotation\"].tolist()\n        \n        colors = False #True\n\n        if colors:\n            mask = np.zeros((520, 704, 3))\n            for label in labels:\n                mask += rle_decode(label, shape=(520, 704, 3), color=np.random.rand(3))\n        else:\n            mask = np.zeros((520, 704, 1))\n            for label in labels:\n                mask += rle_decode(label, shape=(520, 704, 1))\n        mask = mask.clip(0, 1)\n        \n\n        image = cv2.imread(f\"../input/sartorius-cell-instance-segmentation/train/{image_id}.png\")\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if self.function != None:\n            image = self.function(image)\n\n\n        if self.transform is not None:\n            tranformed = self.transform(image=image , mask=mask)\n            tranformed_image = tranformed['image']\n            tranformed_mask = tranformed['mask']\n\n        return image , mask , tranformed_image ,  tranformed_mask\n\n    def __len__(self):\n        return len(self.paths)\n\n\ndef image_batch_comparison_visualizer2d(dataset, count=24, subplot=(6, 4),  unnormalize = True ,\n                               cmap='gist_gray', random_img=True,\n                             figheight=10, figwidth=20   , alpha = 0.1 , title = True):\n    fig = plt.figure(figsize=(figwidth, figheight))\n\n    #dataset.df = shuffle(dataset.df)\n    images_shown = 0\n\n    while images_shown < count-2:\n        if random_img == True:\n            index = random.randint(0, dataset.df.shape[0] -1  )\n\n        image , mask , transformed_image ,  transformed_mask = dataset[index]\n        image = image #.numpy().transpose((1, 2, 0))\n        transformed_image = transformed_image.numpy().transpose((1, 2, 0))\n        mask = mask #.numpy() #.transpose((1, 2, 0))\n        transformed_mask = transformed_mask.numpy() #.transpose((1, 2, 0))\n\n\n        if unnormalize == True:\n            mean = np.array([0.485, 0.456, 0.406])\n            std = np.array([0.229, 0.224, 0.225])\n            transformed_image = std * transformed_image + mean\n            transformed_image = np.clip(transformed_image, 0, 1)\n            \n        \n        \n        plt.subplot(subplot[0], subplot[1], images_shown + 1)\n        plt.axis('off')\n        plt.tight_layout()\n        plt.imshow(image )\n        if title == True: \n            plt.title('Image')\n\n        plt.subplot(subplot[0], subplot[1], images_shown + 2)\n        plt.axis('off')\n        plt.tight_layout()\n        plt.imshow(image )\n        plt.imshow(mask, alpha=alpha)\n        if title == True: \n            plt.title('Image + Mask Overlap')\n\n        plt.subplot(subplot[0], subplot[1], images_shown + 3)        \n        plt.axis('off')\n        plt.tight_layout()\n        plt.imshow(transformed_image , cmap=cmap)\n        if title == True: \n            plt.title('Tansformed Image')\n        \n        plt.subplot(subplot[0], subplot[1], images_shown + 4)        \n        plt.axis('off')\n        plt.tight_layout()\n        plt.imshow(transformed_image , cmap=cmap)\n        plt.imshow(transformed_mask , alpha = alpha)\n        if title == True: \n            plt.title('Tansformed Image + Mask  Overlap')\n\n        images_shown += 4\n    plt.show()\n    \ndef original_image_batch_visualizer2d(dataset, plot_mask =True ,  count=24, subplot=(6, 4),  unnormalize = True ,\n                               cmap='gist_gray', random_img=True,\n                             figheight=10, figwidth=20   , alpha = 0.1):\n    fig = plt.figure(figsize=(figwidth, figheight))\n    images_shown = 1\n\n    while images_shown < count + 1 :\n        if random_img == True:\n            index = random.randint(0, dataset.df.shape[0] -1  )\n\n        image , mask , transformed_image ,  transformed_mask = dataset[index]\n        image = image #.numpy().transpose((1, 2, 0))\n        mask = mask #.numpy() #.transpose((1, 2, 0))\n        \n        \n        plt.subplot(subplot[0], subplot[1], images_shown )\n        plt.axis('off')\n        plt.tight_layout()\n        plt.imshow(image )\n        if plot_mask == True:\n            plt.imshow(mask, alpha=alpha)\n\n        images_shown += 1\n    plt.show()\n    ","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:41:52.145284Z","iopub.execute_input":"2021-10-17T14:41:52.145503Z","iopub.status.idle":"2021-10-17T14:41:52.162975Z","shell.execute_reply.started":"2021-10-17T14:41:52.145478Z","shell.execute_reply":"2021-10-17T14:41:52.162307Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lets look at the Images\n\nAim is to look at enough samples, with and without masks, and get a fair idea of the distribution.\n\nFor this, we wont use any augmentation, only the basc -Resize and Normalize.","metadata":{}},{"cell_type":"code","source":"count = 100\nsubplot = (10,10)\nwidth = 18 * subplot[0]/  subplot[1] + 2\nheight = 15 * subplot[1]/  subplot[0]\n\nimage_size = 224\nimagenet_stats = {\"mean\": [0.485, 0.456, 0.406],\n                      \"std\": [0.229, 0.224, 0.225]}\n    \nvalid_tfms_albu = A.Compose(\n        [A.Resize(image_size, image_size),\n         Normalize(mean= imagenet_stats['mean'] ,std= imagenet_stats['std'] ,),\n         ToTensorV2() ,])\n\ntrain_dataset = Segmentation_ImageDataset(dataframe= df_train , path_column= 'id' , transform=valid_tfms_albu)\n\noriginal_image_batch_visualizer2d( train_dataset,plot_mask =False ,  count= count, subplot= subplot,   unnormalize =  True,\n                        figheight=height, figwidth=width , alpha = 0.15)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:42:46.484446Z","iopub.execute_input":"2021-10-17T14:42:46.484743Z","iopub.status.idle":"2021-10-17T14:43:21.977722Z","shell.execute_reply.started":"2021-10-17T14:42:46.484711Z","shell.execute_reply":"2021-10-17T14:43:21.976612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_image_batch_visualizer2d( train_dataset,plot_mask =True ,  count= count, subplot= subplot,   unnormalize =  True,\n                        figheight=height, figwidth=width , alpha = 0.25)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:43:21.97929Z","iopub.execute_input":"2021-10-17T14:43:21.979682Z","iopub.status.idle":"2021-10-17T14:44:03.700179Z","shell.execute_reply.started":"2021-10-17T14:43:21.979534Z","shell.execute_reply":"2021-10-17T14:44:03.699164Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Looking at original images, we observe, there is little to no variation in \n1. Hue\n2. Brightness\n3. Contrast\n\n### Looking into Masks overlap, we observe - \n1. Different levels of zooms\n2. Original data also has several cells(on margins) that are cropped partially, so artifically RandomResizedCrop might not harm our data.\n3. Cells have no orientation what so ever, so Rotation is going to helpful.","metadata":{}},{"cell_type":"markdown","source":"## Based on this, lets choose few basic augmentations to start with - \n\n1. RandomResizedCrop\n2. RandomRotation\n3. RandomRotate90\n3. RandomScale #Zoom in and out\n4. Flip\n5. Normalize\n6. ToTensor\n\n\n","metadata":{}},{"cell_type":"code","source":"p = 1\ncount = 16 \nsubplot = (4,4)\nwidth = 15 * subplot[0]/  subplot[1] + 5\nheight = 15 * subplot[1]/  subplot[0]\n\nvalid_tfms_albu = A.Compose(\n        [#A.Resize(image_size, image_size),\n         A.RandomScale((0.8 , 1.5)),\n         A.Rotate(limit=45) , \n         A.RandomResizedCrop(image_size, image_size, scale=(0.9, 1), p=p),\n         A.Flip(p=p),\n        A.RandomRotate90(p=p),    \n         Normalize(mean= imagenet_stats['mean'] ,std= imagenet_stats['std'] ,),\n         ToTensorV2() ,])\n    \ntrain_dataset = Segmentation_ImageDataset(dataframe= df_train , path_column= 'id' , transform=valid_tfms_albu)\n\nimage_batch_comparison_visualizer2d( train_dataset, count= count, subplot= subplot,   unnormalize =  True,\n                        figheight=height, figwidth=width , alpha = 0.3)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:44:19.646641Z","iopub.execute_input":"2021-10-17T14:44:19.646983Z","iopub.status.idle":"2021-10-17T14:44:23.019062Z","shell.execute_reply.started":"2021-10-17T14:44:19.646946Z","shell.execute_reply":"2021-10-17T14:44:23.018227Z"},"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##  Now lets consider other methods - \n\nLets add RandomBrightnessContrast, Hue saturation and cutouts.","metadata":{}},{"cell_type":"code","source":" valid_tfms_albu = A.Compose([\n            A.Resize(image_size, image_size),\n            A.OneOf([ A.RandomBrightnessContrast( brightness_limit=0.2, contrast_limit=0.2,),\n                    A.HueSaturationValue( hue_shift_limit=20, sat_shift_limit=50, val_shift_limit=50),], p=p,),\n            A.Cutout(max_h_size=int(image_size * 0.05), max_w_size=int(image_size * 0.05), num_holes=5, p= 0.5),\n            Normalize(mean=imagenet_stats['mean'], std=imagenet_stats['std'], ),\n            ToTensorV2()\n            ])\n    \ntrain_dataset = Segmentation_ImageDataset(dataframe= df_train , path_column= 'id' , transform=valid_tfms_albu)\n\nimage_batch_comparison_visualizer2d( train_dataset, count= count, subplot= subplot,   unnormalize =  True,\n                        figheight=height, figwidth=width , alpha = 0.3)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:44:23.020444Z","iopub.execute_input":"2021-10-17T14:44:23.020662Z","iopub.status.idle":"2021-10-17T14:44:26.572166Z","shell.execute_reply.started":"2021-10-17T14:44:23.020636Z","shell.execute_reply":"2021-10-17T14:44:26.57079Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since Cutouts dont seem to be applied to masks, it seems they wont help much in understanding pixel level information.\n\nShiftScale seems to generate intersting results. This can be added to out list.","metadata":{}},{"cell_type":"markdown","source":"# Geometrical transformations\n\nNow, lets see if we can use geometrical transformations.\nBut, before that, lets add grid to the image.\n","metadata":{}},{"cell_type":"code","source":"def draw_grid(image, grid_size = 50):\n    for i in range(0, image.shape[1], grid_size):\n        image = cv2.line(image, (i, 0), (i, image.shape[0]), color=(255,))\n    for j in range(0, image.shape[0], grid_size):\n        image = cv2.line(image, (0, j), (image.shape[1], j), color=(255,))\n    return image  ","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:45:04.232382Z","iopub.execute_input":"2021-10-17T14:45:04.232682Z","iopub.status.idle":"2021-10-17T14:45:04.241539Z","shell.execute_reply.started":"2021-10-17T14:45:04.23265Z","shell.execute_reply":"2021-10-17T14:45:04.240628Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lets start by adding Elastic transform and Persepective.","metadata":{}},{"cell_type":"code","source":"valid_tfms_albu = A.Compose([\n    A.ElasticTransform ( p = 1) ,\n    A.Perspective (scale=(0.05, 0.1) , p = 1),\n        Normalize(mean=imagenet_stats['mean'], std=imagenet_stats['std'], ),\n            ToTensorV2()])\n    \ntrain_dataset = Segmentation_ImageDataset(dataframe= df_train , path_column= 'id' , transform=valid_tfms_albu ,\n                                         function = draw_grid )\n\nimage_batch_comparison_visualizer2d( train_dataset, count= count, subplot= subplot,   unnormalize =  True,\n                        figheight=height, figwidth=width , alpha = 0.3)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:45:05.146921Z","iopub.execute_input":"2021-10-17T14:45:05.147189Z","iopub.status.idle":"2021-10-17T14:45:12.427745Z","shell.execute_reply.started":"2021-10-17T14:45:05.14716Z","shell.execute_reply":"2021-10-17T14:45:12.426815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### PieceWise Affine and GridDistort","metadata":{}},{"cell_type":"code","source":"valid_tfms_albu = A.Compose([\n     A.PiecewiseAffine (scale=(0.03, 0.05) , p = 1),\n     A.GridDistortion(p=1),\n        Normalize(mean=imagenet_stats['mean'], std=imagenet_stats['std'], ),\n            ToTensorV2()\n            ])\n\n    \ntrain_dataset = Segmentation_ImageDataset(dataframe= df_train , path_column= 'id' , transform=valid_tfms_albu ,\n                                         function = draw_grid )\n\n\nimage_batch_comparison_visualizer2d( train_dataset, count= count, subplot= subplot,   unnormalize =  True,\n                        figheight=height, figwidth=width , alpha = 0.3)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:45:12.429538Z","iopub.execute_input":"2021-10-17T14:45:12.429787Z","iopub.status.idle":"2021-10-17T14:45:18.435738Z","shell.execute_reply.started":"2021-10-17T14:45:12.429756Z","shell.execute_reply":"2021-10-17T14:45:18.434388Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Optical distortion \n\nThis one is slightly tricky. Over-doing it might distort the scans too much.","metadata":{}},{"cell_type":"code","source":"valid_tfms_albu = A.Compose([\n     A.OpticalDistortion(distort_limit=0.5, shift_limit=0.5, p=1) ,\n        Normalize(mean=imagenet_stats['mean'], std=imagenet_stats['std'], ),\n            ToTensorV2()])\n\n    \ntrain_dataset = Segmentation_ImageDataset(dataframe= df_train , path_column= 'id' , transform=valid_tfms_albu ,\n                                         function = draw_grid )\n\nimage_batch_comparison_visualizer2d( train_dataset, count= count, subplot= subplot,   unnormalize =  True,\n                        figheight=height, figwidth=width , alpha = 0.3)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:45:18.437302Z","iopub.execute_input":"2021-10-17T14:45:18.437918Z","iopub.status.idle":"2021-10-17T14:45:22.181667Z","shell.execute_reply.started":"2021-10-17T14:45:18.43787Z","shell.execute_reply":"2021-10-17T14:45:22.180681Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### We can use most of these geometrical augmentations work, we will include them in list.","metadata":{}},{"cell_type":"markdown","source":"### Lets try other methods now,\n\n### CLAHE augmentation\nCLAHE, Coarse dropout","metadata":{}},{"cell_type":"code","source":"valid_tfms_albu = A.Compose([\n            A.Resize(image_size, image_size),\n            A.ShiftScaleRotate(shift_limit=0.15, scale_limit=0.4, rotate_limit=45, p=p), #Randomly apply affine transforms: translate, scale and rotate the input.\n            A.CLAHE(clip_limit=(1, 8), p=p),\n            A.CoarseDropout(max_holes=10, p=p),\n            Normalize(mean=imagenet_stats['mean'], std=imagenet_stats['std'], ),\n            ToTensorV2()\n            ])\n\ntrain_dataset = Segmentation_ImageDataset(dataframe= df_train , path_column= 'id' , transform=valid_tfms_albu,\n                                                        function = draw_grid)\n\nimage_batch_comparison_visualizer2d( train_dataset, count= count, subplot= subplot,   unnormalize =  True,\n                        figheight=height, figwidth=width , alpha = 0.3)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:45:40.820652Z","iopub.execute_input":"2021-10-17T14:45:40.821365Z","iopub.status.idle":"2021-10-17T14:45:44.59356Z","shell.execute_reply.started":"2021-10-17T14:45:40.821328Z","shell.execute_reply":"2021-10-17T14:45:44.592617Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nUsing CLAHE seems to improve visualisation of cells. This can be used.","metadata":{}},{"cell_type":"markdown","source":"# Finally, lets combine the them.\n\nWe will use one of the geometrical optionally, and set probability of most transforms as 0.25.","metadata":{}},{"cell_type":"code","source":"p = 0.25\n\nvalid_tfms_albu = A.Compose(\n        [A.Resize(image_size+25, image_size+25),\n        A.RandomScale((0.8 , 1.5) , p = p ),\n        A.Rotate(limit=45 , p = p ) , \n        A.RandomResizedCrop(image_size, image_size, scale=(0.9, 1), p=1),\n        A.Flip(p=p),\n        A.RandomRotate90(p=p),    \n\n        A.OneOf([ A.ElasticTransform () ,\n                  A.Perspective (scale=(0.05, 0.1) ),\n                  A.PiecewiseAffine (scale=(0.03, 0.05) ),\n                 A.ShiftScaleRotate(shift_limit=0.15, scale_limit=0.4, rotate_limit=45),],\n                p=p,),\n        A.OneOf([ A.GridDistortion() ,\n                  A.OpticalDistortion(distort_limit=0.5, shift_limit=0.5) ],\n                p=p,),\n        \n        A.CLAHE(clip_limit=(1, 8), p=p),\n        A.CoarseDropout(max_holes=10, p=p),\n        A.Cutout(max_h_size=int(image_size * 0.05), max_w_size=int(image_size * 0.05), num_holes=5, p= p),\n        \n        Normalize(mean=imagenet_stats['mean'], std=imagenet_stats['std'], ),\n        ToTensorV2()\n            ])\n\ncount = 4 * 15\nsubplot = (15,4)\nwidth = 7  #* subplot[0]/  subplot[1] + 5\nheight = 30 #* subplot[1]/  subplot[0]\n\ntrain_dataset = Segmentation_ImageDataset(dataframe= df_train , path_column= 'id' , \n                                          transform=valid_tfms_albu, function = draw_grid)\n\nimage_batch_comparison_visualizer2d( train_dataset, count= count, subplot= subplot,   unnormalize =  True,\n                        figheight=height, figwidth=width , alpha = 0.3 , title = False)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:45:57.101082Z","iopub.execute_input":"2021-10-17T14:45:57.10139Z","iopub.status.idle":"2021-10-17T14:46:08.512596Z","shell.execute_reply.started":"2021-10-17T14:45:57.101358Z","shell.execute_reply":"2021-10-17T14:46:08.511794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# More info on Albumentation package - \n\nhttps://albumentations.ai/docs/examples/example_kaggle_salt/","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}