{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction:\n\nSemantic segmentation is a problem of computer vision in which our task is to assign a class to each pixel in the image using that image as an input. In the case of semantic segmentation, we don’t care if we have multiple instances (objects) of the same class, we simply label them all with their class.\n\n\nThere are many reviews on Unet, how it changed the field forever. It is unitively very clear architecture, which consists of an encoder, which generates a representation of the image and a decoder, which uses that representation to build the segmentation. Two maps at each spatial resolution are concatenated (the grey arrows), so you combine two different representations of the image together. \n\n\nThe next bog thing was using a pre-trained encoder. Think about the image classification problem. Effectively, we try to build a feature representation of the image, such that different classes in that feature space can be separated. We can take (almost) any CNN and use it as an encoder, take features from that encoder and feed them to our decoder.\n\n![](https://miro.medium.com/max/700/1*lvXoKMHoPJMKpKK7keZMEA.png)\n\n\n* Description\n\nThis kernel provides a starter Pytorch-GPU code for inference that performs dividing the images into tiles, selection of tiles with tissue, Train and evaluate of UNet(encode = efficientnet-b3) model, combining the tile masks back into image level masks, conversion into RLE, and visualize the predicted mask for images tests.\nThe inference is pretrained based on models trained in the kernel.","attachments":{}},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install git+https://github.com/qubvel/segmentation_models.pytorch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading Libraries, Parameters, Functions"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import uuid\nimport numba, cv2, gc, pickle\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport tifffile as tiff \nimport seaborn as sns\nimport rasterio\nfrom rasterio.windows import Window\nimport pathlib, sys, os, random, time\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom tqdm.notebook import tqdm\n\nimport albumentations as A\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as D\n\nimport torchvision\nfrom torchvision import transforms as T\n\nDATA_PATH = '../input/hubmap-kidney-segmentation'\nEPOCHES = 15\nBATCH_SIZE = 16\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu' \n\ndef set_seeds(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n\nset_seeds();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"../input/hubmap-kidney-segmentation/train.csv\")\ndf.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle_encode(im):\n    pixels = im.flatten(order = 'F')\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef rle_decode(mask_rle, shape=(256, 256)):\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape, order='F')\n\n@numba.njit()\ndef rle_numba(pixels):\n    size = len(pixels)\n    points = []\n    if pixels[0] == 1: points.append(0)\n    flag = True\n    for i in range(1, size):\n        if pixels[i] != pixels[i-1]:\n            if flag:\n                points.append(i+1)\n                flag = False\n            else:\n                points.append(i+1 - points[-1])\n                flag = True\n    if pixels[-1] == 1: points.append(size-points[-1]+1)    \n    return points\n\ndef rle_numba_encode(image):\n    pixels = image.flatten(order = 'F')\n    points = rle_numba(pixels)\n    return ' '.join(str(x) for x in points)\n\ndef make_grid(shape, window=256, min_overlap=32):\n    x, y = shape\n    nx = x // (window - min_overlap) + 1\n    x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)\n    x1[-1] = x - window\n    x2 = (x1 + window).clip(0, x)\n    ny = y // (window - min_overlap) + 1\n    y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)\n    y1[-1] = y - window\n    y2 = (y1 + window).clip(0, y)\n    slices = np.zeros((nx,ny, 4), dtype=np.int64)\n    \n    for i in range(nx):\n        for j in range(ny):\n            slices[i,j] = x1[i], x2[i], y1[j], y2[j]    \n    return slices.reshape(nx*ny,4)\n\ndef img2tensor(img,dtype:np.dtype=np.float32):\n    if img.ndim==2 : img = np.expand_dims(img,2)\n    img = np.transpose(img,(2,0,1))\n    return torch.from_numpy(img.astype(dtype, copy=False))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_id_1 = df.iloc[1,0]\nBASE_PATH = '../input/hubmap-kidney-segmentation/train/'\ndef plot_image(image_id, BASE_PATH = BASE_PATH, scale=None, verbose=1, df = df):\n    image = tiff.imread(os.path.join(BASE_PATH, f\"{image_id}.tiff\"))\n    if len(image.shape) == 5:\n        image = image.squeeze().transpose(1, 2, 0)\n    \n    mask = rle_decode(df[df[\"id\"] == image_id][\"encoding\"].values[0], (image.shape[0],image.shape[1]))\n    if verbose:\n        print(f\" Image shape: {image.shape}\")\n        print(f\" Mask shape: {mask.shape}\")\n    \n    if scale:\n        new_size = (image.shape[1] // scale, image.shape[0] // scale)\n        image = cv2.resize(image, new_size)\n        mask = cv2.resize(mask, new_size)\n        \n        if verbose:\n            print(f\" Resized Image shape: {image.shape}\")\n            print(f\" Resized Mask shape: {mask.shape}\")\n    plt.figure(figsize=(32, 20))\n    plt.subplot(1, 3, 1)\n    plt.imshow(image)\n    plt.title(f\"Image {image_id}\", fontsize=18)\n    \n    plt.subplot(1, 3, 2)\n    plt.imshow(image)\n    plt.imshow(mask, cmap=\"hot\", alpha=0.5)\n    plt.title(\"Image  + mask\", fontsize=18)    \n    \n    plt.subplot(1, 3, 3)\n    plt.imshow(mask, cmap=\"hot\")\n    plt.title(f\"Mask\", fontsize=18)    \n    \n    plt.show()\n            \n    del image, mask\n\nplot_image(img_id_1, BASE_PATH ,scale = 20 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"identity = rasterio.Affine(1, 0, 0, 0, 1, 0)\n\nclass HubDataset(D.Dataset):\n\n    def __init__(self, root_dir, transform,\n                 window=256, overlap=0, threshold = 100):\n        self.path = pathlib.Path(root_dir)\n        self.overlap = overlap\n        self.window = window\n        self.transform = transform\n        self.csv = pd.read_csv((self.path / 'train.csv').as_posix(),\n                               index_col=[0])\n        self.threshold = threshold\n    \n        self.ids = {}\n        self.x, self.y = [], []\n        self.build_slices()\n        self.len = len(self.x)\n    \n    def build_slices(self):\n        self.masks = []\n        self.files = []\n        self.slices = []\n        count = 0\n        for i, filename in enumerate(self.csv.index.values):\n            tmp = []\n            filepath = (self.path /'train'/(filename+'.tiff')).as_posix()\n            self.files.append(filepath)\n            print(f'Transform-{filename}')\n            with rasterio.open(filepath, transform = identity) as dataset:\n                self.masks.append(rle_decode(self.csv.loc[filename, 'encoding'], dataset.shape))\n                slices = make_grid(dataset.shape, window=self.window, min_overlap=self.overlap)\n                \n                for slc in tqdm(slices, leave=False):\n                    x1,x2,y1,y2 = slc\n                    if self.masks[-1][x1:x2,y1:y2].sum() > self.threshold or np.random.randint(100) > 120:\n                        self.slices.append([i,x1,x2,y1,y2])\n                        \n                        image = dataset.read([1,2,3],\n                            window=Window.from_slices((x1,x2),(y1,y2)))\n                        \n                        image = np.moveaxis(image, 0, -1)\n                        self.x.append(image)\n                        self.y.append(self.masks[-1][x1:x2,y1:y2])\n                        tmp.append(count)\n                        count += 1\n            self.ids[i] = tmp\n        with open(\"folds.pkl\", \"wb\") as f:\n            pickle.dump(self.ids, f)\n    \n    # get data operation\n    def __getitem__(self, index):\n        img, mask = self.x[index], self.y[index]\n        if self.transform is not None:\n            augmented = self.transform(image=img,mask=mask)\n            img,mask = augmented['image'],augmented['mask']\n        return img2tensor((img/255.0 - mean)/std),img2tensor(mask)#self.as_tensor(img), mask\n    \n    def __len__(self):\n        \"\"\"\n        Total number of samples in the dataset\n        \"\"\"\n        return self.len\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations as A\n\nWINDOW = 256\nMIN_OVERLAP = 32\n#NEW_SIZE = 512\n\ntrfm = A.Compose([\n        A.HorizontalFlip(),\n        A.VerticalFlip(),\n        A.RandomRotate90(),\n        A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=15, p=0.9, \n                         border_mode=cv2.BORDER_REFLECT),\n        A.OneOf([\n            A.OpticalDistortion(p=0.3),\n            A.GridDistortion(p=.1),\n            A.IAAPiecewiseAffine(p=0.3),\n        ], p=0.3),\n        A.OneOf([\n            A.HueSaturationValue(10,15,10),\n            A.CLAHE(clip_limit=2),\n            A.RandomBrightnessContrast(),            \n        ], p=0.3),\n    ], p=1.)\n\ngc.collect()\nds = HubDataset(DATA_PATH, window=WINDOW, overlap=MIN_OVERLAP, transform=trfm)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean = np.array([0.65459856,0.48386562,0.69428385])\nstd = np.array([0.15167958,0.23584107,0.13146145])\n\nd_train = D.DataLoader(ds,batch_size=16,shuffle=False,num_workers=4)\nimgs,masks = next(iter(d_train))\nprint(imgs.shape)\nplt.figure(figsize=(32,32))\nfor i,(img,mask) in enumerate(zip(imgs,masks)):\n    img = ((img.permute(1,2,0)*std + mean)*255.0).numpy().astype(np.uint8)\n    plt.subplot(8,8,i+1)\n    plt.imshow(img)\n    plt.imshow(mask.squeeze().numpy(), alpha=0.2)\n    plt.axis('off')\n    plt.subplots_adjust(wspace=None, hspace=None)\n    \ndel imgs,masks\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{},"cell_type":"markdown","source":"## Fonction Loss_DICE"},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/bigironsphere/loss-function-library-keras-pytorch\nclass DiceLoss(nn.Module):\n    def __init__(self, weight=None, size_average=True):\n        super(DiceLoss, self).__init__()\n\n    def forward(self, inputs, targets, smooth=1):\n        \n        #comment out if your model contains a sigmoid or equivalent activation layer\n        inputs = F.sigmoid(inputs)       \n        \n        #flatten label and prediction tensors\n        inputs = inputs.view(-1)\n        targets = targets.view(-1)\n        \n        intersection = (inputs * targets).sum()                            \n        dice = (2.*intersection + smooth)/(inputs.sum() + targets.sum() + smooth)  \n        \n        return 1 - dice\n    \ndef dice_coef(output, target):\n    smooth = 1e-5\n    output_d = torch.sigmoid(output).view(-1).data.cpu().numpy()\n    target_d = target.view(-1).data.cpu().numpy()\n    intersection = (output_d * target_d).sum()\n\n    return (2. * intersection + smooth) / (output_d.sum() + target_d.sum() + smooth)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## U-Net Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import segmentation_models_pytorch as smp\n\nmodel= smp.Unet(encoder_name=\"efficientnet-b3\",        # choose encoder, e.g. mobilenet_v2 or efficientnet-b7\n                encoder_weights=\"imagenet\",     # use `imagenet` pretrained weights for encoder initialization\n                in_channels=3,                  # model input channels (1 for grayscale images, 3 for RGB, etc.)\n                classes = 1 )\n\nmodel.float()\nmodel.to(DEVICE)\nmodel.train()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from catalyst import dl, contrib\ntorch.cuda.empty_cache()\nnum_epochs = 10\nbase_optimizer = contrib.nn.RAdam([\n    {'params': model.decoder.parameters(), 'lr':  5e-3,'weight_decay': 0.00003 }, \n    {'params': model.encoder.parameters(), 'lr':  5e-3/10, 'weight_decay': 0.00003},])\n\noptimizer = contrib.nn.Lookahead(base_optimizer)\n\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.2, patience=5)\ndiceloss = DiceLoss()\nloss_list = [] \ndice_list = []\nfor epoch in tqdm(range(num_epochs)):\n    ###Train\n    train_loss = 0\n    for data in d_train:\n        optimizer.zero_grad()\n        img, mask = data\n        img = img.to(DEVICE)\n        mask = mask.to(DEVICE)\n\n        outputs = model(img)\n        dice = dice_coef(outputs, mask)\n        loss = diceloss(outputs, mask)\n        loss.backward()\n        optimizer.step()\n\n        train_loss += loss.item()\n        del mask, img, outputs\n        gc.collect()\n    train_loss /= =len(d_train)\n    print(f\"EPOCH: {epoch + 1}/ : {num_epochs}, train_loss: {train_loss}, dice:{dice}\")\n    loss_list.append(train_loss)\n    dice_list.append(dice)\n    scheduler.step(train_loss)\ndel d_train, ds\ngc.collect()    \ntorch.save(model.state_dict(), 'model.pth')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Results"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,5))\nn_e = np.arange(len(loss_list))\nplt.plot(n_e, loss_list)\nplt.title('Train loss ')\n#plt.plot(n_e,history.history['val_dice_coe'],'-o',label='Val dice_coe',color='#1f77b4')\n\nplt.figure(figsize=(10,5))\nn_e = np.arange(len(dice_list))\nplt.plot(n_e, dice_list)\nplt.title('Train diece ')\n#plt.plot(n_e,history.history['val_dice_coe'],'-o',label='Val dice_coe',color='#1f77b4')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Results for test images public"},{"metadata":{"trusted":true},"cell_type":"code","source":"WINDOW = 2024\nMIN_OVERLAP = 32\nNEW_SIZE = 512\n\ntrfm = T.Compose([\n    T.ToPILImage(),\n    T.Resize(NEW_SIZE),\n    T.ToTensor(),\n    T.Normalize([0.625, 0.448, 0.688],\n                [0.131, 0.177, 0.101]), ])\n\np = pathlib.Path(DATA_PATH)\nsubm = {}\nmodel.eval()\n\nfor i, filename in enumerate(p.glob('test/*.tiff')):\n    dataset = rasterio.open(filename.as_posix(), transform = identity)\n    slices = make_grid(dataset.shape, window=WINDOW, min_overlap=MIN_OVERLAP)\n    preds = np.zeros(dataset.shape, dtype=np.uint8)\n    for (x1,x2,y1,y2) in slices:\n        image = dataset.read([1,2,3], window = Window.from_slices((x1,x2),(y1,y2)))\n        image = np.moveaxis(image, 0, -1)\n        image = trfm(image)\n        with torch.no_grad():\n            image = image.to(DEVICE)[None]\n            score = model(image)[0][0]\n            score_sigmoid = score.sigmoid().cpu().numpy()\n            score_sigmoid = cv2.resize(score_sigmoid, (WINDOW, WINDOW))\n            \n            preds[x1:x2,y1:y2] = (score_sigmoid > 0.5).astype(np.uint8)\n            \n    subm[i] = {'id':filename.stem, 'predicted': rle_numba_encode(preds)}\n    del preds\n    gc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame.from_dict(subm, orient='index')\nsubmission.to_csv('submission.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE_PATH = '../input/hubmap-kidney-segmentation/test'\ndef plot_image(image_id, BASE_PATH = BASE_PATH, scale=None, verbose=1, df = submission):\n    image = tiff.imread(os.path.join(BASE_PATH, f\"{image_id}.tiff\"))\n    if len(image.shape) == 5:\n        image = image.squeeze().transpose(1, 2, 0)\n    \n    mask = rle_decode(df[df[\"id\"] == image_id][\"predicted\"].values[0], (image.shape[0],image.shape[1]))\n    if verbose:\n        print(f\"[{image_id}] Image shape: {image.shape}\")\n        print(f\"[{image_id}] Mask shape: {mask.shape}\")\n    \n    if scale:\n        new_size = (image.shape[1] // scale, image.shape[0] // scale)\n        image = cv2.resize(image, new_size)\n        mask = cv2.resize(mask, new_size)\n        \n        if verbose:\n            print(f\"[{image_id}] Resized Image shape: {image.shape}\")\n            print(f\"[{image_id}] Resized Mask shape: {mask.shape}\")\n    plt.figure(figsize=(32, 20))\n    plt.subplot(1, 3, 1)\n    plt.imshow(image)\n    plt.title(f\"Image {image_id}\", fontsize=18)\n    \n    plt.subplot(1, 3, 2)\n    plt.imshow(image)\n    plt.imshow(mask, cmap=\"hot\", alpha=0.5)\n    plt.title(\"Image  + Predicted Mask\", fontsize=18)    \n    \n    plt.subplot(1, 3, 3)\n    plt.imshow(mask, cmap=\"hot\")\n    plt.title(f\"Predicted Mask\", fontsize=18)    \n    \n    plt.show()\n            \n    del image, mask\n\nfor i in range(submission.shape[0]):    \n    img_id_1 = submission.iloc[i,0]\n    plot_image(img_id_1, scale = 20, verbose=1)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}