{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfrom PIL import Image\n\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport tensorflow as tf\nimport tensorflow_datasets as tfds\n\nimport torch\nfrom torch.utils.data import Dataset\nfrom torchvision import datasets\nfrom torchvision.transforms import ToTensor\nfrom torchvision.transforms import ToPILImage\nimport matplotlib.pyplot as plt\n\nfrom tqdm import tqdm\n\n# for dirname, _, filenames in os.walk('/kaggle/input/hubmap-hacking-the-human-vasculature/train'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-13T08:36:29.589968Z","iopub.execute_input":"2023-06-13T08:36:29.590811Z","iopub.status.idle":"2023-06-13T08:36:45.437046Z","shell.execute_reply.started":"2023-06-13T08:36:29.590763Z","shell.execute_reply":"2023-06-13T08:36:45.436077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = \"/kaggle/input/hubmap-hacking-the-human-vasculature/train\"\ntest_dir = \"/kaggle/input/hubmap-hacking-the-human-vasculature/test\"","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:45.439030Z","iopub.execute_input":"2023-06-13T08:36:45.440302Z","iopub.status.idle":"2023-06-13T08:36:45.446461Z","shell.execute_reply.started":"2023-06-13T08:36:45.440268Z","shell.execute_reply":"2023-06-13T08:36:45.445585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"/kaggle/input/hubmap-hacking-the-human-vasculature/sample_submission.csv\" , \"r\") as f:\n    k = list(f)\nprint(k[0])","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:45.447823Z","iopub.execute_input":"2023-06-13T08:36:45.449529Z","iopub.status.idle":"2023-06-13T08:36:45.472950Z","shell.execute_reply.started":"2023-06-13T08:36:45.449430Z","shell.execute_reply":"2023-06-13T08:36:45.471892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"A.Compose([\n        A.Resize(width = 512 , height = 512) , \n        A.Normalize(\n            mean = [0 , 0] , \n            std = [1 , 1] , \n            max_pixel_value = 255\n        ) , \n        ToTensorV2()\n    ])\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:45.475736Z","iopub.execute_input":"2023-06-13T08:36:45.476254Z","iopub.status.idle":"2023-06-13T08:36:45.486035Z","shell.execute_reply.started":"2023-06-13T08:36:45.476223Z","shell.execute_reply":"2023-06-13T08:36:45.485091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = A.Compose([\n        A.Resize(width = 512 , height = 512) , \n        A.Normalize(\n            mean = [0 , 0 , 0] , \n            std = [1 , 1 , 1] , \n            max_pixel_value = 255\n        ) , \n        ToTensorV2()\n    ])","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:45.487226Z","iopub.execute_input":"2023-06-13T08:36:45.487606Z","iopub.status.idle":"2023-06-13T08:36:45.496759Z","shell.execute_reply.started":"2023-06-13T08:36:45.487577Z","shell.execute_reply":"2023-06-13T08:36:45.495790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display(im , augments = False):\n\n    img = im\n    \n    if augments :\n        \n        img = A.Compose([\n        A.Resize(width = 512 , height = 512) , \n        A.Normalize(\n            mean = [0 , 0 , 0] , \n            std = [1 , 1 , 1] , \n            max_pixel_value = 255\n        ) , \n        ToTensorV2()\n    ])(image = im)[\"image\"]\n\n    # return image\n    \n    plt.imshow(tf.reshape(img , (512 , 512 , 3)))","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:45.498156Z","iopub.execute_input":"2023-06-13T08:36:45.498608Z","iopub.status.idle":"2023-06-13T08:36:45.513110Z","shell.execute_reply.started":"2023-06-13T08:36:45.498578Z","shell.execute_reply":"2023-06-13T08:36:45.512293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = hubmapDataset(image_dir = train_dir, labels_file = '../input/hubmap-hacking-the-human-vasculature/polygons.jsonl')","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:45.528012Z","iopub.execute_input":"2023-06-13T08:36:45.528355Z","iopub.status.idle":"2023-06-13T08:36:46.454961Z","shell.execute_reply.started":"2023-06-13T08:36:45.528325Z","shell.execute_reply":"2023-06-13T08:36:46.452681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class hubmapDataset(Dataset):\n    \n    def __init__(self, image_dir, labels_file , augments = False):\n        \n        with open(labels_file, 'r') as json_file:\n            self.json_labels = [json.loads(line) for line in json_file]\n\n        self.image_dir = image_dir\n#         self.transform = transform\n        self.augments = augments\n\n    __len__ = lambda self : len(self.json_labels)    \n        \n    def __getitem__(self, idx):\n        \n        image_path = os.path.join(self.image_dir, f\"{self.json_labels[idx]['id']}.tif\")\n        image = Image.open(image_path)\n        \n        if self.augments:\n            \n            image = a(image = image)[\"image\"]\n        \n        mask = np.zeros((512, 512), dtype=np.float32)\n\n        for annot in self.json_labels[idx]['annotations']:\n\n            cords = annot['coordinates']\n            \n            if annot['type'] == \"blood_vessel\":\n                \n                for cord in cords:\n                    \n                    rr, cc = np.array([i[1] for i in cord]), np.asarray([i[0] for i in cord])\n                    \n                    mask[rr, cc] = 1\n\n        image = torch.tensor(np.array(image), dtype=torch.float32).permute(2, 0, 1)  # Shape: [C, H, W]\n        mask = torch.tensor(mask, dtype=torch.float32)\n\n#         if self.transform:\n#             image = self.transform(image)\n\n        return image, mask","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:45.514558Z","iopub.execute_input":"2023-06-13T08:36:45.514909Z","iopub.status.idle":"2023-06-13T08:36:45.526679Z","shell.execute_reply.started":"2023-06-13T08:36:45.514877Z","shell.execute_reply":"2023-06-13T08:36:45.525839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"/kaggle/working/\"\n\nos.makedirs('/kaggle/working/Sample_data/Image', exist_ok = True)\nos.makedirs('/kaggle/working/Sample_data/Mask', exist_ok = True)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:46.456188Z","iopub.status.idle":"2023-06-13T08:36:46.456716Z","shell.execute_reply.started":"2023-06-13T08:36:46.456485Z","shell.execute_reply":"2023-06-13T08:36:46.456506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = ToPILImage()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:46.458838Z","iopub.status.idle":"2023-06-13T08:36:46.459295Z","shell.execute_reply.started":"2023-06-13T08:36:46.459058Z","shell.execute_reply":"2023-06-13T08:36:46.459078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index , data in tqdm(enumerate(train_dataset) , total = len(train_dataset)):\n    \n    img , mask = data\n    \n    img = transform(img)\n    mask = transform(mask)\n    \n    img.save(path + \"Sample_data/Image/img_\" + str(index) + \".jpg\")\n    mask.save(path + \"Sample_data/Mask/mas_\" + str(index) + \".jpg\")","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:46.461083Z","iopub.status.idle":"2023-06-13T08:36:46.461538Z","shell.execute_reply.started":"2023-06-13T08:36:46.461310Z","shell.execute_reply":"2023-06-13T08:36:46.461332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_dir = \"/kaggle/working/Sample_data/Image/\"\nmask_dir = \"/kaggle/working/Sample_data/Mask/\"\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:46.463463Z","iopub.status.idle":"2023-06-13T08:36:46.464446Z","shell.execute_reply.started":"2023-06-13T08:36:46.464210Z","shell.execute_reply":"2023-06-13T08:36:46.464233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = os.listdir(image_dir)\ntest = os.listdir(mask_dir)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:46.465491Z","iopub.status.idle":"2023-06-13T08:36:46.466502Z","shell.execute_reply.started":"2023-06-13T08:36:46.466275Z","shell.execute_reply":"2023-06-13T08:36:46.466297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\" Parts of the U-Net model \"\"\"\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass DoubleConv(nn.Module):\n    \"\"\"(convolution => [BN] => ReLU) * 2\"\"\"\n\n    def __init__(self, in_channels, out_channels, mid_channels=None):\n        super().__init__()\n        if not mid_channels:\n            mid_channels = out_channels\n        self.double_conv = nn.Sequential(\n            nn.Conv2d(in_channels, mid_channels, kernel_size=3, padding=1, bias=False),\n            nn.BatchNorm2d(mid_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(mid_channels, out_channels, kernel_size=3, padding=1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True)\n        )\n\n    def forward(self, x):\n        return self.double_conv(x)\n\n\nclass Down(nn.Module):\n    \"\"\"Downscaling with maxpool then double conv\"\"\"\n\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        self.maxpool_conv = nn.Sequential(\n            nn.MaxPool2d(2),\n            DoubleConv(in_channels, out_channels)\n        )\n\n    def forward(self, x):\n        return self.maxpool_conv(x)\n\n\nclass Up(nn.Module):\n    \"\"\"Upscaling then double conv\"\"\"\n\n    def __init__(self, in_channels, out_channels, bilinear=True):\n        super().__init__()\n\n        # if bilinear, use the normal convolutions to reduce the number of channels\n        if bilinear:\n            self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n            self.conv = DoubleConv(in_channels, out_channels, in_channels // 2)\n        else:\n            self.up = nn.ConvTranspose2d(in_channels, in_channels // 2, kernel_size=2, stride=2)\n            self.conv = DoubleConv(in_channels, out_channels)\n\n    def forward(self, x1, x2):\n        x1 = self.up(x1)\n        # input is CHW\n        diffY = x2.size()[2] - x1.size()[2]\n        diffX = x2.size()[3] - x1.size()[3]\n\n        x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2,\n                        diffY // 2, diffY - diffY // 2])\n        # if you have padding issues, see\n        # https://github.com/HaiyongJiang/U-Net-Pytorch-Unstructured-Buggy/commit/0e854509c2cea854e247a9c615f175f76fbb2e3a\n        # https://github.com/xiaopeng-liao/Pytorch-UNet/commit/8ebac70e633bac59fc22bb5195e513d5832fb3bd\n        x = torch.cat([x2, x1], dim=1)\n        return self.conv(x)\n\n\nclass OutConv(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super(OutConv, self).__init__()\n        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=1)\n\n    def forward(self, x):\n        return self.conv(x)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:46.467711Z","iopub.status.idle":"2023-06-13T08:36:46.468369Z","shell.execute_reply.started":"2023-06-13T08:36:46.468138Z","shell.execute_reply":"2023-06-13T08:36:46.468160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class UNet(nn.Module):\n    def __init__(self, n_channels, n_classes, bilinear=False):\n        super(UNet, self).__init__()\n        self.n_channels = n_channels\n        self.n_classes = n_classes\n        self.bilinear = bilinear\n\n        self.inc = (DoubleConv(n_channels, 64))\n        self.down1 = (Down(64, 128))\n        self.down2 = (Down(128, 256))\n        self.down3 = (Down(256, 512))\n        factor = 2 if bilinear else 1\n        self.down4 = (Down(512, 1024 // factor))\n        self.up1 = (Up(1024, 512 // factor, bilinear))\n        self.up2 = (Up(512, 256 // factor, bilinear))\n        self.up3 = (Up(256, 128 // factor, bilinear))\n        self.up4 = (Up(128, 64, bilinear))\n        self.outc = (OutConv(64, n_classes))\n\n    def forward(self, x):\n        x1 = self.inc(x)\n        x2 = self.down1(x1)\n        x3 = self.down2(x2)\n        x4 = self.down3(x3)\n        x5 = self.down4(x4)\n        x = self.up1(x5, x4)\n        x = self.up2(x, x3)\n        x = self.up3(x, x2)\n        x = self.up4(x, x1)\n        logits = self.outc(x)\n        return logits\n\n    def use_checkpointing(self):\n        self.inc = torch.utils.checkpoint(self.inc)\n        self.down1 = torch.utils.checkpoint(self.down1)\n        self.down2 = torch.utils.checkpoint(self.down2)\n        self.down3 = torch.utils.checkpoint(self.down3)\n        self.down4 = torch.utils.checkpoint(self.down4)\n        self.up1 = torch.utils.checkpoint(self.up1)\n        self.up2 = torch.utils.checkpoint(self.up2)\n        self.up3 = torch.utils.checkpoint(self.up3)\n        self.up4 = torch.utils.checkpoint(self.up4)\n        self.outc = torch.utils.checkpoint(self.outc)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:46.469673Z","iopub.status.idle":"2023-06-13T08:36:46.470521Z","shell.execute_reply.started":"2023-06-13T08:36:46.470290Z","shell.execute_reply":"2023-06-13T08:36:46.470311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch import Tensor\n\ndef dice_coeff(input: Tensor, target: Tensor, reduce_batch_first: bool = False, epsilon: float = 1e-6):\n    # Average of Dice coefficient for all batches, or for a single mask\n    assert input.size() == target.size()\n    assert input.dim() == 3 or not reduce_batch_first\n\n    sum_dim = (-1, -2) if input.dim() == 2 or not reduce_batch_first else (-1, -2, -3)\n\n    inter = 2 * (input * target).sum(dim=sum_dim)\n    sets_sum = input.sum(dim=sum_dim) + target.sum(dim=sum_dim)\n    sets_sum = torch.where(sets_sum == 0, inter, sets_sum)\n\n    dice = (inter + epsilon) / (sets_sum + epsilon)\n    return dice.mean()\n\n\ndef multiclass_dice_coeff(input: Tensor, target: Tensor, reduce_batch_first: bool = False, epsilon: float = 1e-6):\n    # Average of Dice coefficient for all classes\n    return dice_coeff(input.flatten(0, 1), target.flatten(0, 1), reduce_batch_first, epsilon)\n\n\ndef dice_loss(input: Tensor, target: Tensor, multiclass: bool = False):\n    # Dice loss (objective to minimize) between 0 and 1\n    fn = multiclass_dice_coeff if multiclass else dice_coeff\n    return 1 - fn(input, target, reduce_batch_first=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T08:36:46.471832Z","iopub.status.idle":"2023-06-13T08:36:46.472731Z","shell.execute_reply.started":"2023-06-13T08:36:46.472488Z","shell.execute_reply":"2023-06-13T08:36:46.472509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}