{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 1. Setup","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('../input/hubmap-hpa-hacking-the-human-body-dataset/packages/pretrained-models.pytorch-master')\nsys.path.append('../input/hubmap-hpa-hacking-the-human-body-dataset/packages/EfficientNet-PyTorch-master')\nsys.path.append('../input/hubmap-hpa-hacking-the-human-body-dataset/packages/segmentation_models.pytorch-master')\nsys.path.append('../input/hubmap-hpa-hacking-the-human-body-dataset/packages/coat_daformer')\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-10T04:17:35.032790Z","iopub.execute_input":"2022-09-10T04:17:35.033212Z","iopub.status.idle":"2022-09-10T04:17:35.058533Z","shell.execute_reply.started":"2022-09-10T04:17:35.033177Z","shell.execute_reply":"2022-09-10T04:17:35.057564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir packages\n!cp -r ../input/hubmap-hpa-hacking-the-human-body-dataset/packages ./","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:17:35.842866Z","iopub.execute_input":"2022-09-10T04:17:35.843340Z","iopub.status.idle":"2022-09-10T04:17:39.416820Z","shell.execute_reply.started":"2022-09-10T04:17:35.843299Z","shell.execute_reply":"2022-09-10T04:17:39.415531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index --find-links=./packages ./packages/spams-2.6.5.4\n!pip install --no-index --find-links=./packages ./packages/staintools-2.1.2\n!pip install ./packages/timm-0.4.12-py3-none-any.whl\n!pip install ../input/hubmap-hpa-hacking-the-human-body-dataset/packages/einops-0.4.1-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:17:39.421012Z","iopub.execute_input":"2022-09-10T04:17:39.421399Z","iopub.status.idle":"2022-09-10T04:20:43.096451Z","shell.execute_reply.started":"2022-09-10T04:17:39.421351Z","shell.execute_reply":"2022-09-10T04:20:43.095255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom pathlib import Path\nfrom glob import glob\nfrom tqdm import tqdm\nimport yaml\nimport json\n\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport tifffile\n\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, SequentialSampler\nimport segmentation_models_pytorch as smp\nimport transformers\ntransformers.utils.logging.set_verbosity_error()\nimport coat\nimport daformer\n\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom albumentations import ImageOnlyTransform\nimport staintools","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:20:43.098852Z","iopub.execute_input":"2022-09-10T04:20:43.099300Z","iopub.status.idle":"2022-09-10T04:20:50.232184Z","shell.execute_reply.started":"2022-09-10T04:20:43.099258Z","shell.execute_reply":"2022-09-10T04:20:50.230998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INTERNAL_DATASET = Path('../input/hubmap-organ-segmentation')\nEXTERNAL_DATASET = Path('../input/hubmap-hpa-hacking-the-human-body-dataset')","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:20:50.234749Z","iopub.execute_input":"2022-09-10T04:20:50.235420Z","iopub.status.idle":"2022-09-10T04:20:50.241257Z","shell.execute_reply.started":"2022-09-10T04:20:50.235379Z","shell.execute_reply":"2022-09-10T04:20:50.239756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(INTERNAL_DATASET / 'train.csv')\ndf_train = df_train.merge(pd.read_csv(EXTERNAL_DATASET / 'folds.csv'), on='id', how='left')\ndf_test = pd.read_csv(INTERNAL_DATASET / 'test.csv')\n\ntrain_images = 'train_images/'\ntrain_annotations = 'train_annotations/'\ntest_images = 'test_images/'\n\ndf_train['image_filename'] = df_train['id'].apply(lambda x:  str(INTERNAL_DATASET) + '/' + train_images + str(x) + '.tiff')\ndf_train['polygon_filename'] = df_train['id'].apply(lambda x:  str(INTERNAL_DATASET) + '/' + train_annotations + str(x) + '.json')\ndf_test['image_filename'] = df_test['id'].apply(lambda x:  str(INTERNAL_DATASET) + '/' + test_images + str(x) + '.tiff')\ndf_test['age'] = np.nan\ndf_test['sex'] = np.nan\n\nprint(f'Training Set Shape: {df_train.shape} - Memory Usage: {df_train.memory_usage().sum() / 1024 ** 2:.2f} MB')\nprint(f'Test Set Shape: {df_test.shape} - Memory Usage: {df_test.memory_usage().sum() / 1024 ** 2:.2f} MB')","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:20:50.242808Z","iopub.execute_input":"2022-09-10T04:20:50.243475Z","iopub.status.idle":"2022-09-10T04:20:50.577604Z","shell.execute_reply.started":"2022-09-10T04:20:50.243410Z","shell.execute_reply":"2022-09-10T04:20:50.576412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Annotations Utilities","metadata":{}},{"cell_type":"code","source":"def decode_rle_mask(rle_mask, shape):\n\n    \"\"\"\n    Decode run-length encoded segmentation mask string into 2d array\n\n    Parameters\n    ----------\n    rle_mask (str): Run-length encoded segmentation mask string\n    shape (tuple of shape (2)): Height and width of the mask\n\n    Returns\n    -------\n    mask (numpy.ndarray of shape (height, width)): Decoded 2d segmentation mask\n    \"\"\"\n\n    rle_mask = rle_mask.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (rle_mask[0:][::2], rle_mask[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n\n    mask = np.zeros((shape[0] * shape[1]), dtype=np.uint8)\n    for start, end in zip(starts, ends):\n        mask[start:end] = 1\n\n    mask = mask.reshape(shape[0], shape[1])\n    return mask\n\n\ndef encode_rle_mask(mask):\n\n    \"\"\"\n    Encode 2d array into run-length encoded segmentation mask string\n\n    Parameters\n    ----------\n    mask (numpy.ndarray of shape (height, width)): 2d segmentation mask\n\n    Returns\n    -------\n    rle_mask (str): Run-length encoded segmentation mask string\n    \"\"\"\n\n    mask = mask.T.flatten()\n    mask = np.concatenate([[0], mask, [0]])\n    runs = np.where(mask[1:] != mask[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:21:57.004426Z","iopub.execute_input":"2022-09-10T04:21:57.004981Z","iopub.status.idle":"2022-09-10T04:21:57.021691Z","shell.execute_reply.started":"2022-09-10T04:21:57.004935Z","shell.execute_reply":"2022-09-10T04:21:57.020705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Dataset","metadata":{}},{"cell_type":"code","source":"imaging_measurements = {\n  'HPA': {\n    'pixel_size': {\n      'kidney': 0.4,\n      'prostate': 0.4,\n      'largeintestine': 0.4,\n      'spleen': 0.4,\n      'lung': 0.4\n    },\n    'tissue_thickness': {\n      'kidney': 4,\n      'prostate': 4,\n      'largeintestine': 4,\n      'spleen': 4,\n      'lung': 4\n    }\n  },\n  'Hubmap': {\n    'pixel_size': {\n      'kidney': 0.5,\n      'prostate': 6.263,\n      'largeintestine': 0.229,\n      'spleen': 0.4945,\n      'lung': 0.7562\n    },\n    'tissue_thickness': {\n      'kidney': 10,\n      'prostate': 5,\n      'largeintestine': 8,\n      'spleen': 4,\n      'lung': 5\n    }\n  }\n}","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:21:57.673673Z","iopub.execute_input":"2022-09-10T04:21:57.676562Z","iopub.status.idle":"2022-09-10T04:21:57.686524Z","shell.execute_reply.started":"2022-09-10T04:21:57.676520Z","shell.execute_reply":"2022-09-10T04:21:57.685327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SemanticSegmentationDataset(Dataset):\n\n    def __init__(self, image_paths, organs, data_sources, masks=None, imaging_measurement_adaptation_probability=0, transforms=None):\n\n        self.image_paths = image_paths\n        self.organs = organs\n        self.data_sources = data_sources\n        self.masks = masks\n        self.imaging_measurement_adaptation_probability = imaging_measurement_adaptation_probability\n        self.transforms = transforms\n\n    def __len__(self):\n        return len(self.image_paths)\n\n    def __getitem__(self, idx):\n\n        \"\"\"\n        Get the idxth element in the dataset\n\n        Parameters\n        ----------\n        idx (int): Index of the sample (0 <= idx < len(self.image_paths))\n\n        Returns\n        -------\n        image (torch.FloatTensor of shape (3, height, width)): Image tensor\n        mask (torch.FloatTensor of shape (1, height, width)): Mask tensor\n        \"\"\"\n\n        organ = self.organs[idx]\n        data_source = self.data_sources[idx]\n\n        if data_source == 'HPA' or data_source == 'Hubmap':\n            image = tifffile.imread(str(self.image_paths[idx]))\n        elif data_source == 'GTEx':\n            image = cv2.imread(str(self.image_paths[idx]))\n        else:\n            raise ValueError(f'Invalid data source: {data_source}')\n\n        if data_source == 'HPA':\n            if self.imaging_measurement_adaptation_probability > 0:\n                if np.random.rand() < self.imaging_measurement_adaptation_probability:\n                    # Simulate pixel size artifacts in HPA images randomly\n                    domain_pixel_size = imaging_measurements[data_source]['pixel_size'][organ]\n                    target_pixel_size = imaging_measurements[data_source]['pixel_size'][organ]\n                    pixel_size_scale_factor = domain_pixel_size / target_pixel_size\n\n                    image = cv2.resize(image, fx=pixel_size_scale_factor, fy=pixel_size_scale_factor, interpolation=cv2.INTER_LINEAR)\n                    image = cv2.resize(image, dsize=(image.shape[1], image.shape[0]), interpolation=cv2.INTER_LINEAR)\n\n        if self.masks is not None:\n\n            # Decode RLE mask string into 2d binary semantic segmentation mask array\n            mask = decode_rle_mask(rle_mask=self.masks[idx], shape=image.shape[:2])\n            if data_source == 'Hubmap' or data_source == 'HPA':\n                mask = mask.T\n\n            if self.transforms is not None:\n                # Apply transforms to image and semantic segmentation mask\n                transformed = self.transforms(image=image, mask=mask)\n                image = transformed['image'].float()\n                mask = transformed['mask'].float()\n                mask = torch.unsqueeze(mask, dim=0)\n\n            else:\n                image = torch.as_tensor(image, dtype=torch.float)\n                mask = torch.as_tensor(mask, dtype=torch.float)\n                mask = torch.unsqueeze(mask, dim=0)\n                # Scale pixel values by max 8 bit pixel value\n                image /= 255.\n\n            return image, mask\n\n        else:\n\n            if self.transforms is not None:\n                # Apply transforms to image\n                transformed = self.transforms(image=image)\n                image = transformed['image'].float()\n            else:\n                image = torch.as_tensor(image, dtype=torch.float)\n                # Scale pixel values by max 8 bit pixel value\n                image /= 255.\n\n            return image\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:21:58.148104Z","iopub.execute_input":"2022-09-10T04:21:58.148899Z","iopub.status.idle":"2022-09-10T04:21:58.172531Z","shell.execute_reply.started":"2022-09-10T04:21:58.148857Z","shell.execute_reply":"2022-09-10T04:21:58.171423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Transforms","metadata":{}},{"cell_type":"code","source":"class Scale(ImageOnlyTransform):\n\n    def apply(self, image, **kwargs):\n\n        \"\"\"\n        Scale pixel values between 0 and 1\n\n        Parameters\n        ----------\n        image (numpy.ndarray of shape (height, width)): Image array\n\n        Returns\n        -------\n        image (numpy.ndarray of shape (height, width)): Image array divided by max 8 bit integer\n        \"\"\"\n\n        image = np.float32(image) / 255.\n\n        return image\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:21:59.970708Z","iopub.execute_input":"2022-09-10T04:21:59.971140Z","iopub.status.idle":"2022-09-10T04:21:59.982671Z","shell.execute_reply.started":"2022-09-10T04:21:59.971103Z","shell.execute_reply":"2022-09-10T04:21:59.981317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_semantic_segmentation_transforms(**transform_parameters):\n\n    \"\"\"\n    Get transforms for semantic segmentation dataset\n\n    Parameters\n    ----------\n    transform_parameters (dict): Dictionary of transform parameters\n\n    Returns\n    -------\n    transforms (dict): Transforms for training, validation and test sets\n    \"\"\"\n\n    train_transforms = A.Compose([\n        A.Resize(\n            height=transform_parameters['resize_height'],\n            width=transform_parameters['resize_width'],\n            interpolation=cv2.INTER_NEAREST,\n            always_apply=True\n        ),\n        A.HorizontalFlip(p=transform_parameters['horizontal_flip_probability']),\n        A.VerticalFlip(p=transform_parameters['vertical_flip_probability']),\n        A.RandomRotate90(p=transform_parameters['random_rotate_90_probability']),\n        A.ShiftScaleRotate(\n            shift_limit=transform_parameters['shift_limit'],\n            scale_limit=transform_parameters['scale_limit'],\n            rotate_limit=transform_parameters['rotate_limit'],\n            p=transform_parameters['shift_scale_rotate_probability']\n        ),\n        A.HueSaturationValue(\n            hue_shift_limit=transform_parameters['hue_shift_limit'],\n            sat_shift_limit=transform_parameters['saturation_shift_limit'],\n            val_shift_limit=transform_parameters['value_shift_limit'],\n            p=transform_parameters['hue_saturation_value_probability']\n        ),\n        A.RandomBrightnessContrast(\n            brightness_limit=transform_parameters['brightness_limit'],\n            contrast_limit=transform_parameters['contrast_limit'],\n            p=transform_parameters['random_brightness_contrast_probability']\n        ),\n        A.OneOf([\n            A.CLAHE(\n                clip_limit=transform_parameters['clahe_clip_limit'],\n                tile_grid_size=transform_parameters['clahe_tile_grid_size'],\n                p=transform_parameters['clahe_probability']\n            ),\n            A.Equalize(\n                mode='cv',\n                by_channels=True,\n                p=transform_parameters['equalize_probability']\n            )\n        ], p=transform_parameters['histogram_equalization_probability']),\n        A.OneOf([\n            A.GridDistortion(\n                num_steps=transform_parameters['grid_distortion_num_steps'],\n                distort_limit=transform_parameters['grid_distortion_distort_limit'],\n                interpolation=cv2.INTER_NEAREST,\n                border_mode=cv2.BORDER_REPLICATE,\n                p=transform_parameters['grid_distortion_probability']\n            ),\n            A.OpticalDistortion(\n                distort_limit=transform_parameters['optical_distortion_distort_limit'],\n                shift_limit=transform_parameters['optical_distortion_shift_limit'],\n                interpolation=cv2.INTER_NEAREST,\n                border_mode=cv2.BORDER_REPLICATE,\n                p=transform_parameters['optical_distortion_probability']\n            )\n        ], p=transform_parameters['distortion_probability']),\n        A.OneOf([\n            A.ChannelShuffle(p=transform_parameters['channel_shuffle_probability']),\n            A.ChannelDropout(\n                channel_drop_range=transform_parameters['channel_dropout_channel_drop_range'],\n                fill_value=transform_parameters['channel_dropout_fill_value'],\n                p=transform_parameters['channel_dropout_probability']\n            )\n        ], p=transform_parameters['channel_transform_probability']),\n        A.OneOf([\n            A.CoarseDropout(\n                max_holes=transform_parameters['coarse_dropout_max_holes'],\n                max_height=transform_parameters['coarse_dropout_max_height'],\n                max_width=transform_parameters['coarse_dropout_max_width'],\n                min_holes=transform_parameters['coarse_dropout_min_holes'],\n                min_height=transform_parameters['coarse_dropout_min_height'],\n                min_width=transform_parameters['coarse_dropout_min_width'],\n                fill_value=transform_parameters['coarse_dropout_fill_value'],\n                mask_fill_value=transform_parameters['coarse_dropout_mask_fill_value'],\n                p=transform_parameters['coarse_dropout_probability']\n            ),\n            A.PixelDropout(\n                dropout_prob=transform_parameters['pixel_dropout_dropout_probability'],\n                per_channel=transform_parameters['pixel_dropout_per_channel'],\n                drop_value=transform_parameters['pixel_dropout_drop_value'],\n                mask_drop_value=transform_parameters['pixel_dropout_mask_drop_value'],\n                p=transform_parameters['pixel_dropout_probability']\n            ),\n            A.MaskDropout(\n                max_objects=transform_parameters['mask_dropout_max_objects'],\n                image_fill_value=transform_parameters['mask_dropout_image_fill_value'],\n                mask_fill_value=transform_parameters['mask_dropout_mask_fill_value'],\n                p=transform_parameters['mask_dropout_probability']\n            )\n        ], p=transform_parameters['dropout_probability']),\n        A.Normalize(\n            mean=transform_parameters['normalize_mean'],\n            std=transform_parameters['normalize_std'],\n            max_pixel_value=transform_parameters['normalize_max_pixel_value'],\n            always_apply=True\n        ),\n        ToTensorV2(always_apply=True)\n    ])\n\n    val_transforms = A.Compose([\n        A.Resize(\n            height=transform_parameters['resize_height'],\n            width=transform_parameters['resize_width'],\n            interpolation=cv2.INTER_NEAREST,\n            always_apply=True\n        ),\n        A.Normalize(\n            mean=transform_parameters['normalize_mean'],\n            std=transform_parameters['normalize_std'],\n            max_pixel_value=transform_parameters['normalize_max_pixel_value'],\n            always_apply=True\n        ),\n        ToTensorV2(always_apply=True)\n    ])\n    \n    test_transforms = A.Compose([\n        A.Normalize(\n            mean=transform_parameters['normalize_mean'],\n            std=transform_parameters['normalize_std'],\n            max_pixel_value=transform_parameters['normalize_max_pixel_value'],\n            always_apply=True\n        ),\n        ToTensorV2(always_apply=True)\n    ])\n\n    transforms = {'train': train_transforms, 'val': val_transforms, 'test': test_transforms}\n    return transforms\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:22:00.811850Z","iopub.execute_input":"2022-09-10T04:22:00.812379Z","iopub.status.idle":"2022-09-10T04:22:00.840221Z","shell.execute_reply.started":"2022-09-10T04:22:00.812316Z","shell.execute_reply":"2022-09-10T04:22:00.839123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def horizontal_flip(x):\n    \n    \"\"\"\n    Flip given 2 dimensional array on x axis\n\n    Parameters\n    ----------\n    image (numpy.ndarray of shape (height, width, channel)): Image array\n\n    Returns\n    -------\n    image (numpy.ndarray of shape (height, width, channel)): Horizontally flipped image array\n    \"\"\"\n    \n    return np.flip(x, axis=1)\n\n\ndef vertical_flip(x):\n    \n    \"\"\"\n    Flip given 2 dimensional array on y axis\n\n    Parameters\n    ----------\n    image (numpy.ndarray of shape (height, width, channel)): Image array\n\n    Returns\n    -------\n    image (numpy.ndarray of shape (height, width, channel)): Vertically flipped image array\n    \"\"\"\n        \n    return np.flip(x, axis=0)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:22:01.685316Z","iopub.execute_input":"2022-09-10T04:22:01.685814Z","iopub.status.idle":"2022-09-10T04:22:01.691902Z","shell.execute_reply.started":"2022-09-10T04:22:01.685771Z","shell.execute_reply":"2022-09-10T04:22:01.690848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Model","metadata":{}},{"cell_type":"code","source":"class SemanticSegmentationModel(nn.Module):\n\n    def __init__(self, model_module, model_class, model_args):\n\n        super(SemanticSegmentationModel, self).__init__()\n\n        if model_module == 'smp':\n            self.model = getattr(smp, model_class)(**model_args)\n        else:\n            raise ValueError('Invalid model_module')\n\n    def forward(self, x):\n\n        return self.model(x)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:22:03.744460Z","iopub.execute_input":"2022-09-10T04:22:03.744955Z","iopub.status.idle":"2022-09-10T04:22:03.753419Z","shell.execute_reply.started":"2022-09-10T04:22:03.744896Z","shell.execute_reply":"2022-09-10T04:22:03.752436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HuggingFaceTransformersModel(nn.Module):\n\n    def __init__(self, model_class, model_args, upsample_args):\n\n        super(HuggingFaceTransformersModel, self).__init__()\n\n        self.model = getattr(transformers, model_class).from_pretrained(**model_args)\n        self.upsample = nn.Upsample(\n            size=upsample_args['upsample_size'],\n            mode=upsample_args['upsample_mode'],\n            align_corners=upsample_args['upsample_align_corners']\n        )\n\n    def forward(self, x):\n\n        return self.upsample(self.model(x)[0])\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:22:04.704562Z","iopub.execute_input":"2022-09-10T04:22:04.704991Z","iopub.status.idle":"2022-09-10T04:22:04.711766Z","shell.execute_reply.started":"2022-09-10T04:22:04.704948Z","shell.execute_reply":"2022-09-10T04:22:04.710511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CoaTDAFormer(nn.Module):\n\n    def __init__(self, encoder_name, decoder_args, encoder_weights=None):\n\n        super(CoaTDAFormer, self).__init__()\n\n        self.encoder = getattr(coat, encoder_name)()\n        if encoder_weights is not None:\n            self.encoder.load_state_dict(\n                state_dict=torch.load(encoder_weights)['model'],\n                strict=False\n            )\n        self.decoder = daformer.DAFormerDecoder(**decoder_args)\n        self.conv_head = nn.Sequential(\n            nn.Conv2d(self.decoder.decoder_dim, 1, kernel_size=1),\n            nn.Upsample(scale_factor=4, mode='bilinear', align_corners=False),\n        )\n\n    def forward(self, x):\n\n        x = self.encoder(x)\n        last, decoder = self.decoder(x)\n        out = self.conv_head(last)\n\n        return out\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:22:05.159951Z","iopub.execute_input":"2022-09-10T04:22:05.160927Z","iopub.status.idle":"2022-09-10T04:22:05.169528Z","shell.execute_reply.started":"2022-09-10T04:22:05.160887Z","shell.execute_reply":"2022-09-10T04:22:05.167662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(model_directory_path, folds=(1, 2, 3, 4, 5), verbose=False):\n    \n    \"\"\"\n    Load models and config file from the given directory path\n\n    Parameters\n    ----------\n    model_directory_path (str): Path of the model directory\n    verbose (bool): Verbosity flag\n    \n    Returns\n    -------\n    config (dict): Dictionary of model configurations\n    models (list): List of trained models\n    \"\"\"\n    \n    config = yaml.load(open(f'{model_directory_path}/config.yaml', 'r'), Loader=yaml.FullLoader)\n    \n    if config['model_parameters']['model_module'] == 'smp':\n        # Set encoder_weights to None if model module is segmentation_models_pytorch\n        config['model_parameters']['model_args']['encoder_weights'] = None\n    elif config['model_parameters']['model_module'] in 'transformers':\n        # Set pretrained_model_name_or_path to model_path if model module is transformers\n        pretrained_model_name_or_path = config['model_parameters']['model_args']['pretrained_model_name_or_path']\n        config['model_parameters']['model_args']['pretrained_model_name_or_path']  = f'../input/hugging-face-segformer/{pretrained_model_name_or_path}'\n    elif config['model_parameters']['model_module'] in 'coat_daformer':\n        # Set encoder_weights to None if model module is coat_daformer\n        config['model_parameters']['model_args']['encoder_weights'] = None\n    else:\n        raise ValueError('Invalid Model Module')\n        \n    model_paths = sorted(glob(f'{model_directory_path}/*.pt'))\n    models = {}\n    \n    for fold, model_path in enumerate(model_paths, 1):\n        \n        if fold not in folds:\n            continue\n        else:\n            if config['model_parameters']['model_module'] == 'smp':\n                model = SemanticSegmentationModel(\n                    config['model_parameters']['model_module'],\n                    config['model_parameters']['model_class'],\n                    config['model_parameters']['model_args']\n                )\n            elif config['model_parameters']['model_module'] == 'transformers':\n                model = HuggingFaceTransformersModel(\n                    config['model_parameters']['model_class'],\n                    config['model_parameters']['model_args'],\n                    config['model_parameters']['upsample_args']\n                )\n            elif config['model_parameters']['model_module'] == 'coat_daformer':\n                model = CoaTDAFormer(**config['model_parameters']['model_args'])\n            \n            model.load_state_dict(torch.load(model_path))\n            model = model.to(config['training_parameters']['device'])\n            model.eval()\n            models[fold] = model\n            print(f'Loaded pretrained weights from {model_path}')\n            \n    if verbose:\n        print(json.dumps(config, indent=2))\n        \n    return config, models\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:22:34.538491Z","iopub.execute_input":"2022-09-10T04:22:34.539245Z","iopub.status.idle":"2022-09-10T04:22:34.553003Z","shell.execute_reply.started":"2022-09-10T04:22:34.539180Z","shell.execute_reply":"2022-09-10T04:22:34.551453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unet_semantic_segmentation_raw_efficientnetb3_1088_config, unet_semantic_segmentation_raw_efficientnetb3_1088_models = load_model(\n    model_directory_path=f'{EXTERNAL_DATASET}/unet_semantic_segmentation_raw_efficientnetb3_1088',\n    folds=(1, 2, 3, 4, 5),\n    verbose=False\n)\n\nunet_semantic_segmentation_raw_efficientnetb6_768_config, unet_semantic_segmentation_raw_efficientnetb6_768_models = load_model(\n    model_directory_path=f'{EXTERNAL_DATASET}/unet_semantic_segmentation_raw_efficientnetb6_768',\n    folds=(1, 2, 3, 4, 5),\n    verbose=False\n)\n\nsegformer_semantic_segmentation_raw_mitb3_768_config, segformer_semantic_segmentation_raw_mitb3_768_models = load_model(\n    model_directory_path=f'{EXTERNAL_DATASET}/segformer_semantic_segmentation_raw_mitb3_768',\n    folds=(1, 2, 3, 4, 5),\n    verbose=False\n)\n\ncoat_daformer_semantic_segmentation_raw_coat_lite_small_1088_config, coat_daformer_semantic_segmentation_raw_coat_lite_small_1088_models = load_model(\n    model_directory_path=f'{EXTERNAL_DATASET}/coat_daformer_semantic_segmentation_raw_coat_lite_small_1088',\n    folds=(1, 2, 3, 4, 5),\n    verbose=False\n)\n\ncoat_daformer_semantic_segmentation_raw_coat_lite_medium_768_config, coat_daformer_semantic_segmentation_raw_coat_lite_medium_768_models = load_model(\n    model_directory_path=f'{EXTERNAL_DATASET}/coat_daformer_semantic_segmentation_raw_coat_lite_medium_768',\n    folds=(1, 2, 3, 4, 5),\n    verbose=False\n)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:27:51.626819Z","iopub.execute_input":"2022-09-10T04:27:51.627831Z","iopub.status.idle":"2022-09-10T04:29:20.786499Z","shell.execute_reply.started":"2022-09-10T04:27:51.627791Z","shell.execute_reply":"2022-09-10T04:29:20.785281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Evaluation","metadata":{}},{"cell_type":"code","source":"def soft_predictions_to_labels(x, threshold):\n\n    \"\"\"\n    Convert soft predictions into hard labels in given array\n\n    Parameters\n    ----------\n    x (array-like of any shape): Soft predictions array\n    threshold (float): Threshold for converting soft predictions into hard labels (0 <= threshold <= 1)\n\n    Returns\n    -------\n    x (array-like of any shape): Hard labels array\n    \"\"\"\n\n    if isinstance(x, torch.Tensor):\n        x = x.numpy()\n    else:\n        x = np.array(x)\n\n    x = np.uint8(x >= threshold)\n\n    return x\n\n\ndef binary_dice_coefficient(ground_truth, predictions, threshold=0.5, eps=0.00001):\n\n    \"\"\"\n    Calculate dice coefficient on given ground truth and predictions arrays\n\n    Parameters\n    ----------\n    ground_truth (array-like of shape (batch_size, height, width) or (height, width)): Ground truth array\n    predictions (array-like of shape (batch_size, height, width) or (height, width)): Predictions array\n    threshold (float): Threshold for converting soft predictions into hard labels (0 <= threshold <= 1)\n    eps (float): A small number for avoiding division by zero\n\n    Returns\n    -------\n    dice_coefficient (float): Calculated dice coefficient\n    \"\"\"\n\n    if isinstance(ground_truth, torch.Tensor):\n        ground_truth = ground_truth.numpy().astype(np.float32)\n    else:\n        ground_truth = np.array(ground_truth).astype(np.float32)\n\n    if isinstance(predictions, torch.Tensor):\n        predictions = predictions.numpy().astype(np.float32)\n    else:\n        predictions = np.array(predictions).astype(np.float32)\n\n    if len(ground_truth.shape) != len(predictions.shape):\n        raise ValueError('Shape mismatch')\n\n    predictions = soft_predictions_to_labels(x=predictions, threshold=threshold).astype(np.float32)\n\n    if len(ground_truth.shape) == 2:\n        # Calculate dice coefficient for single data point\n        intersection = np.sum(ground_truth * predictions)\n        denominator = np.sum(ground_truth) + np.sum(predictions)\n        dice_coefficient = (2 * intersection + eps) / (denominator + eps)\n    elif len(ground_truth.shape) == 3:\n        # Calculate dice coefficient for batch of data points (assuming first dimension is batch)\n        intersection = np.sum(ground_truth * predictions, axis=(1, 2))\n        denominator = np.sum(ground_truth, axis=(1, 2)) + np.sum(predictions, axis=(1, 2))\n        dice_coefficient = np.mean((2 * intersection + eps) / (denominator + eps))\n    else:\n        raise ValueError('Invalid shape')\n\n    return float(dice_coefficient)\n\n\ndef binary_intersection_over_union(ground_truth, predictions, threshold=0.5, eps=0.00001):\n\n    \"\"\"\n    Calculate intersection over union on given ground truth and predictions arrays\n\n    Parameters\n    ----------\n    ground_truth (array-like of shape (batch_size, height, width) or (height, width)): Ground truth array\n    predictions (array-like of shape (batch_size, height, width) or (height, width)): Predictions array\n    threshold (float): Threshold for converting soft predictions into hard labels (0 <= threshold <= 1)\n    eps (float): A small number for avoiding division by zero\n\n    Returns\n    -------\n    intersection_over_union (float): Calculated intersection over union\n    \"\"\"\n\n    if isinstance(ground_truth, torch.Tensor):\n        ground_truth = ground_truth.numpy().astype(np.float32)\n    else:\n        ground_truth = np.array(ground_truth).astype(np.float32)\n\n    if isinstance(predictions, torch.Tensor):\n        predictions = predictions.numpy().astype(np.float32)\n    else:\n        predictions = np.array(predictions).astype(np.float32)\n\n    if len(ground_truth.shape) != len(predictions.shape):\n        raise ValueError('Shape mismatch')\n\n    predictions = soft_predictions_to_labels(x=predictions, threshold=threshold).astype(np.float32)\n\n    if len(ground_truth.shape) == 2:\n        # Calculate intersection over union for single data point\n        intersection = np.sum(ground_truth * predictions)\n        union = np.sum(ground_truth + predictions - ground_truth * predictions)\n        intersection_over_union = (intersection + eps) / (union + eps)\n    elif len(ground_truth.shape) == 3:\n        # Calculate intersection over union for batch of data points (assuming first dimension is batch)\n        intersection = np.sum(ground_truth * predictions, axis=(1, 2))\n        union = np.sum(ground_truth + predictions - ground_truth * predictions, axis=(1, 2))\n        intersection_over_union = np.mean((intersection + eps) / (union + eps))\n    else:\n        raise ValueError('Invalid shape')\n\n    return float(intersection_over_union)\n\n\ndef mean_binary_dice_coefficient(ground_truth, predictions, thresholds=(0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7), eps=0.00001):\n\n    \"\"\"\n    Calculate dice coefficients on given ground truth and predictions arrays using different thresholds and average them\n\n    Parameters\n    ----------\n    ground_truth (array-like of shape (batch_size, height, width) or (height, width)): Ground truth array\n    predictions (array-like of shape (batch_size, height, width) or (height, width)): Predictions array\n    thresholds (tuple of shape (n_thresholds)): Thresholds for converting soft predictions into hard labels (0 <= threshold <= 1)\n    eps (float): A small number for avoiding division by zero\n\n    Returns\n    -------\n    dice_coefficients (dict): Calculated dice coefficients using given thresholds\n    mean_dice_coefficient (float): Average of calculated dice coefficients\n    \"\"\"\n\n    dice_coefficients = {}\n    for threshold in thresholds:\n        dice_coefficients[threshold] = binary_dice_coefficient(ground_truth=ground_truth, predictions=predictions, threshold=threshold, eps=eps)\n\n    mean_dice_coefficient = float(np.mean(list(dice_coefficients.values())))\n\n    return dice_coefficients, mean_dice_coefficient\n\n\ndef mean_binary_intersection_over_union(ground_truth, predictions, thresholds=(0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7), eps=0.00001):\n\n    \"\"\"\n    Calculate intersection over unions on given ground truth and predictions arrays using different thresholds and average them\n\n    Parameters\n    ----------\n    ground_truth (array-like of shape (batch_size, height, width) or (height, width)): Ground truth array\n    predictions (array-like of shape (batch_size, height, width) or (height, width)): Predictions array\n    thresholds (tuple of shape (n_thresholds)): Thresholds for converting soft predictions into hard labels (0 <= threshold <= 1)\n    eps (float): A small number for avoiding division by zero\n\n    Returns\n    -------\n    intersection_over_unions (dict): Calculated intersection over unions using given thresholds\n    mean_intersection_over_union (float): Average of calculated intersection over unions\n    \"\"\"\n\n    intersection_over_unions = {}\n    for threshold in thresholds:\n        intersection_over_unions[threshold] = binary_intersection_over_union(ground_truth=ground_truth, predictions=predictions, threshold=threshold, eps=eps)\n\n    mean_intersection_over_union = float(np.mean(list(intersection_over_unions.values())))\n\n    return intersection_over_unions, mean_intersection_over_union\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:31:28.648027Z","iopub.execute_input":"2022-09-10T04:31:28.648412Z","iopub.status.idle":"2022-09-10T04:31:28.672565Z","shell.execute_reply.started":"2022-09-10T04:31:28.648377Z","shell.execute_reply":"2022-09-10T04:31:28.671273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_spatial_properties(mask):\n\n    \"\"\"\n    Extract spatial properties from given mask\n\n    Parameters\n    ----------\n    mask (numpy.ndarray of shape (height, width)): 2d segmentation mask\n\n    Returns\n    -------\n    spatial_properties (dict): Dictionary of spatial properties extracted from the given mask\n    \"\"\"\n\n    contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n    n_objects = len(contours)\n    if n_objects > 0:\n        object_areas = [cv2.contourArea(contour) for contour in contours]\n        mean_object_area = np.mean(object_areas)\n        median_object_area = np.median(object_areas)\n        min_object_area = np.min(object_areas)\n        max_object_area = np.max(object_areas)\n    else:\n        mean_object_area = 0\n        median_object_area = 0\n        min_object_area = 0\n        max_object_area = 0\n\n    spatial_properties = {\n        'n_objects': n_objects,\n        'mean_object_area': float(mean_object_area),\n        'median_object_area': int(median_object_area),\n        'min_object_area': int(min_object_area),\n        'max_object_area': int(max_object_area),\n    }\n\n    return spatial_properties\n\n\ndef evaluate_predictions(ground_truth, predictions, threshold, thresholds=(0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7)):\n\n    \"\"\"\n    Evaluate predictions and ground-truth if it is given\n\n    Parameters\n    ----------\n    ground_truth (array-like of shape (height, width)): Ground truth array\n    predictions (array-like of shape (height, width)): Predictions array\n    threshold (float): Threshold for converting soft predictions into hard labels (0 <= threshold <= 1)\n    thresholds (tuple of shape (n_thresholds)): Thresholds for converting soft predictions into hard labels (0 <= threshold <= 1)\n\n    Returns\n    -------\n    evaluation_summary (dict): Dictionary of evaluation summary\n    \"\"\"\n\n    if ground_truth is not None:\n        dice_coefficient = binary_dice_coefficient(ground_truth=ground_truth, predictions=predictions, threshold=threshold)\n        dice_coefficients = mean_binary_dice_coefficient(ground_truth=ground_truth, predictions=predictions, thresholds=thresholds)\n        intersection_over_union = binary_intersection_over_union(ground_truth=ground_truth, predictions=predictions, threshold=threshold)\n        intersection_over_unions = mean_binary_intersection_over_union(ground_truth=ground_truth, predictions=predictions, thresholds=thresholds)\n\n        scores = {\n            'dice_coefficient': dice_coefficient,\n            'dice_coefficients': dice_coefficients[0],\n            'mean_dice_coefficient': dice_coefficients[1],\n            'intersection_over_union': intersection_over_union,\n            'intersection_over_unions': intersection_over_unions[0],\n            'mean_intersection_over_union': intersection_over_unions[1],\n            'threshold': threshold,\n            'thresholds': thresholds\n        }\n    else:\n        scores = None\n\n    evaluation_summary = {\n        'scores': scores,\n        'statistics':  {\n            'ground_truth': {\n                'mean': float(np.mean(ground_truth)) if ground_truth is not None else None,\n                'sum': int(np.sum(ground_truth)) if ground_truth is not None else None\n            },\n            'predictions': {\n                'mean': float(np.mean(predictions)),\n                'sum': int(np.sum(predictions))\n            }\n        },\n        'spatial_properties': {\n            'ground_truth': extract_spatial_properties(ground_truth) if ground_truth is not None else None,\n            'predictions': extract_spatial_properties(np.uint8(predictions >= threshold))\n        }\n    }\n\n    return evaluation_summary\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:31:31.015941Z","iopub.execute_input":"2022-09-10T04:31:31.016534Z","iopub.status.idle":"2022-09-10T04:31:31.038780Z","shell.execute_reply.started":"2022-09-10T04:31:31.016499Z","shell.execute_reply":"2022-09-10T04:31:31.037726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_predictions(image, ground_truth, predictions, metadata, evaluation_summary, path=None):\n\n    \"\"\"\n    Visualize image along with its annotations and predictions\n\n    Parameters\n    ----------\n    image (numpy.ndarray of shape (height, width)): Image array\n    ground_truth (numpy.ndarray of shape (height, width)): Ground-truth mask array\n    predictions (numpy.ndarray of shape (height, width)): Predictions mask array\n    metadata (dict): Dictionary of metadata used in the visualization title\n    evaluation_summary (dict): Dictionary of evaluation summary used in the visualization title\n    path (path-like str or None): Path of the output file or None (if path is None, plot is displayed with selected backend)\n    \"\"\"\n\n    if ground_truth is not None:\n        dice_coefficient = evaluation_summary['scores']['dice_coefficient']\n        intersection_over_union = evaluation_summary['scores']['intersection_over_union']\n        ground_truth_evaluation = f'Mean: {evaluation_summary[\"statistics\"][\"ground_truth\"][\"mean\"]:4f} - Sum: {evaluation_summary[\"statistics\"][\"ground_truth\"][\"sum\"]} - Object Count: {evaluation_summary[\"spatial_properties\"][\"ground_truth\"][\"n_objects\"]}'\n    else:\n        dice_coefficient = 0\n        intersection_over_union = 0\n        ground_truth_evaluation = ''\n\n    predictions_evaluation = f'Mean: {evaluation_summary[\"statistics\"][\"predictions\"][\"mean\"]:4f} - Sum: {evaluation_summary[\"statistics\"][\"predictions\"][\"sum\"]} - Object Count: {evaluation_summary[\"spatial_properties\"][\"predictions\"][\"n_objects\"]}'\n\n    if isinstance(image, np.ndarray) is False:\n        # Raise TypeError if image argument is not an array-like object\n        raise TypeError('Image is not an array')\n\n    fig, axes = plt.subplots(figsize=(32, 20), ncols=2)\n\n    axes[0].imshow(image)\n    if ground_truth is not None:\n        axes[0].imshow(ground_truth, alpha=0.5)\n    axes[1].imshow(image)\n    axes[1].imshow(predictions, alpha=0.5)\n\n    for i in range(2):\n        axes[i].set_xlabel('')\n        axes[i].set_ylabel('')\n        axes[i].tick_params(axis='x', labelsize=15, pad=10)\n        axes[i].tick_params(axis='y', labelsize=15, pad=10)\n\n    axes[0].set_title('Image + Ground-truth\\n' + ground_truth_evaluation, size=25, pad=15)\n    axes[1].set_title('Image + Predictions\\n' + predictions_evaluation, size=25, pad=15)\n    fig.suptitle(\n        f'''\n        Image ID {metadata[\"id\"]} - {metadata[\"organ\"]} - {metadata[\"data_source\"]} - {metadata[\"age\"]} - {metadata[\"sex\"]}\n        Image Shape: {metadata[\"img_height\"]}x{metadata[\"img_width\"]} - Pixel Size: {metadata[\"pixel_size\"]}µm - Tissue Thickness: {metadata[\"tissue_thickness\"]}µm\n        Dice Coefficient: {dice_coefficient:.4f} - Intersection over Union: {intersection_over_union:.4f}\n        ''',\n        fontsize=30\n    )\n\n    if path is None:\n        plt.show()\n    else:\n        plt.savefig(path)\n        plt.close(fig)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:31:40.329293Z","iopub.execute_input":"2022-09-10T04:31:40.329680Z","iopub.status.idle":"2022-09-10T04:31:40.343531Z","shell.execute_reply.started":"2022-09-10T04:31:40.329640Z","shell.execute_reply":"2022-09-10T04:31:40.342417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7. Inference","metadata":{}},{"cell_type":"code","source":"unet_semantic_segmentation_raw_efficientnetb3_1088_config['inference_parameters']['size'] = {\n    'HPA': {\n        'kidney': [1088, 1088],\n        'prostate': [1088, 1088],\n        'spleen': [1088, 1088],\n        'largeintestine': [1088, 1088],\n        'lung': [1088, 1088]\n    },\n    'Hubmap': {\n        'kidney': [1088, 1088],\n        'prostate': [1088, 1088],\n        'spleen': [1088, 1088],\n        'largeintestine': [1088, 1088],\n        'lung': [1088, 1088]\n    }\n}\n\nunet_semantic_segmentation_raw_efficientnetb6_768_config['inference_parameters']['size'] = {\n    'HPA': {\n        'kidney': [768, 768],\n        'prostate': [768, 768],\n        'spleen': [768, 768],\n        'largeintestine': [768, 768],\n        'lung': [768, 768]\n    },\n    'Hubmap': {\n        'kidney': [768, 768],\n        'prostate': [768, 768],\n        'spleen': [768, 768],\n        'largeintestine': [768, 768],\n        'lung': [768, 768]\n    }\n}\n\nsegformer_semantic_segmentation_raw_mitb3_768_config['inference_parameters']['size'] = {\n    'HPA': {\n        'kidney': [768, 768],\n        'prostate': [768, 768],\n        'spleen': [768, 768],\n        'largeintestine': [768, 768],\n        'lung': [768, 768]\n    },\n    'Hubmap': {\n        'kidney': [768, 768],\n        'prostate': [768, 768],\n        'spleen': [768, 768],\n        'largeintestine': [768, 768],\n        'lung': [768, 768]\n    }\n}\n\ncoat_daformer_semantic_segmentation_raw_coat_lite_small_1088_config['inference_parameters']['size'] = {\n    'HPA': {\n        'kidney': [1088, 1088],\n        'prostate': [1088, 1088],\n        'spleen': [1088, 1088],\n        'largeintestine': [1088, 1088],\n        'lung': [1088, 1088]\n    },\n    'Hubmap': {\n        'kidney': [1088, 1088],\n        'prostate': [1088, 1088],\n        'spleen': [1088, 1088],\n        'largeintestine': [1088, 1088],\n        'lung': [1088, 1088]\n    }\n}\n\ncoat_daformer_semantic_segmentation_raw_coat_lite_medium_768_config['inference_parameters']['size'] = {\n    'HPA': {\n        'kidney': [768, 768],\n        'prostate': [768, 768],\n        'spleen': [768, 768],\n        'largeintestine': [768, 768],\n        'lung': [768, 768]\n    },\n    'Hubmap': {\n        'kidney': [768, 768],\n        'prostate': [768, 768],\n        'spleen': [768, 768],\n        'largeintestine': [768, 768],\n        'lung': [768, 768]\n    }\n}\n\nconfigs = [\n    unet_semantic_segmentation_raw_efficientnetb3_1088_config,\n    unet_semantic_segmentation_raw_efficientnetb6_768_config,\n    segformer_semantic_segmentation_raw_mitb3_768_config,\n    coat_daformer_semantic_segmentation_raw_coat_lite_small_1088_config,\n    coat_daformer_semantic_segmentation_raw_coat_lite_medium_768_config\n]\n\nmodels = [\n    unet_semantic_segmentation_raw_efficientnetb3_1088_models,\n    unet_semantic_segmentation_raw_efficientnetb6_768_models,\n    segformer_semantic_segmentation_raw_mitb3_768_models,\n    coat_daformer_semantic_segmentation_raw_coat_lite_small_1088_models,\n    coat_daformer_semantic_segmentation_raw_coat_lite_medium_768_models\n]\n\nweights = {\n    'HPA': {\n        'kidney': [0.20, 0.25, 0.10, 0.20, 0.25],\n        'prostate': [0.20, 0.25, 0.15, 0.15, 0.25],\n        'spleen': [0.15, 0.25, 0.15, 0.20, 0.25],\n        'largeintestine': [0.20, 0.25, 0.10, 0.20, 0.25],\n        'lung': [0.20, 0.70, 0.0, 0.0, 0.10]\n    },\n    'Hubmap': {\n        'kidney': [0.20, 0.25, 0.10, 0.20, 0.25],\n        'prostate': [0.20, 0.25, 0.15, 0.15, 0.25],\n        'spleen': [0.15, 0.25, 0.15, 0.20, 0.25],\n        'largeintestine': [0.20, 0.25, 0.10, 0.20, 0.25],\n        'lung': [0.20, 0.70, 0.0, 0.0, 0.10]\n    }\n}\n\n\nlabel_thresholds = {\n    'HPA': {\n        'kidney': 0.5,\n        'prostate': 0.5,\n        'spleen': 0.5,\n        'largeintestine': 0.5,\n        'lung': 0.05\n    },\n    'Hubmap': {\n        'kidney': 0.25,\n        'prostate': 0.21,\n        'spleen': 0.25,\n        'largeintestine': 0.20,\n        'lung': 0.05\n    }\n}","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:32:51.652575Z","iopub.execute_input":"2022-09-10T04:32:51.652943Z","iopub.status.idle":"2022-09-10T04:32:51.686453Z","shell.execute_reply.started":"2022-09-10T04:32:51.652910Z","shell.execute_reply":"2022-09-10T04:32:51.685386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select one sample from each organ type in first fold\nnp.random.seed(42)\ndf_train_sample = df_train.loc[df_train['fold1'] == 1].groupby('organ').sample(1).reset_index(drop=True)\n\ndataset_transforms = get_semantic_segmentation_transforms(**unet_semantic_segmentation_raw_efficientnetb6_768_config['transform_parameters'])","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:32:52.343361Z","iopub.execute_input":"2022-09-10T04:32:52.344229Z","iopub.status.idle":"2022-09-10T04:32:52.364580Z","shell.execute_reply.started":"2022-09-10T04:32:52.344192Z","shell.execute_reply":"2022-09-10T04:32:52.363573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if df_test.shape[0] == 1:\n\n    for idx, row in df_train_sample.iterrows():\n\n        if row['image_filename'].split('.')[-1] == 'tiff' or row['image_filename'].split('.')[-1] == 'tif':\n            image = tifffile.imread(row['image_filename'])\n        else:\n            image = cv2.imread(row['image_filename'])\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            \n        predictions_mask_ensemble = np.zeros(image.shape[:2])\n        \n        for config, models_, weight in zip(configs, models, weights[row['data_source']][row['organ']]):\n\n            image_resized = cv2.resize(\n                image,\n                dsize=config['inference_parameters']['size'][row['data_source']][row['organ']],\n                interpolation=cv2.INTER_CUBIC\n            )\n\n            inputs = [\n                image_resized,\n                horizontal_flip(image_resized),\n                vertical_flip(image_resized),\n                horizontal_flip(vertical_flip(image_resized))\n            ]\n            inputs = torch.cat([\n                torch.unsqueeze(dataset_transforms['test'](image=image)['image'], dim=0)\n                for image in inputs\n            ], dim=0)\n            inputs = inputs.to('cuda')\n\n            with torch.no_grad():\n                outputs = models_[1](inputs)\n\n            outputs = outputs.detach().cpu()\n            predictions_mask = torch.sigmoid(torch.squeeze(outputs, dim=1)).numpy().astype(np.float32)\n\n            # Apply inverse of test-time augmentations and aggregate predictions\n            predictions_mask[1, :, :] = horizontal_flip(predictions_mask[1, :, :])\n            predictions_mask[2, :, :] = vertical_flip(predictions_mask[2, :, :])\n            predictions_mask[3, :, :] = horizontal_flip(vertical_flip(predictions_mask[3, :, :]))\n            predictions_mask = np.mean(predictions_mask, axis=0)\n            predictions_mask = cv2.resize(predictions_mask, (image.shape[1], image.shape[0]), interpolation=cv2.INTER_CUBIC)\n            predictions_mask_ensemble += (predictions_mask * weight)\n\n        try:\n            label_threshold = label_thresholds[row['data_source']][row['organ']]\n        except KeyError:\n            # Set label threshold to 0.1 for unseen organs or data sources\n            label_threshold = 0.1\n\n        ground_truth_mask = decode_rle_mask(row['rle'], shape=image.shape[:2]).T\n        predictions_evaluation_summary = evaluate_predictions(\n            ground_truth=ground_truth_mask,\n            predictions=predictions_mask_ensemble,\n            threshold=label_threshold,\n            thresholds=[0.5]\n        )\n        predictions_mask_ensemble = np.uint8(predictions_mask_ensemble >= label_threshold)\n        visualize_predictions(\n            image=image,\n            ground_truth=ground_truth_mask,\n            predictions=predictions_mask_ensemble,\n            metadata=row.to_dict(),\n            evaluation_summary=predictions_evaluation_summary\n        )\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:32:52.699143Z","iopub.execute_input":"2022-09-10T04:32:52.700107Z","iopub.status.idle":"2022-09-10T04:33:34.894194Z","shell.execute_reply.started":"2022-09-10T04:32:52.700059Z","shell.execute_reply":"2022-09-10T04:33:34.893275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_hpa = False\npredict_hubmap = True\n\nfor test_idx, row in df_test.iterrows():\n    \n    test_image = tifffile.imread(row['image_filename'])\n    \n    # Fill row with empty string if predict_hpa is False\n    if (row['data_source'] == 'HPA') and (predict_hpa is False):\n        df_test.loc[test_idx, 'rle'] = ''\n        continue\n    \n    # Fill row with empty string if predict_hubmap is False\n    if (row['data_source'] == 'Hubmap') and (predict_hubmap is False):\n        df_test.loc[test_idx, 'rle'] = ''\n        continue\n        \n    predictions_mask_ensemble = np.zeros(test_image.shape[:2])\n    \n    if row['data_source'] == 'Hubmap':\n        # Randomly select an image from training set for stain normalization\n        np.random.seed(42)\n        train_images = df_train.loc[df_train['organ'] == row['organ'], 'image_filename'].values\n        train_image = tifffile.imread(np.random.choice(train_images))\n        train_image = staintools.LuminosityStandardizer.standardize(train_image)\n        test_image_normalized = staintools.LuminosityStandardizer.standardize(test_image)\n\n        # Normalize stain of HuBMAP images\n        vahadane_normalizer = staintools.StainNormalizer(method='vahadane')\n        vahadane_normalizer.fit(train_image)\n        test_image_normalized = vahadane_normalizer.transform(test_image_normalized)\n        \n        for config, models_, weight in zip(configs, models, weights[row['data_source']][row['organ']]):\n            \n            test_image_resized = cv2.resize(\n                test_image,\n                dsize=config['inference_parameters']['size'][row['data_source']][row['organ']],\n                interpolation=cv2.INTER_CUBIC\n            )\n            test_image_normalized_resized = cv2.resize(\n                test_image_normalized,\n                dsize=config['inference_parameters']['size'][row['data_source']][row['organ']],\n                interpolation=cv2.INTER_CUBIC\n            )\n\n            inputs = [\n                test_image_resized,\n                horizontal_flip(test_image_resized),\n                vertical_flip(test_image_resized),\n                horizontal_flip(vertical_flip(test_image_resized)),\n                test_image_normalized_resized,\n                horizontal_flip(test_image_normalized_resized),\n                vertical_flip(test_image_normalized_resized),\n                horizontal_flip(vertical_flip(test_image_normalized_resized)),\n            ]\n            inputs = torch.cat([\n                torch.unsqueeze(dataset_transforms['test'](image=image)['image'], dim=0)\n                for image in inputs\n            ], dim=0)\n            inputs = inputs.to('cuda')\n\n            predictions_mask = np.zeros((\n                len(inputs),\n                test_image_resized.shape[0],\n                test_image_resized.shape[1]\n            ), dtype=np.float32)\n\n            for fold, model in models_.items():\n                with torch.no_grad():\n                    outputs = models_[fold](inputs)\n\n                fold_predictions_mask = outputs.detach().cpu()\n                fold_predictions_mask = torch.sigmoid(torch.squeeze(fold_predictions_mask, dim=1)).numpy().astype(np.float32)\n                predictions_mask += (fold_predictions_mask / len(models_))\n\n            # Apply inverse of test-time augmentations and aggregate predictions\n            predictions_mask[1, :, :] = horizontal_flip(predictions_mask[1, :, :])\n            predictions_mask[2, :, :] = vertical_flip(predictions_mask[2, :, :])\n            predictions_mask[3, :, :] = horizontal_flip(vertical_flip(predictions_mask[3, :, :]))\n            predictions_mask[5, :, :] = horizontal_flip(predictions_mask[5, :, :])\n            predictions_mask[6, :, :] = vertical_flip(predictions_mask[6, :, :])\n            predictions_mask[7, :, :] = horizontal_flip(vertical_flip(predictions_mask[7, :, :]))\n            predictions_mask = np.mean(predictions_mask, axis=0)\n            predictions_mask = cv2.resize(predictions_mask, (test_image.shape[1], test_image.shape[0]), interpolation=cv2.INTER_CUBIC)\n            predictions_mask_ensemble += (predictions_mask * weight)\n        \n    elif row['data_source'] == 'HPA':\n        \n        for config, models_, weight in zip(configs, models, weights[row['data_source']][row['organ']]):\n\n            test_image_resized = cv2.resize(\n                test_image,\n                dsize=config['inference_parameters']['size'][row['data_source']][row['organ']],\n                interpolation=cv2.INTER_CUBIC\n            )\n\n            inputs = [\n                test_image_resized,\n                horizontal_flip(test_image_resized),\n                vertical_flip(test_image_resized),\n                horizontal_flip(vertical_flip(test_image_resized)),\n            ]\n            inputs = torch.cat([\n                torch.unsqueeze(dataset_transforms['test'](image=image)['image'], dim=0)\n                for image in inputs\n            ], dim=0)\n            inputs = inputs.to('cuda')\n\n            predictions_mask = np.zeros((\n                len(inputs),\n                test_image_resized.shape[0],\n                test_image_resized.shape[1]\n            ), dtype=np.float32)\n\n            for fold, model in models_.items():\n                with torch.no_grad():\n                    outputs = models_[fold](inputs)\n\n                fold_predictions_mask = outputs.detach().cpu()\n                fold_predictions_mask = torch.sigmoid(torch.squeeze(fold_predictions_mask, dim=1)).numpy().astype(np.float32)\n                predictions_mask += (fold_predictions_mask / len(models_))\n\n            # Apply inverse of test-time augmentations and aggregate predictions\n            predictions_mask[1, :, :] = horizontal_flip(predictions_mask[1, :, :])\n            predictions_mask[2, :, :] = vertical_flip(predictions_mask[2, :, :])\n            predictions_mask[3, :, :] = horizontal_flip(vertical_flip(predictions_mask[3, :, :]))\n            predictions_mask = np.mean(predictions_mask, axis=0)\n            predictions_mask = cv2.resize(predictions_mask, (test_image.shape[1], test_image.shape[0]), interpolation=cv2.INTER_CUBIC)\n            predictions_mask_ensemble += (predictions_mask * weight)\n\n    if df_test.shape[0] == 1:\n        \n        predictions_evaluation_summary = evaluate_predictions(\n            ground_truth=None,\n            predictions=predictions_mask_ensemble,\n            threshold=label_thresholds[row['data_source']][row['organ']],\n            thresholds=[0.5]\n        )\n        predictions_mask_ensemble = np.uint8(predictions_mask_ensemble >= label_thresholds[row['data_source']][row['organ']])\n        visualize_predictions(\n            image=test_image,\n            ground_truth=None,\n            predictions=predictions_mask_ensemble,\n            metadata=row.to_dict(),\n            evaluation_summary=predictions_evaluation_summary\n        )\n        \n    else:\n        predictions_mask_ensemble = np.uint8(predictions_mask_ensemble >= label_thresholds[row['data_source']][row['organ']])\n        \n    df_test.loc[test_idx, 'rle'] = encode_rle_mask(predictions_mask_ensemble)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T04:34:26.308718Z","iopub.execute_input":"2022-09-10T04:34:26.309113Z","iopub.status.idle":"2022-09-10T04:34:52.569154Z","shell.execute_reply.started":"2022-09-10T04:34:26.309078Z","shell.execute_reply":"2022-09-10T04:34:52.568221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8. Submission","metadata":{}},{"cell_type":"code","source":"df_test[['id', 'rle']].to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}