{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import json\nimport os\nimport numpy as np\nimport PIL.Image\nimport cv2\nimport matplotlib.pyplot as plt\n\nimport pandas as pd\n\nimport os\nimport cv2\nimport time\nimport random\n\nimport torch\nfrom torch import nn\nimport torch.cuda.amp as amp\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import RandomSampler \nfrom torch.utils.data import SequentialSampler\nimport torch.nn.functional as F\nfrom torchmetrics.functional import dice_score\nfrom torch.optim.lr_scheduler import StepLR\nimport torch.nn as nn\nfrom functools import partial\n\nis_amp = True\nimport logging\nimport pandas as pd\nfrom sklearn.model_selection import KFold\n\nimport numpy as np\nfrom itertools import repeat\nimport collections.abc\nimport math\nimport warnings\nfrom torch.nn.init import _calculate_fan_in_and_fan_out\nfrom itertools import repeat\nimport collections.abc","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:01:49.47521Z","iopub.execute_input":"2022-08-22T08:01:49.476335Z","iopub.status.idle":"2022-08-22T08:01:53.594218Z","shell.execute_reply.started":"2022-08-22T08:01:49.476291Z","shell.execute_reply":"2022-08-22T08:01:53.59275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model function definition","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\ndef trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):\n    # type: (Tensor, float, float, float, float) -> Tensor\n    r\"\"\"Fills the input Tensor with values drawn from a truncated\n    normal distribution. The values are effectively drawn from the\n    normal distribution :math:`\\mathcal{N}(\\text{mean}, \\text{std}^2)`\n    with values outside :math:`[a, b]` redrawn until they are within\n    the bounds. The method used for generating the random values works\n    best when :math:`a \\leq \\text{mean} \\leq b`.\n    NOTE: this impl is similar to the PyTorch trunc_normal_, the bounds [a, b] are\n    applied while sampling the normal with mean/std applied, therefore a, b args\n    should be adjusted to match the range of mean, std args.\n    Args:\n        tensor: an n-dimensional `torch.Tensor`\n        mean: the mean of the normal distribution\n        std: the standard deviation of the normal distribution\n        a: the minimum cutoff value\n        b: the maximum cutoff value\n    Examples:\n        >>> w = torch.empty(3, 5)\n        >>> nn.init.trunc_normal_(w)\n    \"\"\"\n    return _no_grad_trunc_normal_(tensor, mean, std, a, b)\n\n\n\ndef drop_block_2d(\n        x, drop_prob: float = 0.1, block_size: int = 7, gamma_scale: float = 1.0,\n        with_noise: bool = False, inplace: bool = False, batchwise: bool = False):\n    \"\"\" DropBlock. See https://arxiv.org/pdf/1810.12890.pdf\n    DropBlock with an experimental gaussian noise option. This layer has been tested on a few training\n    runs with success, but needs further validation and possibly optimization for lower runtime impact.\n    \"\"\"\n    B, C, H, W = x.shape\n    total_size = W * H\n    clipped_block_size = min(block_size, min(W, H))\n    # seed_drop_rate, the gamma parameter\n    gamma = gamma_scale * drop_prob * total_size / clipped_block_size ** 2 / (\n            (W - block_size + 1) * (H - block_size + 1))\n\n    # Forces the block to be inside the feature map.\n    w_i, h_i = torch.meshgrid(torch.arange(W).to(x.device), torch.arange(H).to(x.device))\n    valid_block = ((w_i >= clipped_block_size // 2) & (w_i < W - (clipped_block_size - 1) // 2)) & \\\n                  ((h_i >= clipped_block_size // 2) & (h_i < H - (clipped_block_size - 1) // 2))\n    valid_block = torch.reshape(valid_block, (1, 1, H, W)).to(dtype=x.dtype)\n\n    if batchwise:\n        # one mask for whole batch, quite a bit faster\n        uniform_noise = torch.rand((1, C, H, W), dtype=x.dtype, device=x.device)\n    else:\n        uniform_noise = torch.rand_like(x)\n    block_mask = ((2 - gamma - valid_block + uniform_noise) >= 1).to(dtype=x.dtype)\n    block_mask = -F.max_pool2d(\n        -block_mask,\n        kernel_size=clipped_block_size,  # block_size,\n        stride=1,\n        padding=clipped_block_size // 2)\n\n    if with_noise:\n        normal_noise = torch.randn((1, C, H, W), dtype=x.dtype, device=x.device) if batchwise else torch.randn_like(x)\n        if inplace:\n            x.mul_(block_mask).add_(normal_noise * (1 - block_mask))\n        else:\n            x = x * block_mask + normal_noise * (1 - block_mask)\n    else:\n        normalize_scale = (block_mask.numel() / block_mask.to(dtype=torch.float32).sum().add(1e-7)).to(x.dtype)\n        if inplace:\n            x.mul_(block_mask * normalize_scale)\n        else:\n            x = x * block_mask * normalize_scale\n    return x\n\n\ndef drop_block_fast_2d(\n        x: torch.Tensor, drop_prob: float = 0.1, block_size: int = 7,\n        gamma_scale: float = 1.0, with_noise: bool = False, inplace: bool = False):\n    \"\"\" DropBlock. See https://arxiv.org/pdf/1810.12890.pdf\n    DropBlock with an experimental gaussian noise option. Simplied from above without concern for valid\n    block mask at edges.\n    \"\"\"\n    B, C, H, W = x.shape\n    total_size = W * H\n    clipped_block_size = min(block_size, min(W, H))\n    gamma = gamma_scale * drop_prob * total_size / clipped_block_size ** 2 / (\n            (W - block_size + 1) * (H - block_size + 1))\n\n    block_mask = torch.empty_like(x).bernoulli_(gamma)\n    block_mask = F.max_pool2d(\n        block_mask.to(x.dtype), kernel_size=clipped_block_size, stride=1, padding=clipped_block_size // 2)\n\n    if with_noise:\n        normal_noise = torch.empty_like(x).normal_()\n        if inplace:\n            x.mul_(1. - block_mask).add_(normal_noise * block_mask)\n        else:\n            x = x * (1. - block_mask) + normal_noise * block_mask\n    else:\n        block_mask = 1 - block_mask\n        normalize_scale = (block_mask.numel() / block_mask.to(dtype=torch.float32).sum().add(1e-6)).to(dtype=x.dtype)\n        if inplace:\n            x.mul_(block_mask * normalize_scale)\n        else:\n            x = x * block_mask * normalize_scale\n    return x\n\n\nclass DropBlock2d(nn.Module):\n    \"\"\" DropBlock. See https://arxiv.org/pdf/1810.12890.pdf\n    \"\"\"\n\n    def __init__(\n            self,\n            drop_prob: float = 0.1,\n            block_size: int = 7,\n            gamma_scale: float = 1.0,\n            with_noise: bool = False,\n            inplace: bool = False,\n            batchwise: bool = False,\n            fast: bool = True):\n        super(DropBlock2d, self).__init__()\n        self.drop_prob = drop_prob\n        self.gamma_scale = gamma_scale\n        self.block_size = block_size\n        self.with_noise = with_noise\n        self.inplace = inplace\n        self.batchwise = batchwise\n        self.fast = fast  # FIXME finish comparisons of fast vs not\n\n    def forward(self, x):\n        if not self.training or not self.drop_prob:\n            return x\n        if self.fast:\n            return drop_block_fast_2d(\n                x, self.drop_prob, self.block_size, self.gamma_scale, self.with_noise, self.inplace)\n        else:\n            return drop_block_2d(\n                x, self.drop_prob, self.block_size, self.gamma_scale, self.with_noise, self.inplace, self.batchwise)\n\n\ndef drop_path(x, drop_prob: float = 0., training: bool = False, scale_by_keep: bool = True):\n    \"\"\"Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).\n    This is the same as the DropConnect impl I created for EfficientNet, etc networks, however,\n    the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...\n    See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for\n    changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use\n    'survival rate' as the argument.\n    \"\"\"\n    if drop_prob == 0. or not training:\n        return x\n    keep_prob = 1 - drop_prob\n    shape = (x.shape[0],) + (1,) * (x.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets\n    random_tensor = x.new_empty(shape).bernoulli_(keep_prob)\n    if keep_prob > 0.0 and scale_by_keep:\n        random_tensor.div_(keep_prob)\n    return x * random_tensor\n\n\nclass DropPath(nn.Module):\n    \"\"\"Drop paths (Stochastic Depth) per sample  (when applied in main path of residual blocks).\n    \"\"\"\n    def __init__(self, drop_prob: float = 0., scale_by_keep: bool = True):\n        super(DropPath, self).__init__()\n        self.drop_prob = drop_prob\n        self.scale_by_keep = scale_by_keep\n\n    def forward(self, x):\n        return drop_path(x, self.drop_prob, self.training, self.scale_by_keep)\n\n    def extra_repr(self):\n        return f'drop_prob={round(self.drop_prob,3):0.3f}'\n    \n\n\n# From PyTorch internals\ndef _ntuple(n):\n    def parse(x):\n        if isinstance(x, collections.abc.Iterable) and not isinstance(x, str):\n            return x\n        return tuple(repeat(x, n))\n    return parse\n\n\nto_1tuple = _ntuple(1)\nto_2tuple = _ntuple(2)\nto_3tuple = _ntuple(3)\nto_4tuple = _ntuple(4)\nto_ntuple = _ntuple\n\n\ndef make_divisible(v, divisor=8, min_value=None, round_limit=.9):\n    min_value = min_value or divisor\n    new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)\n    # Make sure that round down does not go down by more than 10%.\n    if new_v < round_limit * v:\n        new_v += divisor\n    return new_v","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:02.186579Z","iopub.execute_input":"2022-08-22T08:08:02.186924Z","iopub.status.idle":"2022-08-22T08:08:02.213392Z","shell.execute_reply.started":"2022-08-22T08:08:02.186895Z","shell.execute_reply":"2022-08-22T08:08:02.212463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nimport re\nimport fnmatch\nfrom collections import defaultdict\nfrom copy import deepcopy\n\n__all__ = ['list_models', 'is_model', 'model_entrypoint', 'list_modules', 'is_model_in_modules',\n           'is_pretrained_cfg_key', 'has_pretrained_cfg_key', 'get_pretrained_cfg_value', 'is_model_pretrained']\n\n_module_to_models = defaultdict(set)  # dict of sets to check membership of model in module\n_model_to_module = {}  # mapping of model names to module names\n_model_entrypoints = {}  # mapping of model names to entrypoint fns\n_model_has_pretrained = set()  # set of model names that have pretrained weight url present\n_model_pretrained_cfgs = dict()  # central repo for model default_cfgs\n\n\ndef register_model(fn):\n    # lookup containing module\n    mod = sys.modules[fn.__module__]\n    module_name_split = fn.__module__.split('.')\n    module_name = module_name_split[-1] if len(module_name_split) else ''\n\n    # add model to __all__ in module\n    model_name = fn.__name__\n    if hasattr(mod, '__all__'):\n        mod.__all__.append(model_name)\n    else:\n        mod.__all__ = [model_name]\n\n    # add entries to registry dict/sets\n    _model_entrypoints[model_name] = fn\n    _model_to_module[model_name] = module_name\n    _module_to_models[module_name].add(model_name)\n    has_valid_pretrained = False  # check if model has a pretrained url to allow filtering on this\n    if hasattr(mod, 'default_cfgs') and model_name in mod.default_cfgs:\n        # this will catch all models that have entrypoint matching cfg key, but miss any aliasing\n        # entrypoints or non-matching combos\n        cfg = mod.default_cfgs[model_name]\n        has_valid_pretrained = (\n            ('url' in cfg and 'http' in cfg['url']) or\n            ('file' in cfg and cfg['file']) or\n            ('hf_hub_id' in cfg and cfg['hf_hub_id'])\n        )\n        _model_pretrained_cfgs[model_name] = mod.default_cfgs[model_name]\n    if has_valid_pretrained:\n        _model_has_pretrained.add(model_name)\n    return fn\n\n\ndef _natural_key(string_):\n    return [int(s) if s.isdigit() else s for s in re.split(r'(\\d+)', string_.lower())]\n\n\ndef list_models(filter='', module='', pretrained=False, exclude_filters='', name_matches_cfg=False):\n    \"\"\" Return list of available model names, sorted alphabetically\n    Args:\n        filter (str) - Wildcard filter string that works with fnmatch\n        module (str) - Limit model selection to a specific sub-module (ie 'gen_efficientnet')\n        pretrained (bool) - Include only models with pretrained weights if True\n        exclude_filters (str or list[str]) - Wildcard filters to exclude models after including them with filter\n        name_matches_cfg (bool) - Include only models w/ model_name matching default_cfg name (excludes some aliases)\n    Example:\n        model_list('gluon_resnet*') -- returns all models starting with 'gluon_resnet'\n        model_list('*resnext*, 'resnet') -- returns all models with 'resnext' in 'resnet' module\n    \"\"\"\n    if module:\n        all_models = list(_module_to_models[module])\n    else:\n        all_models = _model_entrypoints.keys()\n    if filter:\n        models = []\n        include_filters = filter if isinstance(filter, (tuple, list)) else [filter]\n        for f in include_filters:\n            include_models = fnmatch.filter(all_models, f)  # include these models\n            if len(include_models):\n                models = set(models).union(include_models)\n    else:\n        models = all_models\n    if exclude_filters:\n        if not isinstance(exclude_filters, (tuple, list)):\n            exclude_filters = [exclude_filters]\n        for xf in exclude_filters:\n            exclude_models = fnmatch.filter(models, xf)  # exclude these models\n            if len(exclude_models):\n                models = set(models).difference(exclude_models)\n    if pretrained:\n        models = _model_has_pretrained.intersection(models)\n    if name_matches_cfg:\n        models = set(_model_pretrained_cfgs).intersection(models)\n    return list(sorted(models, key=_natural_key))\n\n\ndef is_model(model_name):\n    \"\"\" Check if a model name exists\n    \"\"\"\n    return model_name in _model_entrypoints\n\n\ndef model_entrypoint(model_name):\n    \"\"\"Fetch a model entrypoint for specified model name\n    \"\"\"\n    return _model_entrypoints[model_name]\n\n\ndef list_modules():\n    \"\"\" Return list of module names that contain models / model entrypoints\n    \"\"\"\n    modules = _module_to_models.keys()\n    return list(sorted(modules))\n\n\ndef is_model_in_modules(model_name, module_names):\n    \"\"\"Check if a model exists within a subset of modules\n    Args:\n        model_name (str) - name of model to check\n        module_names (tuple, list, set) - names of modules to search in\n    \"\"\"\n    assert isinstance(module_names, (tuple, list, set))\n    return any(model_name in _module_to_models[n] for n in module_names)\n\n\ndef is_model_pretrained(model_name):\n    return model_name in _model_has_pretrained\n\n\ndef get_pretrained_cfg(model_name):\n    if model_name in _model_pretrained_cfgs:\n        return deepcopy(_model_pretrained_cfgs[model_name])\n    return {}\n\n\ndef has_pretrained_cfg_key(model_name, cfg_key):\n    \"\"\" Query model default_cfgs for existence of a specific key.\n    \"\"\"\n    if model_name in _model_pretrained_cfgs and cfg_key in _model_pretrained_cfgs[model_name]:\n        return True\n    return False\n\n\ndef is_pretrained_cfg_key(model_name, cfg_key):\n    \"\"\" Return truthy value for specified model default_cfg key, False if does not exist.\n    \"\"\"\n    if model_name in _model_pretrained_cfgs and _model_pretrained_cfgs[model_name].get(cfg_key, False):\n        return True\n    return False\n\n\ndef get_pretrained_cfg_value(model_name, cfg_key):\n    \"\"\" Get a specific model default_cfg value by key. None if it doesn't exist.\n    \"\"\"\n    if model_name in _model_pretrained_cfgs:\n        return _model_pretrained_cfgs[model_name].get(cfg_key, None)\n    return None\n\n## config setting\ndef _cfg(url='', **kwargs):\n    return {\n        'url': url,\n        'num_classes': 1000, 'input_size': (3, 224, 224), 'pool_size': None,\n        'crop_pct': .9, 'interpolation': 'bicubic', 'fixed_input_size': True,\n        'mean': IMAGENET_INCEPTION_MEAN, 'std': IMAGENET_INCEPTION_STD,\n        'first_conv': 'patch_embed.proj', 'classifier': 'head',\n        **kwargs\n    }","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:03.613789Z","iopub.execute_input":"2022-08-22T08:08:03.614138Z","iopub.status.idle":"2022-08-22T08:08:03.634956Z","shell.execute_reply.started":"2022-08-22T08:08:03.614106Z","shell.execute_reply":"2022-08-22T08:08:03.633716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Backbone model contructed -- PVTV2-B4","metadata":{}},{"cell_type":"code","source":"class Mlp(nn.Module):\n    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0., linear=False):\n        super().__init__()\n        out_features = out_features or in_features\n        hidden_features = hidden_features or in_features\n        self.fc1 = nn.Linear(in_features, hidden_features)\n        self.dwconv = DWConv(hidden_features)\n        self.act = act_layer()\n        self.fc2 = nn.Linear(hidden_features, out_features)\n        self.drop = nn.Dropout(drop)\n        self.linear = linear\n\n        if self.linear:\n            self.relu = nn.ReLU(inplace=True)\n        self.apply(self._init_weights)\n\n    def _init_weights(self, m):\n        if isinstance(m, nn.Linear):\n            trunc_normal_(m.weight, std=.02)\n            if isinstance(m, nn.Linear) and m.bias is not None:\n                nn.init.constant_(m.bias, 0)\n        elif isinstance(m, nn.LayerNorm):\n            nn.init.constant_(m.bias, 0)\n            nn.init.constant_(m.weight, 1.0)\n        elif isinstance(m, nn.Conv2d):\n            fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n            fan_out //= m.groups\n            m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n            if m.bias is not None:\n                m.bias.data.zero_()\n\n    def forward(self, x, H, W):\n        x = self.fc1(x)\n        if self.linear:\n            x = self.relu(x)\n        x = self.dwconv(x, H, W)\n        x = self.act(x)\n        x = self.drop(x)\n        x = self.fc2(x)\n        x = self.drop(x)\n        return x\n\n\nclass Attention(nn.Module):\n    def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., sr_ratio=1, linear=False):\n        super().__init__()\n        assert dim % num_heads == 0, f\"dim {dim} should be divided by num_heads {num_heads}.\"\n\n        self.dim = dim\n        self.num_heads = num_heads\n        head_dim = dim // num_heads\n        self.scale = qk_scale or head_dim ** -0.5\n\n        self.q = nn.Linear(dim, dim, bias=qkv_bias)\n        self.kv = nn.Linear(dim, dim * 2, bias=qkv_bias)\n        self.attn_drop = nn.Dropout(attn_drop)\n        self.proj = nn.Linear(dim, dim)\n        self.proj_drop = nn.Dropout(proj_drop)\n\n        self.linear = linear\n        self.sr_ratio = sr_ratio\n        if not linear:\n            if sr_ratio > 1:\n                self.sr = nn.Conv2d(dim, dim, kernel_size=sr_ratio, stride=sr_ratio)\n                self.norm = nn.LayerNorm(dim)\n        else:\n            self.pool = nn.AdaptiveAvgPool2d(7)\n            self.sr = nn.Conv2d(dim, dim, kernel_size=1, stride=1)\n            self.norm = nn.LayerNorm(dim)\n            self.act = nn.GELU()\n        self.apply(self._init_weights)\n\n    def _init_weights(self, m):\n        if isinstance(m, nn.Linear):\n            trunc_normal_(m.weight, std=.02)\n            if isinstance(m, nn.Linear) and m.bias is not None:\n                nn.init.constant_(m.bias, 0)\n        elif isinstance(m, nn.LayerNorm):\n            nn.init.constant_(m.bias, 0)\n            nn.init.constant_(m.weight, 1.0)\n        elif isinstance(m, nn.Conv2d):\n            fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n            fan_out //= m.groups\n            m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n            if m.bias is not None:\n                m.bias.data.zero_()\n\n    def forward(self, x, H, W):\n        B, N, C = x.shape\n        q = self.q(x).reshape(B, N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3)\n\n        if not self.linear:\n            if self.sr_ratio > 1:\n                x_ = x.permute(0, 2, 1).reshape(B, C, H, W)\n                x_ = self.sr(x_).reshape(B, C, -1).permute(0, 2, 1)\n                x_ = self.norm(x_)\n                kv = self.kv(x_).reshape(B, -1, 2, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)\n            else:\n                kv = self.kv(x).reshape(B, -1, 2, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)\n        else:\n            x_ = x.permute(0, 2, 1).reshape(B, C, H, W)\n            x_ = self.sr(self.pool(x_)).reshape(B, C, -1).permute(0, 2, 1)\n            x_ = self.norm(x_)\n            x_ = self.act(x_)\n            kv = self.kv(x_).reshape(B, -1, 2, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)\n        k, v = kv[0], kv[1]\n\n        attn = (q @ k.transpose(-2, -1)) * self.scale\n        attn = attn.softmax(dim=-1)\n        attn = self.attn_drop(attn)\n\n        x = (attn @ v).transpose(1, 2).reshape(B, N, C)\n        x = self.proj(x)\n        x = self.proj_drop(x)\n\n        return x\n\n\nclass Block(nn.Module):\n    \n    def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,\n                 drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, sr_ratio=1, linear=False):\n        super().__init__()\n        self.norm1 = norm_layer(dim)\n        self.attn = Attention(\n            dim,\n            num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale,\n            attn_drop=attn_drop, proj_drop=drop, sr_ratio=sr_ratio, linear=linear)\n        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here\n        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()\n        self.norm2 = norm_layer(dim)\n        mlp_hidden_dim = int(dim * mlp_ratio)\n        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop, linear=linear)\n        \n        self.apply(self._init_weights)\n\n    def _init_weights(self, m):\n        if isinstance(m, nn.Linear):\n            trunc_normal_(m.weight, std=.02)\n            if isinstance(m, nn.Linear) and m.bias is not None:\n                nn.init.constant_(m.bias, 0)\n        elif isinstance(m, nn.LayerNorm):\n            nn.init.constant_(m.bias, 0)\n            nn.init.constant_(m.weight, 1.0)\n        elif isinstance(m, nn.Conv2d):\n            fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n            fan_out //= m.groups\n            m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n            if m.bias is not None:\n                m.bias.data.zero_()\n\n    def forward(self, x, H, W):\n        x = x + self.drop_path(self.attn(self.norm1(x), H, W))\n        x = x + self.drop_path(self.mlp(self.norm2(x), H, W))\n\n        return x\n\n\nclass OverlapPatchEmbed(nn.Module):\n    \"\"\" Image to Patch Embedding\n    \"\"\"\n\n    def __init__(self, img_size=224, patch_size=7, stride=4, in_chans=3, embed_dim=768):\n        super().__init__()\n        img_size = to_2tuple(img_size)\n        patch_size = to_2tuple(patch_size)\n\n        assert max(patch_size) > stride, \"Set larger patch_size than stride\"\n\n        self.img_size = img_size\n        self.patch_size = patch_size\n        self.H, self.W = img_size[0] // stride, img_size[1] // stride\n        self.num_patches = self.H * self.W\n        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=stride,\n                              padding=(patch_size[0] // 2, patch_size[1] // 2))\n        self.norm = nn.LayerNorm(embed_dim)\n\n        self.apply(self._init_weights)\n\n    def _init_weights(self, m):\n        if isinstance(m, nn.Linear):\n            trunc_normal_(m.weight, std=.02)\n            if isinstance(m, nn.Linear) and m.bias is not None:\n                nn.init.constant_(m.bias, 0)\n        elif isinstance(m, nn.LayerNorm):\n            nn.init.constant_(m.bias, 0)\n            nn.init.constant_(m.weight, 1.0)\n        elif isinstance(m, nn.Conv2d):\n            fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n            fan_out //= m.groups\n            m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n            if m.bias is not None:\n                m.bias.data.zero_()\n\n    def forward(self, x):\n        x = self.proj(x)\n        _, _, H, W = x.shape\n        x = x.flatten(2).transpose(1, 2)\n        x = self.norm(x)\n\n        return x, H, W\n\n\nclass PyramidVisionTransformerV2(nn.Module):\n    def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dims=[64, 128, 256, 512],\n                 num_heads=[1, 2, 4, 8], mlp_ratios=[4, 4, 4, 4], qkv_bias=False, qk_scale=None, drop_rate=0.,\n                 attn_drop_rate=0., drop_path_rate=0., norm_layer=nn.LayerNorm, depths=[3, 4, 6, 3],\n                 sr_ratios=[8, 4, 2, 1], num_stages=4, linear=False):\n        super().__init__()\n        # self.num_classes = num_classes\n        self.depths = depths\n        self.num_stages = num_stages\n        self.linear = linear\n        self.embed_dims = embed_dims\n\n        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]  # stochastic depth decay rule\n        cur = 0\n\n        for i in range(num_stages):\n            patch_embed = OverlapPatchEmbed(img_size=img_size if i == 0 else img_size // (2 ** (i + 1)),\n                                            patch_size=7 if i == 0 else 3,\n                                            stride=4 if i == 0 else 2,\n                                            in_chans=in_chans if i == 0 else embed_dims[i - 1],\n                                            embed_dim=embed_dims[i])\n            \n            block = nn.ModuleList([Block(\n                dim=embed_dims[i], num_heads=num_heads[i], mlp_ratio=mlp_ratios[i], qkv_bias=qkv_bias,\n                qk_scale=qk_scale,\n                drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[cur + j], norm_layer=norm_layer,\n                sr_ratio=sr_ratios[i], linear=linear)\n                for j in range(depths[i])])\n            norm = norm_layer(embed_dims[i])\n            cur += depths[i]\n            \n            setattr(self, f\"patch_embed{i + 1}\", patch_embed)\n            setattr(self, f\"block{i + 1}\", block)\n            setattr(self, f\"norm{i + 1}\", norm)\n        \n        # classification head\n        # self.head = nn.Linear(embed_dims[3], num_classes) if num_classes > 0 else nn.Identity()\n        \n        self.apply(self._init_weights)\n    \n    def _init_weights(self, m):\n        if isinstance(m, nn.Linear):\n            trunc_normal_(m.weight, std=.02)\n            if isinstance(m, nn.Linear) and m.bias is not None:\n                nn.init.constant_(m.bias, 0)\n        elif isinstance(m, nn.LayerNorm):\n            nn.init.constant_(m.bias, 0)\n            nn.init.constant_(m.weight, 1.0)\n        elif isinstance(m, nn.Conv2d):\n            fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n            fan_out //= m.groups\n            m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n            if m.bias is not None:\n                m.bias.data.zero_()\n\n    def init_weights(self, pretrained=None):\n        pass\n        #if isinstance(pretrained, str):\n            #logger = get_root_logger()\n            #load_checkpoint(self, pretrained, map_location='cpu', strict=False, logger=logger)\n\n    def freeze_patch_emb(self):\n        self.patch_embed1.requires_grad = False\n\n    @torch.jit.ignore\n    def no_weight_decay(self):\n        return {'pos_embed1', 'pos_embed2', 'pos_embed3', 'pos_embed4', 'cls_token'}  # has pos_embed may be better\n\n    def get_classifier(self):\n        return self.head\n\n    def reset_classifier(self, num_classes, global_pool=''):\n        self.num_classes = num_classes\n        self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity()\n\n    def forward_features(self, x):\n        B = x.shape[0]\n        outs = []\n\n        for i in range(self.num_stages):\n            patch_embed = getattr(self, f\"patch_embed{i + 1}\")\n            block = getattr(self, f\"block{i + 1}\")\n            norm = getattr(self, f\"norm{i + 1}\")\n            x, H, W = patch_embed(x)\n            for blk in block:\n                x = blk(x, H, W)\n            x = norm(x)\n            x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous()\n            outs.append(x)\n\n        return outs\n\n    def forward(self, x):\n        x = self.forward_features(x)\n        # x = self.head(x)\n\n        return x\n\n\nclass DWConv(nn.Module):\n    def __init__(self, dim=768):\n        super(DWConv, self).__init__()\n        self.dwconv = nn.Conv2d(dim, dim, 3, 1, 1, bias=True, groups=dim)\n\n    def forward(self, x, H, W):\n        B, N, C = x.shape\n        x = x.transpose(1, 2).view(B, C, H, W)\n        x = self.dwconv(x)\n        x = x.flatten(2).transpose(1, 2)\n\n        return x\n\n\ndef _conv_filter(state_dict, patch_size=16):\n    \"\"\" convert patch embedding weight from manual patchify + linear proj to conv\"\"\"\n    out_dict = {}\n    for k, v in state_dict.items():\n        if 'patch_embed.proj.weight' in k:\n            v = v.reshape((v.shape[0], 3, patch_size, patch_size))\n        out_dict[k] = v\n\n    return out_dict\n\n\n#@BACKBONES.register_module()\nclass pvt_v2_b1(PyramidVisionTransformerV2):\n    def __init__(self, **kwargs):\n        super(pvt_v2_b1, self).__init__(\n            patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4],\n            qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[2, 2, 2, 2], sr_ratios=[8, 4, 2, 1],\n            drop_rate=0.0, drop_path_rate=0.1)\n\n#@BACKBONES.register_module()\nclass pvt_v2_b2(PyramidVisionTransformerV2):\n    def __init__(self, **kwargs):\n        super(pvt_v2_b2, self).__init__(\n            patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4],\n            qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 4, 6, 3], sr_ratios=[8, 4, 2, 1],\n            drop_rate=0.0, drop_path_rate=0.1)\n\n#@BACKBONES.register_module()\nclass pvt_v2_b2_li(PyramidVisionTransformerV2):\n    def __init__(self, **kwargs):\n        super(pvt_v2_b2_li, self).__init__(\n            patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4],\n            qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 4, 6, 3], sr_ratios=[8, 4, 2, 1],\n            drop_rate=0.0, drop_path_rate=0.1, linear=True)\n\n\n#@BACKBONES.register_module()\nclass pvt_v2_b3(PyramidVisionTransformerV2):\n    def __init__(self, **kwargs):\n        super(pvt_v2_b3, self).__init__(\n            patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4],\n            qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 4, 18, 3], sr_ratios=[8, 4, 2, 1],\n            drop_rate=0.0, drop_path_rate=0.1)\n\n#@BACKBONES.register_module()\nclass pvt_v2_b4(PyramidVisionTransformerV2):\n    def __init__(self, **kwargs):\n        super(pvt_v2_b4, self).__init__(\n            patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4],\n            qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 8, 27, 3], sr_ratios=[8, 4, 2, 1],\n            drop_rate=0.0, drop_path_rate=0.1)\n\n\npretrain_dir ='.'\nif 1:\n    cfg = dict(\n        pvt_v2_b1 = dict(\n            checkpoint = pretrain_dir + '/pvt_v2_b1.pth',\n            builder  =  pvt_v2_b1,\n        ),\n        pvt_v2_b2 = dict(\n            checkpoint = pretrain_dir + '/pvt_v2_b2.pth',\n            builder  =  pvt_v2_b2,\n        ),\n        pvt_v2_b3 = dict(\n            checkpoint = pretrain_dir + '/pvt_v2_b3.pth',\n            #checkpoint = pretrain_dir + '/pytorch_model.bin',\n            builder  =  pvt_v2_b3,\n\n        ),\n        pvt_v2_b4 = dict(\n            checkpoint = pretrain_dir + '/pvt_v2_b4.pth',\n            builder  =  pvt_v2_b4,\n        ),\n\n    )\n\nif 0:\n    m = pvt_v2_b4()\n    print(m)\n    x=torch.rand(1,3,224,224)\n    y=m(x)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:04.688427Z","iopub.execute_input":"2022-08-22T08:08:04.689406Z","iopub.status.idle":"2022-08-22T08:08:04.78707Z","shell.execute_reply.started":"2022-08-22T08:08:04.689343Z","shell.execute_reply":"2022-08-22T08:08:04.785844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mask to RLE function","metadata":{}},{"cell_type":"code","source":"def rle_encode_less_memory(img):\n    #the image should be transposed\n    #img = cv2.imread(img)\n    pixels = img.T.flatten()\n    \n    # This simplified method requires first and last pixel to be zero\n    #pixels[0] = 0\n    #pixels[-1] = 0\n    #runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    \n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    \n    runs[1::2] -= runs[::2]\n    \n    return ' '.join(str(x) for x in runs)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:04.789872Z","iopub.execute_input":"2022-08-22T08:08:04.790233Z","iopub.status.idle":"2022-08-22T08:08:04.800225Z","shell.execute_reply.started":"2022-08-22T08:08:04.790196Z","shell.execute_reply":"2022-08-22T08:08:04.799183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data loading","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(\"../input/hubmap-organ-segmentation/test.csv\")\ntrain_df = pd.read_csv(\"../input/hubmap-organ-segmentation/train.csv\")\ntest_path = \"../input/hubmap-organ-segmentation/test_images\"","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:05.005213Z","iopub.execute_input":"2022-08-22T08:08:05.005808Z","iopub.status.idle":"2022-08-22T08:08:05.02195Z","shell.execute_reply.started":"2022-08-22T08:08:05.005769Z","shell.execute_reply":"2022-08-22T08:08:05.021051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image preprocess","metadata":{}},{"cell_type":"code","source":"def image_to_tensor(image, mode='bgr'): #image mode\n    if mode=='bgr':\n        image = image[:,:,::-1]\n    x = image\n    x = x.transpose(2,0,1)\n    x = np.ascontiguousarray(x)\n    x = torch.tensor(x, dtype=torch.float)\n    return x\n\n\ndef mask_to_tensor(mask):\n    x = mask\n    x = torch.tensor(x, dtype=torch.float)\n    return x\n\n\ntensor_list = ['mask', 'image', 'organ']\n\ndef null_collate(batch):\n    d = {}\n    key = batch[0].keys()\n    for k in key:\n        v = [b[k] for b in batch]\n        if k in tensor_list:\n            v = torch.stack(v)\n        d[k] = v\n\n    d['mask'] = d['mask'].unsqueeze(1)\n    d['organ'] = d['organ'].reshape(-1)\n    return d\n\n\ndef _ntuple(n):\n    def parse(x):\n        if isinstance(x, collections.abc.Iterable):\n            return x\n        return tuple(repeat(x, n))\n    return parse\n\n\nto_2tuple = _ntuple(2)\n\n\ndef _no_grad_trunc_normal_(tensor, mean, std, a, b):\n    # Cut & paste from PyTorch official master until it's in a few official releases - RW\n    # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf\n    def norm_cdf(x):\n        # Computes standard normal cumulative distribution function\n        return (1. + math.erf(x / math.sqrt(2.))) / 2.\n\n    if (mean < a - 2 * std) or (mean > b + 2 * std):\n        warnings.warn(\"mean is more than 2 std from [a, b] in nn.init.trunc_normal_. \"\n                      \"The distribution of values may be incorrect.\",\n                      stacklevel=2)\n\n        \ndef trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):\n    return _no_grad_trunc_normal_(tensor, mean, std, a, b)\n\n\ndef drop_path(x, drop_prob: float = 0., training: bool = False, scale_by_keep: bool = True):\n    \"\"\"Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).\n    This is the same as the DropConnect impl I created for EfficientNet, etc networks, however,\n    the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...\n    See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for\n    changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use\n    'survival rate' as the argument.\n    \"\"\"\n    if drop_prob == 0. or not training:\n        return x\n    keep_prob = 1 - drop_prob\n    shape = (x.shape[0],) + (1,) * (x.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets\n    random_tensor = x.new_empty(shape).bernoulli_(keep_prob)\n    if keep_prob > 0.0 and scale_by_keep:\n        random_tensor.div_(keep_prob)\n    return x * random_tensor\n\n\nclass DropPath(nn.Module):\n    \"\"\"Drop paths (Stochastic Depth) per sample  (when applied in main path of residual blocks).\n    \"\"\"\n    def __init__(self, drop_prob: float = 0., scale_by_keep: bool = True):\n        super(DropPath, self).__init__()\n        self.drop_prob = drop_prob\n        self.scale_by_keep = scale_by_keep\n\n    def forward(self, x):\n        return drop_path(x, self.drop_prob, self.training, self.scale_by_keep)\n\n    def extra_repr(self):\n        return f'drop_prob={round(self.drop_prob,3):0.3f}'\n    \n    \nclass RGB(nn.Module):\n    IMAGE_RGB_MEAN = [0.485, 0.456, 0.406] #[0.5, 0.5, 0.5]\n    IMAGE_RGB_STD  = [0.229, 0.224, 0.225] #[0.5, 0.5, 0.5]\n\n    def __init__(self,):\n        super(RGB, self).__init__()\n        self.register_buffer('mean', torch.zeros(1,3,1,1))\n        self.register_buffer('std', torch.ones(1,3,1,1))\n        self.mean.data = torch.FloatTensor(self.IMAGE_RGB_MEAN).view(self.mean.shape)\n        self.std.data = torch.FloatTensor(self.IMAGE_RGB_STD).view(self.std.shape)\n\n    def forward(self, x):\n        x = (x-self.mean)/self.std\n        return x\n    \n    \ndef message(mode='print'):\n    asterisk = ' '\n    if mode==('print'):\n        loss = batch_loss\n    if mode==('log'):\n        loss = train_loss\n        if (iteration % iter_save == 0): asterisk = '*'\n\n    text = \\\n        ('%0.2e   %08d%s %6.2f | '%(rate, iteration, asterisk, epoch,)).replace('e-0','e-').replace('e+0','e+') + \\\n        '%4.3f  %4.3f  %4.4f  %4.3f   | '%(*valid_loss,) + \\\n        '%4.3f  %4.3f   | '%(*loss,) + \\\n        '%s' % ((time.time() - start_timer))\n\n    return text","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:05.849693Z","iopub.execute_input":"2022-08-22T08:08:05.850046Z","iopub.status.idle":"2022-08-22T08:08:05.871579Z","shell.execute_reply.started":"2022-08-22T08:08:05.850014Z","shell.execute_reply":"2022-08-22T08:08:05.870325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset augmentation","metadata":{}},{"cell_type":"code","source":"def valid_augment5(image, mask, organ):\n    #image, mask  = do_crop(image, mask, image_size, xy=(None,None))\n    return image, mask\n\ndef train_augment5b(image, mask, organ):\n    image, mask = do_random_flip(image, mask)\n    image, mask = do_random_rot90(image, mask)\n\n    for fn in np.random.choice([\n        lambda image, mask: (image, mask),\n        lambda image, mask: do_random_noise(image, mask, mag=0.1),\n        lambda image, mask: do_random_contast(image, mask, mag=0.40),\n        lambda image, mask: do_random_hsv(image, mask, mag=[0.40, 0.40, 0])\n    ], 2): image, mask = fn(image, mask)\n\n    for fn in np.random.choice([\n        lambda image, mask: (image, mask),\n        lambda image, mask: do_random_rotate_scale(image, mask, angle=45, scale=[0.50, 2.0]),\n    ], 1): image, mask = fn(image, mask)\n\n    return image, mask","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:06.542043Z","iopub.execute_input":"2022-08-22T08:08:06.542421Z","iopub.status.idle":"2022-08-22T08:08:06.551724Z","shell.execute_reply.started":"2022-08-22T08:08:06.542385Z","shell.execute_reply":"2022-08-22T08:08:06.550575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_random_flip(image, mask):\n    if np.random.rand()>0.5:\n        image = cv2.flip(image,0)\n        mask = cv2.flip(mask,0)\n    if np.random.rand()>0.5:\n        image = cv2.flip(image,1)\n        mask = cv2.flip(mask,1)\n    if np.random.rand()>0.5:\n        image = image.transpose(1,0,2)\n        mask = mask.transpose(1,0)\n    \n    image = np.ascontiguousarray(image)\n    mask = np.ascontiguousarray(mask)\n    return image, mask\n\ndef do_random_rot90(image, mask):\n    r = np.random.choice([\n        0,\n        cv2.ROTATE_90_CLOCKWISE,\n        cv2.ROTATE_90_COUNTERCLOCKWISE,\n        cv2.ROTATE_180,\n    ])\n    if r==0:\n        return image, mask\n    else:\n        image = cv2.rotate(image, r)\n        mask = cv2.rotate(mask, r)\n        return image, mask\n    \ndef do_random_contast(image, mask, mag=0.3):\n    alpha = 1 + random.uniform(-1,1)*mag\n    image = image * alpha\n    image = np.clip(image,0,1)\n    return image, mask\n\ndef do_random_hsv(image, mask, mag=[0.15,0.25,0.25]):\n    image = (image*255).astype(np.uint8)\n    hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)\n\n    h = hsv[:, :, 0].astype(np.float32)  # hue\n    s = hsv[:, :, 1].astype(np.float32)  # saturation\n    v = hsv[:, :, 2].astype(np.float32)  # value\n    h = (h*(1 + random.uniform(-1,1)*mag[0]))%180\n    s =  s*(1 + random.uniform(-1,1)*mag[1])\n    v =  v*(1 + random.uniform(-1,1)*mag[2])\n\n    hsv[:, :, 0] = np.clip(h,0,180).astype(np.uint8)\n    hsv[:, :, 1] = np.clip(s,0,255).astype(np.uint8)\n    hsv[:, :, 2] = np.clip(v,0,255).astype(np.uint8)\n    image = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)\n    image = image.astype(np.float32)/255\n    return image, mask\n\ndef do_random_noise(image, mask, mag=0.1):\n    height, width = image.shape[:2]\n    noise = np.random.uniform(-1,1, (height, width,1))*mag\n    image = image + noise\n    image = np.clip(image,0,1)\n    return image, mask\n\ndef do_random_rotate_scale(image, mask, angle=30, scale=[0.8,1.2] ):\n    angle = np.random.uniform(-angle, angle)\n    scale = np.random.uniform(*scale) if scale is not None else 1\n    \n    height, width = image.shape[:2]\n    center = (height // 2, width // 2)\n    \n    transform = cv2.getRotationMatrix2D(center, angle, scale)\n    image = cv2.warpAffine( image, transform, (width, height), flags=cv2.INTER_LINEAR,\n                            borderMode=cv2.BORDER_CONSTANT, borderValue=(0,0,0))\n    mask  = cv2.warpAffine( mask, transform, (width, height), flags=cv2.INTER_LINEAR,\n                            borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n    return image, mask","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:07.116553Z","iopub.execute_input":"2022-08-22T08:08:07.116856Z","iopub.status.idle":"2022-08-22T08:08:07.134947Z","shell.execute_reply.started":"2022-08-22T08:08:07.116829Z","shell.execute_reply":"2022-08-22T08:08:07.133839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# HubmapDataset function","metadata":{}},{"cell_type":"code","source":"image_size = 768\n\nclass HubmapDataset(Dataset):\n    def __init__(self, df, augment=None):\n\n        self.df = df\n        self.augment = augment\n        self.length = len(self.df)\n        ids = pd.read_csv(LABELS).id.astype(str).values\n        self.fnames = [fname for fname in os.listdir(TRAIN) if fname.split('.')[0] in ids]\n        self.organ_to_label = {'kidney' : 0,\n                               'prostate' : 1,\n                               'largeintestine' : 2,\n                               'spleen' : 3,\n                               'lung' : 4}\n\n    def __str__(self):\n        string = ''\n        string += '\\tlen = %d\\n' % len(self)\n\n        d = self.df.organ.value_counts().to_dict()\n        for k in ['kidney', 'prostate', 'largeintestine', 'spleen', 'lung']:\n            string +=  '%24s %3d (%0.3f) \\n'%(k,d.get(k,0),d.get(k,0)/len(self.df))\n        return string\n\n    def __len__(self):\n        return self.length\n\n    def __getitem__(self, index):\n        fname = self.fnames[index]\n        d = self.df.iloc[index]\n        organ = self.organ_to_label[d.organ]\n\n        image = cv2.cvtColor(cv2.imread(os.path.join(TRAIN,fname)), cv2.COLOR_BGR2RGB)\n        print('Image name.......................',fname)\n        mask = cv2.imread(os.path.join(MASKS,fname.split('.')[0]+'.png'),cv2.IMREAD_GRAYSCALE)\n        print(mask)\n        plt.imshow(mask)\n        \n        image = image.astype(np.float32)/255\n        mask  = mask.astype(np.float32)/255\n\n        s = d.pixel_size/0.4 * (image_size/3000)\n        image = cv2.resize(image,dsize=(image_size,image_size),interpolation=cv2.INTER_LINEAR)\n        mask  = cv2.resize(mask, dsize=(image_size,image_size),interpolation=cv2.INTER_LINEAR)\n\n        if self.augment is not None:\n            image, mask = self.augment(image, mask, organ)\n\n\n        r ={}\n        r['index']= index\n        r['id'] = fname\n        r['organ'] = torch.tensor([organ], dtype=torch.long)\n        r['image'] = image_to_tensor(image)\n        r['mask' ] = mask_to_tensor(mask>0.5)\n        return r","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:08.016286Z","iopub.execute_input":"2022-08-22T08:08:08.017324Z","iopub.status.idle":"2022-08-22T08:08:08.031401Z","shell.execute_reply.started":"2022-08-22T08:08:08.017288Z","shell.execute_reply":"2022-08-22T08:08:08.030153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mixup & Decoder function","metadata":{}},{"cell_type":"code","source":"class MixUpSample(nn.Module):\n    def __init__( self, scale_factor=2):\n        super().__init__()\n        self.mixing = nn.Parameter(torch.tensor(0.5))\n        self.scale_factor = scale_factor\n\n    def forward(self, x):\n        x = self.mixing *F.interpolate(x, scale_factor=self.scale_factor, mode='bilinear', align_corners=False) \\\n            + (1-self.mixing )*F.interpolate(x, scale_factor=self.scale_factor, mode='nearest')\n        return x\n\n\nclass to3x3_Decoder(nn.Module):\n    def __init__(\n            self,\n            encoder_dim = [32, 64, 160, 256],\n            decoder_dim = 256,\n    ):\n        super().__init__()\n        self.mixing = nn.Parameter(torch.FloatTensor([0.5,0.5,0.5,0.5]))\n        self.mlp = nn.ModuleList([\n            nn.Sequential(\n                nn.Conv2d(dim, decoder_dim, 1, padding= 0,  bias=False), #follow mmseg to use conv-bn-relu\n                #nn.Conv2d(dim, decoder_dim, 3, padding= 1,  bias=False),\n                nn.BatchNorm2d(decoder_dim),\n                nn.ReLU(inplace=True),\n                MixUpSample(2**i) if i!=0 else nn.Identity(),\n            ) for i, dim in enumerate(encoder_dim)])\n\n        self.fuse = nn.Sequential(\n            #nn.Conv2d(len(encoder_dim) * decoder_dim, decoder_dim, 1, padding=0, bias=False),\n            nn.Conv2d(len(encoder_dim) * decoder_dim, decoder_dim, 3, padding=1, bias=False),\n            nn.BatchNorm2d(decoder_dim),\n            nn.ReLU(inplace=True),\n            #nn.Conv2d(decoder_dim, decoder_dim, 3, padding=1, bias=False),\n            #nn.BatchNorm2d(decoder_dim),\n            #nn.ReLU(inplace=True),\n        )\n\n    def forward(self, feature):\n\n        out = []\n        for i,f in enumerate(feature):\n            f = self.mlp[i](f)\n            out.append(f)\n\n        x = self.fuse(torch.cat(out, dim = 1))\n        return x, out\n\n\n#<todo>\n# do a reverse mit upsize + conv + Mix FFN\n\n#################################################################\n\nclass RGB(nn.Module):\n    IMAGE_RGB_MEAN = [0.485, 0.456, 0.406] #[0.5, 0.5, 0.5]\n    IMAGE_RGB_STD  = [0.229, 0.224, 0.225] #[0.5, 0.5, 0.5]\n\n    def __init__(self,):\n        super(RGB, self).__init__()\n        self.register_buffer('mean', torch.zeros(1,3,1,1))\n        self.register_buffer('std', torch.ones(1,3,1,1))\n        self.mean.data = torch.FloatTensor(self.IMAGE_RGB_MEAN).view(self.mean.shape)\n        self.std.data = torch.FloatTensor(self.IMAGE_RGB_STD).view(self.std.shape)\n\n    def forward(self, x):\n        x = (x-self.mean)/self.std\n        return x\n\n\nclass Net(nn.Module):\n    def load_pretrain( self,):\n        checkpoint = cfg[self.arch]['checkpoint']\n        print('load %s'%checkpoint)\n        checkpoint = torch.load(checkpoint, map_location=lambda storage, loc: storage)  #True\n        #checkpoint = torch.load('./pytorch_model.bin', map_location=lambda storage, loc: storage)  #True\n        print(self.encoder.load_state_dict(checkpoint,strict=False))  #True\n\n\n    def __init__( self,):\n        super(Net, self).__init__()\n        self.output_type = ['inference', 'loss']\n        self.rgb = RGB()\n        self.dropout = nn.Dropout(0.1)\n\n        self.arch = 'pvt_v2_b4'\n        self.encoder = cfg[self.arch]['builder']()\n        encoder_dim = self.encoder.embed_dims\n        #[64, 128, 320, 512]\n\n        self.decoder = to3x3_Decoder(\n            encoder_dim = encoder_dim,\n            decoder_dim = 320,\n        )\n        self.logit = nn.Sequential(\n            nn.Conv2d(320, 1, kernel_size=1, padding=0),\n        )\n        self.aux = nn.ModuleList([\n            nn.Conv2d(encoder_dim[i], 1, kernel_size=1, padding=0) for i in range(4)\n        ])\n\n\n    def forward(self, batch):\n\n        x = batch['image']\n        x = self.rgb(x)\n\n        B,C,H,W = x.shape\n        encoder = self.encoder(x)\n        #print([f.shape for f in encoder])\n\n        last, decoder = self.decoder(encoder)\n        last  = self.dropout(last)\n        logit = self.logit(last)\n        logit = F.interpolate(logit, size=None, scale_factor=4, mode='bilinear', align_corners=False)\n        #print(logit.shape)\n\n        output = {}\n        if 'loss' in self.output_type:\n            output['bce_loss'] = F.binary_cross_entropy_with_logits(logit,batch['mask'])\n            for i in range(4):\n                output['aux%d_loss'%i] = criterion_aux_loss(self.aux[i](encoder[i]),batch['mask'])\n\n        if 'inference' in self.output_type:\n            output['probability'] = torch.sigmoid(logit)\n\n        return output\n\ndef criterion_aux_loss(logit, mask):\n    mask = F.interpolate(mask,size=logit.shape[-2:], mode='nearest')\n    loss = F.binary_cross_entropy_with_logits(logit,mask)\n    return loss\n\n\ndef run_check_net():\n    batch_size = 4\n    image_size = 512\n\n    #---\n    batch = {\n        'image' : torch.from_numpy( np.random.uniform(-1,1,(batch_size,3,image_size,image_size)) ).float(),\n        'mask'  : torch.from_numpy( np.random.choice(2,(batch_size,1,image_size,image_size)) ).float(),\n        'organ' : torch.from_numpy( np.random.choice(5,(batch_size)) ).long(),\n    }\n    batch = {k:v.cuda() for k,v in batch.items()}\n\n\n\n    net = Net().cuda()\n    # torch.save({ 'state_dict': net.state_dict() },  'model.pth' )\n    # exit(0)\n    net.load_pretrain()\n\n    with torch.no_grad():\n        with torch.cuda.amp.autocast(enabled=True):\n            output = net(batch)\n\n    print('batch')\n    for k,v in batch.items():\n        print('%32s :'%k, v.shape)\n\n    print('output')\n    for k,v in output.items():\n        if 'loss' not in k:\n            print('%32s :'%k, v.shape)\n    for k,v in output.items():\n        if 'loss' in k:\n            print('%32s :'%k, v.item())","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:34.594578Z","iopub.execute_input":"2022-08-22T08:08:34.594933Z","iopub.status.idle":"2022-08-22T08:08:34.629438Z","shell.execute_reply.started":"2022-08-22T08:08:34.594903Z","shell.execute_reply":"2022-08-22T08:08:34.628396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DAformer decoder function","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\n\nclass MixUpSample1(nn.Module):\n    def __init__( self, scale_factor=2):\n        super().__init__()\n        assert(scale_factor!=1)\n\n        self.mixing = nn.Parameter(torch.tensor(0.5))\n        self.scale_factor = scale_factor\n\n    def forward(self, x):\n        x = self.mixing *F.interpolate(x, scale_factor=self.scale_factor, mode='bilinear', align_corners=False) \\\n            + (1-self.mixing )*F.interpolate(x, scale_factor=self.scale_factor, mode='nearest')\n        return x\n\n#https://github.com/lhoyer/DAFormer/blob/master/mmseg/models/decode_heads/daformer_head.py\ndef Conv2dBnReLU(in_channel, out_channel, kernel_size=3, padding=1,stride=1, dilation=1):\n    return nn.Sequential(\n        nn.Conv2d(in_channel, out_channel, kernel_size=kernel_size, padding=padding, stride=stride, dilation=dilation, bias=False),\n        nn.BatchNorm2d(out_channel),\n        nn.ReLU(inplace=True),\n    )\n\nclass ASPP(nn.Module):\n\n    def __init__(self,\n                 in_channel,\n                 channel,\n                 dilation,\n                 ):\n        super(ASPP, self).__init__()\n\n        self.conv = nn.ModuleList()\n        for d in dilation:\n            self.conv.append(\n                Conv2dBnReLU(\n                    in_channel,\n                    channel,\n                    kernel_size=1 if d == 1 else 3,\n                    dilation=d,\n                    padding=0 if d == 1 else d,\n                )\n            )\n\n        self.out = Conv2dBnReLU(\n            len(dilation) * channel,\n            channel,\n            kernel_size=3,\n            padding=1,\n            )\n\n    def forward(self, x):\n        aspp = []\n        for conv in self.conv:\n            aspp.append(conv(x))\n        aspp = torch.cat(aspp, dim=1)\n        out = self.out(aspp)\n        return out\n\n#DepthwiseSeparable\nclass DSConv2d(nn.Module):\n    def __init__(self,\n                 in_channel,\n                 out_channel,\n                 kernel_size,\n                 stride   = 1,\n                 padding  = 0,\n                 dilation = 1\n        ):\n        super().__init__()\n\n        self.depthwise = nn.Sequential(\n            nn.Conv2d( in_channel, in_channel, kernel_size, stride=stride, padding=padding, dilation=dilation),\n            nn.BatchNorm2d(in_channel),\n            nn.ReLU(inplace=True)\n        )\n\n        self.pointwise = nn.Sequential(\n            nn.Conv2d( in_channel, out_channel, kernel_size=1, stride=1, padding=0),\n            nn.BatchNorm2d(out_channel),\n            nn.ReLU(inplace=True)\n        )\n\n    def forward(self, x):\n        x = self.depthwise(x)\n        x = self.pointwise(x)\n        return x\n\nclass DSASPP(nn.Module):\n\n    def __init__(self,\n                 in_channel,\n                 channel,\n                 dilation,\n                 ):\n        super(DSASPP, self).__init__()\n\n        self.conv = nn.ModuleList()\n        for d in dilation:\n            if d == 1:\n                self.conv.append(\n                    Conv2dBnReLU(\n                        in_channel,\n                        channel,\n                        kernel_size=1 if d == 1 else 3,\n                        dilation=d,\n                        padding=0 if d == 1 else d,\n                    )\n                )\n            else:\n                self.conv.append(\n                    DSConv2d(\n                        in_channel,\n                        channel,\n                        kernel_size=3,\n                        dilation=d,\n                        padding=d,\n                    )\n                )\n\n        self.out = Conv2dBnReLU(\n            len(dilation) * channel,\n            channel,\n            kernel_size=3,\n            padding=1,\n        )\n\n    def forward(self, x):\n        aspp = []\n        for conv in self.conv:\n            aspp.append(conv(x))\n        aspp = torch.cat(aspp, dim=1)\n        out = self.out(aspp)\n        return out\n\n\n##############################################################################################33\n\nclass DaformerDecoder(nn.Module):\n    def __init__(\n            self,\n            encoder_dim = [32, 64, 160, 256],\n            decoder_dim = 256,\n            dilation = [1, 6, 12, 18],\n            use_bn_mlp  = True,\n            fuse = 'conv1x1',\n    ):\n        super().__init__()\n        self.mlp = nn.ModuleList([\n            nn.Sequential(\n                # Conv2dBnReLU(dim, decoder_dim, 1, padding=0), #follow mmseg to use conv-bn-relu\n                *(\n                  ( nn.Conv2d(dim, decoder_dim, 1, padding= 0,  bias=False),\n                    nn.BatchNorm2d(decoder_dim),\n                    nn.ReLU(inplace=True),\n                )if use_bn_mlp else\n                  ( nn.Conv2d(dim, decoder_dim, 1, padding= 0,  bias=True),)\n                ),\n\n                MixUpSample1(2**i) if i!=0 else nn.Identity(),\n            ) for i, dim in enumerate(encoder_dim)])\n\n        if fuse=='conv1x1':\n            self.fuse = nn.Sequential(\n                nn.Conv2d(len(encoder_dim) * decoder_dim, decoder_dim, 1, padding=0, bias=False),\n                nn.BatchNorm2d(decoder_dim),\n                nn.ReLU(inplace=True),\n            )\n\n        if fuse=='conv3x3':\n            self.fuse = nn.Sequential(\n                nn.Conv2d(len(encoder_dim) * decoder_dim, decoder_dim, 3, padding=1, bias=False),\n                nn.BatchNorm2d(decoder_dim),\n                nn.ReLU(inplace=True),\n            )\n\n        if fuse=='aspp':\n            self.fuse = ASPP(\n                decoder_dim*len(encoder_dim),\n                decoder_dim,\n                dilation,\n            )\n\n        if fuse=='ds-aspp':\n            self.fuse = DSASPP(\n                decoder_dim*len(encoder_dim),\n                decoder_dim,\n                dilation,\n            )\n\n\n    def forward(self, feature):\n\n        out = []\n        for i,f in enumerate(feature):\n            f = self.mlp[i](f)\n            out.append(f)\n            #print(f.shape)\n        x = self.fuse(torch.cat(out, dim = 1))\n        return x, out\n\n\nclass daformer_conv3x3 (DaformerDecoder):\n    def __init__(self, **kwargs):\n        super(daformer_conv3x3, self).__init__(\n            fuse = 'conv3x3',\n            **kwargs\n        )\nclass daformer_conv1x1 (DaformerDecoder):\n    def __init__(self, **kwargs):\n        super(daformer_conv1x1, self).__init__(\n            fuse = 'conv1x1',\n            **kwargs\n        )\n\nclass daformer_aspp (DaformerDecoder):\n    def __init__(self, **kwargs):\n        super(daformer_aspp, self).__init__(\n            fuse = 'aspp',\n            **kwargs\n        )\n\nif 0:\n    m = daformer_conv1x1(\n        encoder_dim = [32, 64, 160, 256],\n        decoder_dim = 256,\n        dilation = None,\n\n    )\n    print(m)\n    feature = [\n         torch.rand(1,c,64//(2**i),64//(2**i)) for i,c in enumerate([32, 64, 160, 256])\n    ]\n    y, out = m(feature)\n    print(y.shape)\n    \nclass Net_daformer(nn.Module):\n    def load_pretrain( self,):\n        checkpoint = cfg[self.arch]['checkpoint']\n        print('load %s'%checkpoint)\n        checkpoint = torch.load(checkpoint, map_location=lambda storage, loc: storage)  #True\n        #checkpoint = torch.load('./pytorch_model.bin', map_location=lambda storage, loc: storage)  #True\n        print(self.encoder.load_state_dict(checkpoint,strict=False))  #True\n\n\n    def __init__( self,):\n        super(Net_daformer, self).__init__()\n        self.output_type = ['inference', 'loss']\n        self.rgb = RGB()\n        self.dropout = nn.Dropout(0.1)\n\n        self.arch = 'pvt_v2_b4'\n        self.encoder = cfg[self.arch]['builder']()\n        encoder_dim = self.encoder.embed_dims\n        #[64, 128, 320, 512]\n\n        self.decoder = daformer_conv1x1(\n            encoder_dim=encoder_dim,\n            decoder_dim=320,\n            )\n        self.logit = nn.Sequential(\n            nn.Conv2d(320, 1, kernel_size=1, padding=0),\n        )\n        self.aux = nn.ModuleList([\n            nn.Conv2d(encoder_dim[i], 1, kernel_size=1, padding=0) for i in range(4)\n        ])\n\n\n    def forward(self, batch):\n\n        x = batch['image']\n        x = self.rgb(x)\n\n        B,C,H,W = x.shape\n        encoder = self.encoder(x)\n        #print([f.shape for f in encoder])\n\n        last, decoder = self.decoder(encoder)\n        last  = self.dropout(last)\n        logit = self.logit(last)\n        logit = F.interpolate(logit, size=None, scale_factor=4, mode='bilinear', align_corners=False)\n        #print(logit.shape)\n\n        output = {}\n        if 'loss' in self.output_type:\n            output['bce_loss'] = F.binary_cross_entropy_with_logits(logit,batch['mask'])\n            for i in range(4):\n                output['aux%d_loss'%i] = criterion_aux_loss(self.aux[i](encoder[i]),batch['mask'])\n\n        if 'inference' in self.output_type:\n            output['probability'] = torch.sigmoid(logit)\n\n        return output\n\ndef criterion_aux_loss(logit, mask):\n    mask = F.interpolate(mask,size=logit.shape[-2:], mode='nearest')\n    loss = F.binary_cross_entropy_with_logits(logit,mask)\n    return loss\n","metadata":{"execution":{"iopub.status.busy":"2022-09-20T01:36:37.288286Z","iopub.execute_input":"2022-09-20T01:36:37.289022Z","iopub.status.idle":"2022-09-20T01:36:39.403043Z","shell.execute_reply.started":"2022-09-20T01:36:37.288927Z","shell.execute_reply":"2022-09-20T01:36:39.401074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"image_size = 768\n\n\norgan_threshold = {\n    'Hubmap': {\n        'kidney'        : 0.45, \n        'prostate'      : 0.40, #0.40 best single\n        'largeintestine': 0.30, #0.30 BEST ENSEMBLE\n        'spleen'        : 0.30, #0.30 BEST ENSEMBLE\n        'lung'          : 0.07, #0.06 best ensemble(5 ensemvle), 0.07 best ensemble(4 ensemvle)\n    },\n    'HPA': {\n        'kidney'        : 0.50,\n        'prostate'      : 0.50,\n        'largeintestine': 0.50,\n        'spleen'        : 0.50,\n        'lung'          : 0.10,\n    },\n}\n\n\n#0.80 v29 best\n\npretrained1 = torch.load('../input/pvtv2-b4-768-fold3-4/00018900.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \npretrained4 = torch.load('../input/pvtv2-768-spleen/00003780.model.pth', map_location=lambda storage, loc: storage)['state_dict']\npretrained2 = torch.load('../input/d/chris666/pvtv2-b4-768-combined/00017220.model.pth', map_location=lambda storage, loc: storage)['state_dict']\npretrained3 = torch.load('../input/d/chris666/pvtv2-b4-768-combined/00010920.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n\n\nnet1 = Net().cuda()\nnet1.load_state_dict(pretrained1, strict=False)  # True\nnet1 = net1.eval()\nnet1.output_type = ['inference']\n\nnet2 = Net().cuda()\nnet2.load_state_dict(pretrained2, strict=False)  # True\nnet2 = net2.eval()\nnet2.output_type = ['inference']\n\nnet3 = Net().cuda()\nnet3.load_state_dict(pretrained3, strict=False)  # True\nnet3 = net3.eval()\nnet3.output_type = ['inference']\n\nnet4 = Net().cuda()\nnet4.load_state_dict(pretrained4, strict=False)  # True\nnet4 = net4.eval()\nnet4.output_type = ['inference']\n\n\n# 0.80 v29 best\nensemble_mit = [net1, net2, net3, net4]\ndel net1, net2, net3, net4, pretrained1, pretrained2, pretrained3, pretrained4\n\nresult = {\n    'id': [],\n    'probability': [],\n    'rle': [],\n}\n\n\nspleen_df = test_df[((test_df['organ']=='spleen') & (test_df['data_source']=='Hubmap'))]\nprostate_df = test_df[((test_df['organ']=='prostate') & (test_df['data_source']=='Hubmap'))]\nlung_df = test_df[((test_df['organ']=='lung') & (test_df['data_source']=='Hubmap'))]\nother_df = test_df[~(((test_df['organ']=='spleen') & (test_df['data_source']=='Hubmap'))|((test_df['organ']=='prostate') & (test_df['data_source']=='Hubmap'))|((test_df['organ']=='lung') & (test_df['data_source']=='Hubmap')))]\n\nfor person_id in spleen_df['id'].unique():    \n    pid_src = spleen_df[spleen_df['id'] == person_id]['data_source'].values[0]\n    organ_cat = spleen_df[spleen_df['id'] == person_id]['organ'].values[0]\n    image = cv2.imread('../input/hubmap-organ-segmentation/test_images/%d.tiff' % person_id,\n                       cv2.IMREAD_COLOR)\n    image = image.astype(np.float32) / 255\n    H, W, _ = image.shape\n        \n    image_size = 768 #768\n    s = spleen_df[spleen_df['id'] == person_id]['pixel_size'].values[0] / 0.4 * (image_size / 3000)\n    h = int(np.ceil(int(H*s)/32)*32)\n    w = int(np.ceil(int(W*s)/32)*32) \n    image = cv2.resize(image,dsize=(w,h),interpolation=cv2.INTER_LINEAR)  \n\n    \n    new_image = image_to_tensor(image)\n    new_image = new_image.cuda()\n    batch = {\n        'image':\n            torch.stack([\n                new_image,\n                torch.flip(new_image, [1]),\n                torch.flip(new_image, [2]),\n            ]),  # simple TTA\n    }\n    \n\n    prob_list = []\n    for nettt in ensemble_mit:    \n\n        probability = 0\n        p = 0\n        with torch.no_grad():\n            with amp.autocast(enabled=is_amp):\n                output = nettt(batch) \n                probability += F.interpolate(\n                    output['probability'], size=(H, W), mode='bilinear', align_corners=False)\n\n        # undo TTA\n        probability[1] = torch.flip(probability[1], [1])\n        probability[2] = torch.flip(probability[2], [2])\n        probability = probability.float().data.cpu().numpy().mean(0)[0]\n        prob_list.append(probability)\n\n    \n    #0.80 v29 best\n    probability = prob_list[0]*0.25 + prob_list[1]*0.25 + prob_list[2]*0.25 + prob_list[3]*0.25\n\n    del prob_list\n    \n    if pid_src == 'Hubmap':\n        p = probability > organ_threshold['Hubmap'][organ_cat]\n    if pid_src == 'HPA':\n        p = probability > organ_threshold['HPA'][organ_cat]\n        \n\n    mask = cv2.resize(1*p, (H,W),interpolation=cv2.INTER_NEAREST)\n        \n    ##################################################################################################################################\n    \n    plt.imshow(mask)\n    del p\n    rle = rle_encode_less_memory(mask)\n    \n    del mask\n    \n    if pid_src != 'HPA':\n        result['rle'].append(rle)\n\n        result['id'].append(person_id)\n\n\n    else:  \n        #for only Hubmap score test\n        #result['rle'].append('')        \n        #result['id'].append(person_id)   \n        # all score test\n        result['rle'].append(rle)        \n        result['id'].append(person_id)  \n\ndel ensemble_mit\n        \n####################################################################################################################################        \npretrained1 = torch.load('../input/pvtv2-b4-768-fold3-4/00018900.model.pth', map_location=lambda storage, loc: storage)['state_dict']  #Hubmap 0.57\n\nnet1 = Net().cuda()\nnet1.load_state_dict(pretrained1, strict=False)  # True\nnet1 = net1.eval()\nnet1.output_type = ['inference']\n\nensemble_spleen = [net1]\ndel net1, pretrained1\n \nfor person_id in prostate_df['id'].unique():\n\n    pid_src = prostate_df[prostate_df['id'] == person_id]['data_source'].values[0]\n    organ_cat = prostate_df[prostate_df['id'] == person_id]['organ'].values[0]\n\n    image = cv2.imread('../input/hubmap-organ-segmentation/test_images/%d.tiff' % person_id,\n                       cv2.IMREAD_COLOR)\n    image = image.astype(np.float32) / 255\n    H, W, _ = image.shape        \n \n    image_size = 768 #480\n    s = prostate_df[prostate_df['id'] == person_id]['pixel_size'].values[0] / 0.4 * (image_size / 3000)\n    h = int(np.ceil(int(H*s)/32)*32)\n    w = int(np.ceil(int(W*s)/32)*32) \n    image = cv2.resize(image,dsize=(w,h),interpolation=cv2.INTER_LINEAR)  \n\n    new_image = image_to_tensor(image)\n    new_image = new_image.cuda()\n    batch = {\n        'image':\n            torch.stack([\n                new_image,\n                torch.flip(new_image, [1]),\n                torch.flip(new_image, [2]),\n            ]),  # simple TTA\n    }\n\n    prob_list = []\n    for nettt in ensemble_spleen:\n        probability = 0\n        p = 0\n        with torch.no_grad():\n            with amp.autocast(enabled=is_amp):\n                output = nettt(batch)  # data_parallel(net, batch) #          \n                # probability += output['probability']\n                probability += F.interpolate(\n                    output['probability'], size=(H, W), mode='bilinear', align_corners=False)\n\n        # undo TTA\n        probability[1] = torch.flip(probability[1], [1])\n        probability[2] = torch.flip(probability[2], [2])\n        probability = probability.float().data.cpu().numpy().mean(0)[0]\n        prob_list.append(probability)\n\n    #0.80 v18\n    probability = prob_list[0]\n    del prob_list\n    if pid_src == 'Hubmap':\n        p = probability > organ_threshold['Hubmap'][organ_cat]\n    if pid_src == 'HPA':\n        p = probability > organ_threshold['HPA'][organ_cat]\n\n    mask = cv2.resize(1*p, (H,W),interpolation=cv2.INTER_NEAREST)\n\n    del p\n    rle = rle_encode_less_memory(mask)    \n    del mask\n    \n    if pid_src != 'HPA':\n        result['rle'].append(rle)\n        result['id'].append(person_id)\n\n    else:  \n        #for only Hubmap score test\n        #result['rle'].append('')        \n        #result['id'].append(person_id)   \n        # all score test\n        result['rle'].append(rle)        \n        result['id'].append(person_id)   \ndel ensemble_spleen\n\n\n# 0.80 v18 best\npretrained1 = torch.load('../input/pvtv2-b4-768-fold3-4/00018900.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \n# 0.80 v29 best\npretrained_lung = torch.load('../input/pvtv2-daf-lung/00003780.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \npretrained_lung2 = torch.load('../input/pvtv2-daf-lung-fold2/00004590.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \n#0.80 v49 best\npretrained_lung3 = torch.load('../input/d/chris666/pvtv2-lung-allstain/00004590.model.pth', map_location=lambda storage, loc: storage)['state_dict']  \n# 0.80 v49 best\npretrained_lung4 = torch.load('../input/pvtv2-lunggg-2/00004752.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n\nnet1 = Net().cuda()\nnet1.load_state_dict(pretrained1, strict=False)  # True\nnet1 = net1.eval()\nnet1.output_type = ['inference']\n\n\n# 0.80 v29 best\nnet2 = Net_daformer().cuda()\nnet2.load_state_dict(pretrained_lung, strict=False)  # True\nnet2 = net2.eval()\nnet2.output_type = ['inference']\n\nnet3 = Net_daformer().cuda()\nnet3.load_state_dict(pretrained_lung2, strict=False)  # True\nnet3 = net3.eval()\nnet3.output_type = ['inference']\n\n\nnet4 = Net().cuda()\nnet4.load_state_dict(pretrained_lung3, strict=False)  # True\nnet4 = net4.eval()\nnet4.output_type = ['inference']\n\nnet5 = Net().cuda()\nnet5.load_state_dict(pretrained_lung4, strict=False)  # True\nnet5 = net5.eval()\nnet5.output_type = ['inference']\n  \n# 0.80 v27 best\nensemble_lung = [net1, net2, net3, net4, net5]\ndel net1, net2, net3, net4, net5, pretrained1, pretrained_lung, pretrained_lung2, pretrained_lung3, pretrained_lung4   \n\nfor person_id in lung_df['id'].unique():\n    pid_src = lung_df[lung_df['id'] == person_id]['data_source'].values[0]\n    organ_cat = lung_df[lung_df['id'] == person_id]['organ'].values[0]\n\n    image = cv2.imread('../input/hubmap-organ-segmentation/test_images/%d.tiff' % person_id,\n                       cv2.IMREAD_COLOR)\n    image = image.astype(np.float32) / 255\n    H, W, _ = image.shape        \n \n    image_size = 768 #480\n    s = lung_df[lung_df['id'] == person_id]['pixel_size'].values[0] / 0.4 * (image_size / 3000)\n    h = int(np.ceil(int(H*s)/32)*32)\n    w = int(np.ceil(int(W*s)/32)*32) \n    image = cv2.resize(image,dsize=(w,h),interpolation=cv2.INTER_LINEAR)  \n\n    new_image = image_to_tensor(image)\n    new_image = new_image.cuda()\n    batch = {\n        'image':\n            torch.stack([\n                new_image,\n                torch.flip(new_image, [1]),\n                torch.flip(new_image, [2]),\n            ]),  # simple TTA\n    }\n\n    prob_list = []\n    for nettt in ensemble_lung:\n        probability = 0\n        p = 0\n        with torch.no_grad():\n            with amp.autocast(enabled=is_amp):\n                output = nettt(batch)\n                probability += F.interpolate(\n                    output['probability'], size=(H, W), mode='bilinear', align_corners=False)\n\n        # undo TTA\n        probability[1] = torch.flip(probability[1], [1])\n        probability[2] = torch.flip(probability[2], [2])\n        probability = probability.float().data.cpu().numpy().mean(0)[0]\n        prob_list.append(probability)\n    \n    probability = prob_list[0]*0.28 + prob_list[1]*0.18 +prob_list[2]*0.18 + prob_list[3]*0.18 + prob_list[4]*0.18\n    del prob_list\n    if pid_src == 'Hubmap':\n        p = probability > organ_threshold['Hubmap'][organ_cat]\n    if pid_src == 'HPA':\n        p = probability > organ_threshold['HPA'][organ_cat]\n\n    mask = cv2.resize(1*p, (H,W),interpolation=cv2.INTER_NEAREST)\n    del p\n    rle = rle_encode_less_memory(mask)    \n    del mask\n    \n    if pid_src != 'HPA':\n        result['rle'].append(rle)\n        result['id'].append(person_id)\n\n    else:  \n        #for only Hubmap score test\n        #result['rle'].append('')        \n        #result['id'].append(person_id)   \n        # all score test\n        result['rle'].append(rle)        \n        result['id'].append(person_id)   \ndel ensemble_lung\n\npretrained2 = torch.load('../input/d/chris666/pvtv2-b4-768-combined/00017220.model.pth', map_location=lambda storage, loc: storage)['state_dict']\npretrained3 = torch.load('../input/d/chris666/pvtv2-b4-768-combined/00010920.model.pth', map_location=lambda storage, loc: storage)['state_dict']\npretrained1 = torch.load('../input/pvtv2-b4-768-fold3-4/00018900.model.pth', map_location=lambda storage, loc: storage)['state_dict']  #Hubmap 0.57\n\nnet1 = Net().cuda()\nnet1.load_state_dict(pretrained1, strict=False)  # True\nnet1 = net1.eval()\nnet1.output_type = ['inference']\n\nnet2 = Net().cuda()\nnet2.load_state_dict(pretrained2, strict=False)  # True\nnet2 = net2.eval()\nnet2.output_type = ['inference']\n\nnet3 = Net().cuda()\nnet3.load_state_dict(pretrained3, strict=False)  # True\nnet3 = net3.eval()\nnet3.output_type = ['inference']\n\nensemble_weight = [net1, net2, net3]\ndel net1, net2, net3, pretrained1, pretrained2, pretrained3\n\n\nfor person_id in other_df['id'].unique():\n    pid_src = other_df[other_df['id'] == person_id]['data_source'].values[0]\n    organ_cat = other_df[other_df['id'] == person_id]['organ'].values[0]\n\n    image = cv2.imread('../input/hubmap-organ-segmentation/test_images/%d.tiff' % person_id,\n                       cv2.IMREAD_COLOR)\n    image = image.astype(np.float32) / 255\n    H, W, _ = image.shape        \n \n    image_size = 768 #480\n    s = other_df[other_df['id'] == person_id]['pixel_size'].values[0] / 0.4 * (image_size / 3000)\n    h = int(np.ceil(int(H*s)/32)*32)\n    w = int(np.ceil(int(W*s)/32)*32) \n    image = cv2.resize(image,dsize=(w,h),interpolation=cv2.INTER_LINEAR)  \n\n    new_image = image_to_tensor(image)\n    new_image = new_image.cuda()\n    batch = {\n        'image':\n            torch.stack([\n                new_image,\n                torch.flip(new_image, [1]),\n                torch.flip(new_image, [2]),\n            ]),  # simple TTA\n    }\n\n    prob_list = []\n    for nettt in ensemble_weight:\n        probability = 0\n        p = 0\n        with torch.no_grad():\n            with amp.autocast(enabled=is_amp):\n                output = nettt(batch)\n                probability += F.interpolate(\n                    output['probability'], size=(H, W), mode='bilinear', align_corners=False)\n\n        # undo TTA\n        probability[1] = torch.flip(probability[1], [1])\n        probability[2] = torch.flip(probability[2], [2])\n        probability = probability.float().data.cpu().numpy().mean(0)[0]\n        prob_list.append(probability)\n\n    probability = prob_list[0]*0.65 + prob_list[1]*0.3 + prob_list[2]*0.05\n    del prob_list\n    if pid_src == 'Hubmap':\n        p = probability > organ_threshold['Hubmap'][organ_cat]\n    if pid_src == 'HPA':\n        p = probability > organ_threshold['HPA'][organ_cat]\n\n    mask = cv2.resize(1*p, (H,W),interpolation=cv2.INTER_NEAREST)\n\n    del p\n    rle = rle_encode_less_memory(mask)    \n    del mask\n    \n    if pid_src != 'HPA':\n        result['rle'].append(rle)\n        result['id'].append(person_id)\n    else:  \n        #for only Hubmap score test\n        #result['rle'].append('')        \n        #result['id'].append(person_id)   \n        # all score test\n        result['rle'].append(rle)        \n        result['id'].append(person_id)   \ndel ensemble_weight\n\n# ---\nsubmit_df = pd.DataFrame({'id': result['id'], 'rle': result['rle']})\ntmp_list = []\nfor i in test_df['id'].values:\n    tmp_list.append(submit_df[submit_df['id']==i].values[0])\ndel submit_df\nsubmit_df = pd.DataFrame(tmp_list)\ndel tmp_list\nsubmit_df.columns = ['id','rle']\n#submit_df=new_df\n\nprint(submit_df)\nprint('submit_df ok!')\nprint('')\n\nsubmit_df.to_csv('submission.csv', index=False)\n","metadata":{},"execution_count":null,"outputs":[]}]}