{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n'''\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n'''\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-22T08:01:49.43538Z","iopub.execute_input":"2022-08-22T08:01:49.435868Z","iopub.status.idle":"2022-08-22T08:01:49.472961Z","shell.execute_reply.started":"2022-08-22T08:01:49.435764Z","shell.execute_reply":"2022-08-22T08:01:49.471762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nimport os\nimport numpy as np\nimport PIL.Image\nimport cv2\nimport matplotlib.pyplot as plt\n\nimport pandas as pd\n\nimport os\nimport cv2\nimport time\nimport random\n\nimport torch\nfrom torch import nn\nimport torch.cuda.amp as amp\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import RandomSampler \nfrom torch.utils.data import SequentialSampler\nimport torch.nn.functional as F\nfrom torchmetrics.functional import dice_score\nfrom torch.optim.lr_scheduler import StepLR\n\nis_amp = True\nimport logging\nimport pandas as pd\nfrom sklearn.model_selection import KFold\n\nimport numpy as np\nfrom itertools import repeat\nimport collections.abc","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:01:49.47521Z","iopub.execute_input":"2022-08-22T08:01:49.476335Z","iopub.status.idle":"2022-08-22T08:01:53.594218Z","shell.execute_reply.started":"2022-08-22T08:01:49.476291Z","shell.execute_reply":"2022-08-22T08:01:53.59275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport math\nimport warnings\n\nfrom torch.nn.init import _calculate_fan_in_and_fan_out\n\n\ndef trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):\n # type: (Tensor, float, float, float, float) -> Tensor\n r\"\"\"Fills the input Tensor with values drawn from a truncated\n normal distribution. The values are effectively drawn from the\n normal distribution :math:`\\mathcal{N}(\\text{mean}, \\text{std}^2)`\n with values outside :math:`[a, b]` redrawn until they are within\n the bounds. The method used for generating the random values works\n best when :math:`a \\leq \\text{mean} \\leq b`.\n NOTE: this impl is similar to the PyTorch trunc_normal_, the bounds [a, b] are\n applied while sampling the normal with mean/std applied, therefore a, b args\n should be adjusted to match the range of mean, std args.\n Args:\n tensor: an n-dimensional `torch.Tensor`\n mean: the mean of the normal distribution\n std: the standard deviation of the normal distribution\n a: the minimum cutoff value\n b: the maximum cutoff value\n Examples:\n >>> w = torch.empty(3, 5)\n >>> nn.init.trunc_normal_(w)\n \"\"\"\n return _no_grad_trunc_normal_(tensor, mean, std, a, b)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:01.623276Z","iopub.execute_input":"2022-08-22T08:08:01.624241Z","iopub.status.idle":"2022-08-22T08:08:01.631203Z","shell.execute_reply.started":"2022-08-22T08:08:01.624192Z","shell.execute_reply":"2022-08-22T08:08:01.630195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\ndef drop_block_2d(\n x, drop_prob: float = 0.1, block_size: int = 7, gamma_scale: float = 1.0,\n with_noise: bool = False, inplace: bool = False, batchwise: bool = False):\n \"\"\" DropBlock. See https://arxiv.org/pdf/1810.12890.pdf\n DropBlock with an experimental gaussian noise option. This layer has been tested on a few training\n runs with success, but needs further validation and possibly optimization for lower runtime impact.\n \"\"\"\n B, C, H, W = x.shape\n total_size = W * H\n clipped_block_size = min(block_size, min(W, H))\n # seed_drop_rate, the gamma parameter\n gamma = gamma_scale * drop_prob * total_size / clipped_block_size ** 2 / (\n (W - block_size + 1) * (H - block_size + 1))\n\n # Forces the block to be inside the feature map.\n w_i, h_i = torch.meshgrid(torch.arange(W).to(x.device), torch.arange(H).to(x.device))\n valid_block = ((w_i >= clipped_block_size // 2) & (w_i < W - (clipped_block_size - 1) // 2)) & \\\n ((h_i >= clipped_block_size // 2) & (h_i < H - (clipped_block_size - 1) // 2))\n valid_block = torch.reshape(valid_block, (1, 1, H, W)).to(dtype=x.dtype)\n\n if batchwise:\n # one mask for whole batch, quite a bit faster\n uniform_noise = torch.rand((1, C, H, W), dtype=x.dtype, device=x.device)\n else:\n uniform_noise = torch.rand_like(x)\n block_mask = ((2 - gamma - valid_block + uniform_noise) >= 1).to(dtype=x.dtype)\n block_mask = -F.max_pool2d(\n -block_mask,\n kernel_size=clipped_block_size, # block_size,\n stride=1,\n padding=clipped_block_size // 2)\n\n if with_noise:\n normal_noise = torch.randn((1, C, H, W), dtype=x.dtype, device=x.device) if batchwise else torch.randn_like(x)\n if inplace:\n x.mul_(block_mask).add_(normal_noise * (1 - block_mask))\n else:\n x = x * block_mask + normal_noise * (1 - block_mask)\n else:\n normalize_scale = (block_mask.numel() / block_mask.to(dtype=torch.float32).sum().add(1e-7)).to(x.dtype)\n if inplace:\n x.mul_(block_mask * normalize_scale)\n else:\n x = x * block_mask * normalize_scale\n return x\n\n\ndef drop_block_fast_2d(\n x: torch.Tensor, drop_prob: float = 0.1, block_size: int = 7,\n gamma_scale: float = 1.0, with_noise: bool = False, inplace: bool = False):\n \"\"\" DropBlock. See https://arxiv.org/pdf/1810.12890.pdf\n DropBlock with an experimental gaussian noise option. Simplied from above without concern for valid\n block mask at edges.\n \"\"\"\n B, C, H, W = x.shape\n total_size = W * H\n clipped_block_size = min(block_size, min(W, H))\n gamma = gamma_scale * drop_prob * total_size / clipped_block_size ** 2 / (\n (W - block_size + 1) * (H - block_size + 1))\n\n block_mask = torch.empty_like(x).bernoulli_(gamma)\n block_mask = F.max_pool2d(\n block_mask.to(x.dtype), kernel_size=clipped_block_size, stride=1, padding=clipped_block_size // 2)\n\n if with_noise:\n normal_noise = torch.empty_like(x).normal_()\n if inplace:\n x.mul_(1. - block_mask).add_(normal_noise * block_mask)\n else:\n x = x * (1. - block_mask) + normal_noise * block_mask\n else:\n block_mask = 1 - block_mask\n normalize_scale = (block_mask.numel() / block_mask.to(dtype=torch.float32).sum().add(1e-6)).to(dtype=x.dtype)\n if inplace:\n x.mul_(block_mask * normalize_scale)\n else:\n x = x * block_mask * normalize_scale\n return x\n\n\nclass DropBlock2d(nn.Module):\n \"\"\" DropBlock. See https://arxiv.org/pdf/1810.12890.pdf\n \"\"\"\n\n def __init__(\n self,\n drop_prob: float = 0.1,\n block_size: int = 7,\n gamma_scale: float = 1.0,\n with_noise: bool = False,\n inplace: bool = False,\n batchwise: bool = False,\n fast: bool = True):\n super(DropBlock2d, self).__init__()\n self.drop_prob = drop_prob\n self.gamma_scale = gamma_scale\n self.block_size = block_size\n self.with_noise = with_noise\n self.inplace = inplace\n self.batchwise = batchwise\n self.fast = fast # FIXME finish comparisons of fast vs not\n\n def forward(self, x):\n if not self.training or not self.drop_prob:\n return x\n if self.fast:\n return drop_block_fast_2d(\n x, self.drop_prob, self.block_size, self.gamma_scale, self.with_noise, self.inplace)\n else:\n return drop_block_2d(\n x, self.drop_prob, self.block_size, self.gamma_scale, self.with_noise, self.inplace, self.batchwise)\n\n\ndef drop_path(x, drop_prob: float = 0., training: bool = False, scale_by_keep: bool = True):\n \"\"\"Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).\n This is the same as the DropConnect impl I created for EfficientNet, etc networks, however,\n the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...\n See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for\n changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use\n 'survival rate' as the argument.\n \"\"\"\n if drop_prob == 0. or not training:\n return x\n keep_prob = 1 - drop_prob\n shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets\n random_tensor = x.new_empty(shape).bernoulli_(keep_prob)\n if keep_prob > 0.0 and scale_by_keep:\n random_tensor.div_(keep_prob)\n return x * random_tensor\n\n\nclass DropPath(nn.Module):\n \"\"\"Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).\n \"\"\"\n def __init__(self, drop_prob: float = 0., scale_by_keep: bool = True):\n super(DropPath, self).__init__()\n self.drop_prob = drop_prob\n self.scale_by_keep = scale_by_keep\n\n def forward(self, x):\n return drop_path(x, self.drop_prob, self.training, self.scale_by_keep)\n\n def extra_repr(self):\n return f'drop_prob={round(self.drop_prob,3):0.3f}'","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:02.186579Z","iopub.execute_input":"2022-08-22T08:08:02.186924Z","iopub.status.idle":"2022-08-22T08:08:02.213392Z","shell.execute_reply.started":"2022-08-22T08:08:02.186895Z","shell.execute_reply":"2022-08-22T08:08:02.212463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from itertools import repeat\nimport collections.abc\n\n\n# From PyTorch internals\ndef _ntuple(n):\n def parse(x):\n if isinstance(x, collections.abc.Iterable) and not isinstance(x, str):\n return x\n return tuple(repeat(x, n))\n return parse\n\n\nto_1tuple = _ntuple(1)\nto_2tuple = _ntuple(2)\nto_3tuple = _ntuple(3)\nto_4tuple = _ntuple(4)\nto_ntuple = _ntuple\n\n\ndef make_divisible(v, divisor=8, min_value=None, round_limit=.9):\n min_value = min_value or divisor\n new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)\n # Make sure that round down does not go down by more than 10%.\n if new_v < round_limit * v:\n new_v += divisor\n return new_v","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:02.926263Z","iopub.execute_input":"2022-08-22T08:08:02.926953Z","iopub.status.idle":"2022-08-22T08:08:02.93443Z","shell.execute_reply.started":"2022-08-22T08:08:02.926917Z","shell.execute_reply":"2022-08-22T08:08:02.933352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nimport re\nimport fnmatch\nfrom collections import defaultdict\nfrom copy import deepcopy\n\n__all__ = ['list_models', 'is_model', 'model_entrypoint', 'list_modules', 'is_model_in_modules',\n 'is_pretrained_cfg_key', 'has_pretrained_cfg_key', 'get_pretrained_cfg_value', 'is_model_pretrained']\n\n_module_to_models = defaultdict(set) # dict of sets to check membership of model in module\n_model_to_module = {} # mapping of model names to module names\n_model_entrypoints = {} # mapping of model names to entrypoint fns\n_model_has_pretrained = set() # set of model names that have pretrained weight url present\n_model_pretrained_cfgs = dict() # central repo for model default_cfgs\n\n\ndef register_model(fn):\n # lookup containing module\n mod = sys.modules[fn.__module__]\n module_name_split = fn.__module__.split('.')\n module_name = module_name_split[-1] if len(module_name_split) else ''\n\n # add model to __all__ in module\n model_name = fn.__name__\n if hasattr(mod, '__all__'):\n mod.__all__.append(model_name)\n else:\n mod.__all__ = [model_name]\n\n # add entries to registry dict/sets\n _model_entrypoints[model_name] = fn\n _model_to_module[model_name] = module_name\n _module_to_models[module_name].add(model_name)\n has_valid_pretrained = False # check if model has a pretrained url to allow filtering on this\n if hasattr(mod, 'default_cfgs') and model_name in mod.default_cfgs:\n # this will catch all models that have entrypoint matching cfg key, but miss any aliasing\n # entrypoints or non-matching combos\n cfg = mod.default_cfgs[model_name]\n has_valid_pretrained = (\n ('url' in cfg and 'http' in cfg['url']) or\n ('file' in cfg and cfg['file']) or\n ('hf_hub_id' in cfg and cfg['hf_hub_id'])\n )\n _model_pretrained_cfgs[model_name] = mod.default_cfgs[model_name]\n if has_valid_pretrained:\n _model_has_pretrained.add(model_name)\n return fn\n\n\ndef _natural_key(string_):\n return [int(s) if s.isdigit() else s for s in re.split(r'(\\d+)', string_.lower())]\n\n\ndef list_models(filter='', module='', pretrained=False, exclude_filters='', name_matches_cfg=False):\n \"\"\" Return list of available model names, sorted alphabetically\n Args:\n filter (str) - Wildcard filter string that works with fnmatch\n module (str) - Limit model selection to a specific sub-module (ie 'gen_efficientnet')\n pretrained (bool) - Include only models with pretrained weights if True\n exclude_filters (str or list[str]) - Wildcard filters to exclude models after including them with filter\n name_matches_cfg (bool) - Include only models w/ model_name matching default_cfg name (excludes some aliases)\n Example:\n model_list('gluon_resnet*') -- returns all models starting with 'gluon_resnet'\n model_list('*resnext*, 'resnet') -- returns all models with 'resnext' in 'resnet' module\n \"\"\"\n if module:\n all_models = list(_module_to_models[module])\n else:\n all_models = _model_entrypoints.keys()\n if filter:\n models = []\n include_filters = filter if isinstance(filter, (tuple, list)) else [filter]\n for f in include_filters:\n include_models = fnmatch.filter(all_models, f) # include these models\n if len(include_models):\n models = set(models).union(include_models)\n else:\n models = all_models\n if exclude_filters:\n if not isinstance(exclude_filters, (tuple, list)):\n exclude_filters = [exclude_filters]\n for xf in exclude_filters:\n exclude_models = fnmatch.filter(models, xf) # exclude these models\n if len(exclude_models):\n models = set(models).difference(exclude_models)\n if pretrained:\n models = _model_has_pretrained.intersection(models)\n if name_matches_cfg:\n models = set(_model_pretrained_cfgs).intersection(models)\n return list(sorted(models, key=_natural_key))\n\n\ndef is_model(model_name):\n \"\"\" Check if a model name exists\n \"\"\"\n return model_name in _model_entrypoints\n\n\ndef model_entrypoint(model_name):\n \"\"\"Fetch a model entrypoint for specified model name\n \"\"\"\n return _model_entrypoints[model_name]\n\n\ndef list_modules():\n \"\"\" Return list of module names that contain models / model entrypoints\n \"\"\"\n modules = _module_to_models.keys()\n return list(sorted(modules))\n\n\ndef is_model_in_modules(model_name, module_names):\n \"\"\"Check if a model exists within a subset of modules\n Args:\n model_name (str) - name of model to check\n module_names (tuple, list, set) - names of modules to search in\n \"\"\"\n assert isinstance(module_names, (tuple, list, set))\n return any(model_name in _module_to_models[n] for n in module_names)\n\n\ndef is_model_pretrained(model_name):\n return model_name in _model_has_pretrained\n\n\ndef get_pretrained_cfg(model_name):\n if model_name in _model_pretrained_cfgs:\n return deepcopy(_model_pretrained_cfgs[model_name])\n return {}\n\n\ndef has_pretrained_cfg_key(model_name, cfg_key):\n \"\"\" Query model default_cfgs for existence of a specific key.\n \"\"\"\n if model_name in _model_pretrained_cfgs and cfg_key in _model_pretrained_cfgs[model_name]:\n return True\n return False\n\n\ndef is_pretrained_cfg_key(model_name, cfg_key):\n \"\"\" Return truthy value for specified model default_cfg key, False if does not exist.\n \"\"\"\n if model_name in _model_pretrained_cfgs and _model_pretrained_cfgs[model_name].get(cfg_key, False):\n return True\n return False\n\n\ndef get_pretrained_cfg_value(model_name, cfg_key):\n \"\"\" Get a specific model default_cfg value by key. None if it doesn't exist.\n \"\"\"\n if model_name in _model_pretrained_cfgs:\n return _model_pretrained_cfgs[model_name].get(cfg_key, None)\n return None","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:03.613789Z","iopub.execute_input":"2022-08-22T08:08:03.614138Z","iopub.status.idle":"2022-08-22T08:08:03.634956Z","shell.execute_reply.started":"2022-08-22T08:08:03.614106Z","shell.execute_reply":"2022-08-22T08:08:03.633716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _cfg(url='', **kwargs):\n return {\n 'url': url,\n 'num_classes': 1000, 'input_size': (3, 224, 224), 'pool_size': None,\n 'crop_pct': .9, 'interpolation': 'bicubic', 'fixed_input_size': True,\n 'mean': IMAGENET_INCEPTION_MEAN, 'std': IMAGENET_INCEPTION_STD,\n 'first_conv': 'patch_embed.proj', 'classifier': 'head',\n **kwargs\n }","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:04.203015Z","iopub.execute_input":"2022-08-22T08:08:04.203865Z","iopub.status.idle":"2022-08-22T08:08:04.210459Z","shell.execute_reply.started":"2022-08-22T08:08:04.203816Z","shell.execute_reply":"2022-08-22T08:08:04.209464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom functools import partial\n\n#from timm.models.layers import DropPath, to_2tuple, trunc_normal_\n#from timm.models.registry import register_model\n#from timm.models.vision_transformer import _cfg\n#from mmseg.models.builder import BACKBONES\n#from mmseg.utils import get_root_logger\n#from mmcv.runner import load_checkpoint\nimport math\n\n############################################################################################\n\nclass Mlp(nn.Module):\n def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0., linear=False):\n super().__init__()\n out_features = out_features or in_features\n hidden_features = hidden_features or in_features\n self.fc1 = nn.Linear(in_features, hidden_features)\n self.dwconv = DWConv(hidden_features)\n self.act = act_layer()\n self.fc2 = nn.Linear(hidden_features, out_features)\n self.drop = nn.Dropout(drop)\n self.linear = linear\n\n if self.linear:\n self.relu = nn.ReLU(inplace=True)\n self.apply(self._init_weights)\n\n def _init_weights(self, m):\n if isinstance(m, nn.Linear):\n trunc_normal_(m.weight, std=.02)\n if isinstance(m, nn.Linear) and m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.LayerNorm):\n nn.init.constant_(m.bias, 0)\n nn.init.constant_(m.weight, 1.0)\n elif isinstance(m, nn.Conv2d):\n fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n fan_out //= m.groups\n m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n if m.bias is not None:\n m.bias.data.zero_()\n\n def forward(self, x, H, W):\n x = self.fc1(x)\n if self.linear:\n x = self.relu(x)\n x = self.dwconv(x, H, W)\n x = self.act(x)\n x = self.drop(x)\n x = self.fc2(x)\n x = self.drop(x)\n return x\n\n\nclass Attention(nn.Module):\n def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., sr_ratio=1, linear=False):\n super().__init__()\n assert dim % num_heads == 0, f\"dim {dim} should be divided by num_heads {num_heads}.\"\n\n self.dim = dim\n self.num_heads = num_heads\n head_dim = dim // num_heads\n self.scale = qk_scale or head_dim ** -0.5\n\n self.q = nn.Linear(dim, dim, bias=qkv_bias)\n self.kv = nn.Linear(dim, dim * 2, bias=qkv_bias)\n self.attn_drop = nn.Dropout(attn_drop)\n self.proj = nn.Linear(dim, dim)\n self.proj_drop = nn.Dropout(proj_drop)\n\n self.linear = linear\n self.sr_ratio = sr_ratio\n if not linear:\n if sr_ratio > 1:\n self.sr = nn.Conv2d(dim, dim, kernel_size=sr_ratio, stride=sr_ratio)\n self.norm = nn.LayerNorm(dim)\n else:\n self.pool = nn.AdaptiveAvgPool2d(7)\n self.sr = nn.Conv2d(dim, dim, kernel_size=1, stride=1)\n self.norm = nn.LayerNorm(dim)\n self.act = nn.GELU()\n self.apply(self._init_weights)\n\n def _init_weights(self, m):\n if isinstance(m, nn.Linear):\n trunc_normal_(m.weight, std=.02)\n if isinstance(m, nn.Linear) and m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.LayerNorm):\n nn.init.constant_(m.bias, 0)\n nn.init.constant_(m.weight, 1.0)\n elif isinstance(m, nn.Conv2d):\n fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n fan_out //= m.groups\n m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n if m.bias is not None:\n m.bias.data.zero_()\n\n def forward(self, x, H, W):\n B, N, C = x.shape\n q = self.q(x).reshape(B, N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3)\n\n if not self.linear:\n if self.sr_ratio > 1:\n x_ = x.permute(0, 2, 1).reshape(B, C, H, W)\n x_ = self.sr(x_).reshape(B, C, -1).permute(0, 2, 1)\n x_ = self.norm(x_)\n kv = self.kv(x_).reshape(B, -1, 2, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)\n else:\n kv = self.kv(x).reshape(B, -1, 2, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)\n else:\n x_ = x.permute(0, 2, 1).reshape(B, C, H, W)\n x_ = self.sr(self.pool(x_)).reshape(B, C, -1).permute(0, 2, 1)\n x_ = self.norm(x_)\n x_ = self.act(x_)\n kv = self.kv(x_).reshape(B, -1, 2, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)\n k, v = kv[0], kv[1]\n\n attn = (q @ k.transpose(-2, -1)) * self.scale\n attn = attn.softmax(dim=-1)\n attn = self.attn_drop(attn)\n\n x = (attn @ v).transpose(1, 2).reshape(B, N, C)\n x = self.proj(x)\n x = self.proj_drop(x)\n\n return x\n\n\nclass Block(nn.Module):\n \n def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,\n drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, sr_ratio=1, linear=False):\n super().__init__()\n self.norm1 = norm_layer(dim)\n self.attn = Attention(\n dim,\n num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale,\n attn_drop=attn_drop, proj_drop=drop, sr_ratio=sr_ratio, linear=linear)\n # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here\n self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()\n self.norm2 = norm_layer(dim)\n mlp_hidden_dim = int(dim * mlp_ratio)\n self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop, linear=linear)\n \n self.apply(self._init_weights)\n\n def _init_weights(self, m):\n if isinstance(m, nn.Linear):\n trunc_normal_(m.weight, std=.02)\n if isinstance(m, nn.Linear) and m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.LayerNorm):\n nn.init.constant_(m.bias, 0)\n nn.init.constant_(m.weight, 1.0)\n elif isinstance(m, nn.Conv2d):\n fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n fan_out //= m.groups\n m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n if m.bias is not None:\n m.bias.data.zero_()\n\n def forward(self, x, H, W):\n x = x + self.drop_path(self.attn(self.norm1(x), H, W))\n x = x + self.drop_path(self.mlp(self.norm2(x), H, W))\n\n return x\n\n\nclass OverlapPatchEmbed(nn.Module):\n \"\"\" Image to Patch Embedding\n \"\"\"\n\n def __init__(self, img_size=224, patch_size=7, stride=4, in_chans=3, embed_dim=768):\n super().__init__()\n img_size = to_2tuple(img_size)\n patch_size = to_2tuple(patch_size)\n\n assert max(patch_size) > stride, \"Set larger patch_size than stride\"\n\n self.img_size = img_size\n self.patch_size = patch_size\n self.H, self.W = img_size[0] // stride, img_size[1] // stride\n self.num_patches = self.H * self.W\n self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=stride,\n padding=(patch_size[0] // 2, patch_size[1] // 2))\n self.norm = nn.LayerNorm(embed_dim)\n\n self.apply(self._init_weights)\n\n def _init_weights(self, m):\n if isinstance(m, nn.Linear):\n trunc_normal_(m.weight, std=.02)\n if isinstance(m, nn.Linear) and m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.LayerNorm):\n nn.init.constant_(m.bias, 0)\n nn.init.constant_(m.weight, 1.0)\n elif isinstance(m, nn.Conv2d):\n fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n fan_out //= m.groups\n m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n if m.bias is not None:\n m.bias.data.zero_()\n\n def forward(self, x):\n x = self.proj(x)\n _, _, H, W = x.shape\n x = x.flatten(2).transpose(1, 2)\n x = self.norm(x)\n\n return x, H, W\n\n\nclass PyramidVisionTransformerV2(nn.Module):\n def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dims=[64, 128, 256, 512],\n num_heads=[1, 2, 4, 8], mlp_ratios=[4, 4, 4, 4], qkv_bias=False, qk_scale=None, drop_rate=0.,\n attn_drop_rate=0., drop_path_rate=0., norm_layer=nn.LayerNorm, depths=[3, 4, 6, 3],\n sr_ratios=[8, 4, 2, 1], num_stages=4, linear=False):\n super().__init__()\n # self.num_classes = num_classes\n self.depths = depths\n self.num_stages = num_stages\n self.linear = linear\n self.embed_dims = embed_dims\n\n dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] # stochastic depth decay rule\n cur = 0\n\n for i in range(num_stages):\n patch_embed = OverlapPatchEmbed(img_size=img_size if i == 0 else img_size // (2 ** (i + 1)),\n patch_size=7 if i == 0 else 3,\n stride=4 if i == 0 else 2,\n in_chans=in_chans if i == 0 else embed_dims[i - 1],\n embed_dim=embed_dims[i])\n \n block = nn.ModuleList([Block(\n dim=embed_dims[i], num_heads=num_heads[i], mlp_ratio=mlp_ratios[i], qkv_bias=qkv_bias,\n qk_scale=qk_scale,\n drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[cur + j], norm_layer=norm_layer,\n sr_ratio=sr_ratios[i], linear=linear)\n for j in range(depths[i])])\n norm = norm_layer(embed_dims[i])\n cur += depths[i]\n \n setattr(self, f\"patch_embed{i + 1}\", patch_embed)\n setattr(self, f\"block{i + 1}\", block)\n setattr(self, f\"norm{i + 1}\", norm)\n \n # classification head\n # self.head = nn.Linear(embed_dims[3], num_classes) if num_classes > 0 else nn.Identity()\n \n self.apply(self._init_weights)\n \n def _init_weights(self, m):\n if isinstance(m, nn.Linear):\n trunc_normal_(m.weight, std=.02)\n if isinstance(m, nn.Linear) and m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.LayerNorm):\n nn.init.constant_(m.bias, 0)\n nn.init.constant_(m.weight, 1.0)\n elif isinstance(m, nn.Conv2d):\n fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n fan_out //= m.groups\n m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n if m.bias is not None:\n m.bias.data.zero_()\n\n def init_weights(self, pretrained=None):\n pass\n #if isinstance(pretrained, str):\n #logger = get_root_logger()\n #load_checkpoint(self, pretrained, map_location='cpu', strict=False, logger=logger)\n\n def freeze_patch_emb(self):\n self.patch_embed1.requires_grad = False\n\n @torch.jit.ignore\n def no_weight_decay(self):\n return {'pos_embed1', 'pos_embed2', 'pos_embed3', 'pos_embed4', 'cls_token'} # has pos_embed may be better\n\n def get_classifier(self):\n return self.head\n\n def reset_classifier(self, num_classes, global_pool=''):\n self.num_classes = num_classes\n self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity()\n\n def forward_features(self, x):\n B = x.shape[0]\n outs = []\n\n for i in range(self.num_stages):\n patch_embed = getattr(self, f\"patch_embed{i + 1}\")\n block = getattr(self, f\"block{i + 1}\")\n norm = getattr(self, f\"norm{i + 1}\")\n x, H, W = patch_embed(x)\n for blk in block:\n x = blk(x, H, W)\n x = norm(x)\n x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous()\n outs.append(x)\n\n return outs\n\n def forward(self, x):\n x = self.forward_features(x)\n # x = self.head(x)\n\n return x\n\n\nclass DWConv(nn.Module):\n def __init__(self, dim=768):\n super(DWConv, self).__init__()\n self.dwconv = nn.Conv2d(dim, dim, 3, 1, 1, bias=True, groups=dim)\n\n def forward(self, x, H, W):\n B, N, C = x.shape\n x = x.transpose(1, 2).view(B, C, H, W)\n x = self.dwconv(x)\n x = x.flatten(2).transpose(1, 2)\n\n return x\n\n\ndef _conv_filter(state_dict, patch_size=16):\n \"\"\" convert patch embedding weight from manual patchify + linear proj to conv\"\"\"\n out_dict = {}\n for k, v in state_dict.items():\n if 'patch_embed.proj.weight' in k:\n v = v.reshape((v.shape[0], 3, patch_size, patch_size))\n out_dict[k] = v\n\n return out_dict\n\n\n#@BACKBONES.register_module()\nclass pvt_v2_b1(PyramidVisionTransformerV2):\n def __init__(self, **kwargs):\n super(pvt_v2_b1, self).__init__(\n patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4],\n qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[2, 2, 2, 2], sr_ratios=[8, 4, 2, 1],\n drop_rate=0.0, drop_path_rate=0.1)\n\n#@BACKBONES.register_module()\nclass pvt_v2_b2(PyramidVisionTransformerV2):\n def __init__(self, **kwargs):\n super(pvt_v2_b2, self).__init__(\n patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4],\n qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 4, 6, 3], sr_ratios=[8, 4, 2, 1],\n drop_rate=0.0, drop_path_rate=0.1)\n\n#@BACKBONES.register_module()\nclass pvt_v2_b2_li(PyramidVisionTransformerV2):\n def __init__(self, **kwargs):\n super(pvt_v2_b2_li, self).__init__(\n patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4],\n qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 4, 6, 3], sr_ratios=[8, 4, 2, 1],\n drop_rate=0.0, drop_path_rate=0.1, linear=True)\n\n\n#@BACKBONES.register_module()\nclass pvt_v2_b3(PyramidVisionTransformerV2):\n def __init__(self, **kwargs):\n super(pvt_v2_b3, self).__init__(\n patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4],\n qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 4, 18, 3], sr_ratios=[8, 4, 2, 1],\n drop_rate=0.0, drop_path_rate=0.1)\n\n#@BACKBONES.register_module()\nclass pvt_v2_b4(PyramidVisionTransformerV2):\n def __init__(self, **kwargs):\n super(pvt_v2_b4, self).__init__(\n patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4],\n qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 8, 27, 3], sr_ratios=[8, 4, 2, 1],\n drop_rate=0.0, drop_path_rate=0.1)\n\n\npretrain_dir ='.'\nif 1:\n cfg = dict(\n pvt_v2_b1 = dict(\n checkpoint = pretrain_dir + '/pvt_v2_b1.pth',\n builder = pvt_v2_b1,\n ),\n pvt_v2_b2 = dict(\n checkpoint = pretrain_dir + '/pvt_v2_b2.pth',\n builder = pvt_v2_b2,\n ),\n pvt_v2_b3 = dict(\n checkpoint = pretrain_dir + '/pvt_v2_b3.pth',\n #checkpoint = pretrain_dir + '/pytorch_model.bin',\n builder = pvt_v2_b3,\n\n ),\n pvt_v2_b4 = dict(\n checkpoint = pretrain_dir + '/pvt_v2_b4.pth',\n builder = pvt_v2_b4,\n ),\n\n )\n\nif 0:\n m = pvt_v2_b4()\n print(m)\n x=torch.rand(1,3,224,224)\n y=m(x)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:04.688427Z","iopub.execute_input":"2022-08-22T08:08:04.689406Z","iopub.status.idle":"2022-08-22T08:08:04.78707Z","shell.execute_reply.started":"2022-08-22T08:08:04.689343Z","shell.execute_reply":"2022-08-22T08:08:04.785844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_encode_less_memory(img):\n #the image should be transposed\n #img = cv2.imread(img)\n pixels = img.T.flatten()\n \n # This simplified method requires first and last pixel to be zero\n #pixels[0] = 0\n #pixels[-1] = 0\n #runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n \n pixels = np.concatenate([[0], pixels, [0]])\n runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n \n runs[1::2] -= runs[::2]\n \n return ' '.join(str(x) for x in runs)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:04.789872Z","iopub.execute_input":"2022-08-22T08:08:04.790233Z","iopub.status.idle":"2022-08-22T08:08:04.800225Z","shell.execute_reply.started":"2022-08-22T08:08:04.790196Z","shell.execute_reply":"2022-08-22T08:08:04.799183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(\"../input/hubmap-organ-segmentation/test.csv\")\ntrain_df = pd.read_csv(\"../input/hubmap-organ-segmentation/train.csv\")\ntest_path = \"../input/hubmap-organ-segmentation/test_images\"\n\n#if not os.path.exists('weights'):\n #os.makedirs('weights')\n#pretrain_dir = os.getcwd() + \"/weights\"","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:05.005213Z","iopub.execute_input":"2022-08-22T08:08:05.005808Z","iopub.status.idle":"2022-08-22T08:08:05.02195Z","shell.execute_reply.started":"2022-08-22T08:08:05.005769Z","shell.execute_reply":"2022-08-22T08:08:05.021051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def image_to_tensor(image, mode='bgr'): #image mode\n if mode=='bgr':\n image = image[:,:,::-1]\n x = image\n x = x.transpose(2,0,1)\n x = np.ascontiguousarray(x)\n x = torch.tensor(x, dtype=torch.float)\n return x\n\n\ndef mask_to_tensor(mask):\n x = mask\n x = torch.tensor(x, dtype=torch.float)\n return x\n\n\ntensor_list = ['mask', 'image', 'organ']\n\ndef null_collate(batch):\n d = {}\n key = batch[0].keys()\n for k in key:\n v = [b[k] for b in batch]\n if k in tensor_list:\n v = torch.stack(v)\n d[k] = v\n\n d['mask'] = d['mask'].unsqueeze(1)\n d['organ'] = d['organ'].reshape(-1)\n return d\n\n\ndef _ntuple(n):\n def parse(x):\n if isinstance(x, collections.abc.Iterable):\n return x\n return tuple(repeat(x, n))\n return parse\n\n\nto_2tuple = _ntuple(2)\n\n\ndef _no_grad_trunc_normal_(tensor, mean, std, a, b):\n # Cut & paste from PyTorch official master until it's in a few official releases - RW\n # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf\n def norm_cdf(x):\n # Computes standard normal cumulative distribution function\n return (1. + math.erf(x / math.sqrt(2.))) / 2.\n\n if (mean < a - 2 * std) or (mean > b + 2 * std):\n warnings.warn(\"mean is more than 2 std from [a, b] in nn.init.trunc_normal_. \"\n \"The distribution of values may be incorrect.\",\n stacklevel=2)\n\n \ndef trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):\n return _no_grad_trunc_normal_(tensor, mean, std, a, b)\n\n\ndef drop_path(x, drop_prob: float = 0., training: bool = False, scale_by_keep: bool = True):\n \"\"\"Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).\n This is the same as the DropConnect impl I created for EfficientNet, etc networks, however,\n the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...\n See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for\n changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use\n 'survival rate' as the argument.\n \"\"\"\n if drop_prob == 0. or not training:\n return x\n keep_prob = 1 - drop_prob\n shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets\n random_tensor = x.new_empty(shape).bernoulli_(keep_prob)\n if keep_prob > 0.0 and scale_by_keep:\n random_tensor.div_(keep_prob)\n return x * random_tensor\n\n\nclass DropPath(nn.Module):\n \"\"\"Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).\n \"\"\"\n def __init__(self, drop_prob: float = 0., scale_by_keep: bool = True):\n super(DropPath, self).__init__()\n self.drop_prob = drop_prob\n self.scale_by_keep = scale_by_keep\n\n def forward(self, x):\n return drop_path(x, self.drop_prob, self.training, self.scale_by_keep)\n\n def extra_repr(self):\n return f'drop_prob={round(self.drop_prob,3):0.3f}'\n \n \nclass RGB(nn.Module):\n IMAGE_RGB_MEAN = [0.485, 0.456, 0.406] #[0.5, 0.5, 0.5]\n IMAGE_RGB_STD = [0.229, 0.224, 0.225] #[0.5, 0.5, 0.5]\n\n def __init__(self,):\n super(RGB, self).__init__()\n self.register_buffer('mean', torch.zeros(1,3,1,1))\n self.register_buffer('std', torch.ones(1,3,1,1))\n self.mean.data = torch.FloatTensor(self.IMAGE_RGB_MEAN).view(self.mean.shape)\n self.std.data = torch.FloatTensor(self.IMAGE_RGB_STD).view(self.std.shape)\n\n def forward(self, x):\n x = (x-self.mean)/self.std\n return x\n \n \ndef message(mode='print'):\n asterisk = ' '\n if mode==('print'):\n loss = batch_loss\n if mode==('log'):\n loss = train_loss\n if (iteration % iter_save == 0): asterisk = '*'\n\n text = \\\n ('%0.2e %08d%s %6.2f | '%(rate, iteration, asterisk, epoch,)).replace('e-0','e-').replace('e+0','e+') + \\\n '%4.3f %4.3f %4.4f %4.3f | '%(*valid_loss,) + \\\n '%4.3f %4.3f | '%(*loss,) + \\\n '%s' % ((time.time() - start_timer))\n\n return text","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:05.849693Z","iopub.execute_input":"2022-08-22T08:08:05.850046Z","iopub.status.idle":"2022-08-22T08:08:05.871579Z","shell.execute_reply.started":"2022-08-22T08:08:05.850014Z","shell.execute_reply":"2022-08-22T08:08:05.870325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def valid_augment5(image, mask, organ):\n #image, mask = do_crop(image, mask, image_size, xy=(None,None))\n return image, mask\n\ndef train_augment5b(image, mask, organ):\n image, mask = do_random_flip(image, mask)\n image, mask = do_random_rot90(image, mask)\n\n for fn in np.random.choice([\n lambda image, mask: (image, mask),\n lambda image, mask: do_random_noise(image, mask, mag=0.1),\n lambda image, mask: do_random_contast(image, mask, mag=0.40),\n lambda image, mask: do_random_hsv(image, mask, mag=[0.40, 0.40, 0])\n ], 2): image, mask = fn(image, mask)\n\n for fn in np.random.choice([\n lambda image, mask: (image, mask),\n lambda image, mask: do_random_rotate_scale(image, mask, angle=45, scale=[0.50, 2.0]),\n ], 1): image, mask = fn(image, mask)\n\n return image, mask","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:06.542043Z","iopub.execute_input":"2022-08-22T08:08:06.542421Z","iopub.status.idle":"2022-08-22T08:08:06.551724Z","shell.execute_reply.started":"2022-08-22T08:08:06.542385Z","shell.execute_reply":"2022-08-22T08:08:06.550575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_random_flip(image, mask):\n if np.random.rand()>0.5:\n image = cv2.flip(image,0)\n mask = cv2.flip(mask,0)\n if np.random.rand()>0.5:\n image = cv2.flip(image,1)\n mask = cv2.flip(mask,1)\n if np.random.rand()>0.5:\n image = image.transpose(1,0,2)\n mask = mask.transpose(1,0)\n \n image = np.ascontiguousarray(image)\n mask = np.ascontiguousarray(mask)\n return image, mask\n\ndef do_random_rot90(image, mask):\n r = np.random.choice([\n 0,\n cv2.ROTATE_90_CLOCKWISE,\n cv2.ROTATE_90_COUNTERCLOCKWISE,\n cv2.ROTATE_180,\n ])\n if r==0:\n return image, mask\n else:\n image = cv2.rotate(image, r)\n mask = cv2.rotate(mask, r)\n return image, mask\n \ndef do_random_contast(image, mask, mag=0.3):\n alpha = 1 + random.uniform(-1,1)*mag\n image = image * alpha\n image = np.clip(image,0,1)\n return image, mask\n\ndef do_random_hsv(image, mask, mag=[0.15,0.25,0.25]):\n image = (image*255).astype(np.uint8)\n hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)\n\n h = hsv[:, :, 0].astype(np.float32) # hue\n s = hsv[:, :, 1].astype(np.float32) # saturation\n v = hsv[:, :, 2].astype(np.float32) # value\n h = (h*(1 + random.uniform(-1,1)*mag[0]))%180\n s = s*(1 + random.uniform(-1,1)*mag[1])\n v = v*(1 + random.uniform(-1,1)*mag[2])\n\n hsv[:, :, 0] = np.clip(h,0,180).astype(np.uint8)\n hsv[:, :, 1] = np.clip(s,0,255).astype(np.uint8)\n hsv[:, :, 2] = np.clip(v,0,255).astype(np.uint8)\n image = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)\n image = image.astype(np.float32)/255\n return image, mask\n\ndef do_random_noise(image, mask, mag=0.1):\n height, width = image.shape[:2]\n noise = np.random.uniform(-1,1, (height, width,1))*mag\n image = image + noise\n image = np.clip(image,0,1)\n return image, mask\n\ndef do_random_rotate_scale(image, mask, angle=30, scale=[0.8,1.2] ):\n angle = np.random.uniform(-angle, angle)\n scale = np.random.uniform(*scale) if scale is not None else 1\n \n height, width = image.shape[:2]\n center = (height // 2, width // 2)\n \n transform = cv2.getRotationMatrix2D(center, angle, scale)\n image = cv2.warpAffine( image, transform, (width, height), flags=cv2.INTER_LINEAR,\n borderMode=cv2.BORDER_CONSTANT, borderValue=(0,0,0))\n mask = cv2.warpAffine( mask, transform, (width, height), flags=cv2.INTER_LINEAR,\n borderMode=cv2.BORDER_CONSTANT, borderValue=0)\n return image, mask","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:07.116553Z","iopub.execute_input":"2022-08-22T08:08:07.116856Z","iopub.status.idle":"2022-08-22T08:08:07.134947Z","shell.execute_reply.started":"2022-08-22T08:08:07.116829Z","shell.execute_reply":"2022-08-22T08:08:07.133839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_size = 768\n#image_size = 480\n\nclass HubmapDataset(Dataset):\n def __init__(self, df, augment=None):\n\n self.df = df\n self.augment = augment\n self.length = len(self.df)\n ids = pd.read_csv(LABELS).id.astype(str).values\n self.fnames = [fname for fname in os.listdir(TRAIN) if fname.split('.')[0] in ids]\n self.organ_to_label = {'kidney' : 0,\n 'prostate' : 1,\n 'largeintestine' : 2,\n 'spleen' : 3,\n 'lung' : 4}\n\n def __str__(self):\n string = ''\n string += '\\tlen = %d\\n' % len(self)\n\n d = self.df.organ.value_counts().to_dict()\n for k in ['kidney', 'prostate', 'largeintestine', 'spleen', 'lung']:\n string += '%24s %3d (%0.3f) \\n'%(k,d.get(k,0),d.get(k,0)/len(self.df))\n return string\n\n def __len__(self):\n return self.length\n\n def __getitem__(self, index):\n fname = self.fnames[index]\n d = self.df.iloc[index]\n organ = self.organ_to_label[d.organ]\n\n image = cv2.cvtColor(cv2.imread(os.path.join(TRAIN,fname)), cv2.COLOR_BGR2RGB)\n print('Image name.......................',fname)\n mask = cv2.imread(os.path.join(MASKS,fname.split('.')[0]+'.png'),cv2.IMREAD_GRAYSCALE)\n print(mask)\n plt.imshow(mask)\n \n image = image.astype(np.float32)/255\n mask = mask.astype(np.float32)/255\n\n s = d.pixel_size/0.4 * (image_size/3000)\n image = cv2.resize(image,dsize=(image_size,image_size),interpolation=cv2.INTER_LINEAR)\n mask = cv2.resize(mask, dsize=(image_size,image_size),interpolation=cv2.INTER_LINEAR)\n\n if self.augment is not None:\n image, mask = self.augment(image, mask, organ)\n\n\n r ={}\n r['index']= index\n r['id'] = fname\n r['organ'] = torch.tensor([organ], dtype=torch.long)\n r['image'] = image_to_tensor(image)\n r['mask' ] = mask_to_tensor(mask>0.5)\n return r","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:08.016286Z","iopub.execute_input":"2022-08-22T08:08:08.017324Z","iopub.status.idle":"2022-08-22T08:08:08.031401Z","shell.execute_reply.started":"2022-08-22T08:08:08.017288Z","shell.execute_reply":"2022-08-22T08:08:08.030153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from kaggle_hubmap_v2 import *\n#from common import *\n\n#from mit_pvtv2 import *\n\n#######################################################################################################\n## https://github.com/lucidrains/segformer-pytorch/blob/main/segformer_pytorch/segformer_pytorch.py\n# https://github.com/UAws/CV-3315-Is-All-You-Need\nclass MixUpSample(nn.Module):\n def __init__( self, scale_factor=2):\n super().__init__()\n self.mixing = nn.Parameter(torch.tensor(0.5))\n self.scale_factor = scale_factor\n\n def forward(self, x):\n x = self.mixing *F.interpolate(x, scale_factor=self.scale_factor, mode='bilinear', align_corners=False) \\\n + (1-self.mixing )*F.interpolate(x, scale_factor=self.scale_factor, mode='nearest')\n return x\n\n\nclass to3x3_Decoder(nn.Module):\n def __init__(\n self,\n encoder_dim = [32, 64, 160, 256],\n decoder_dim = 256,\n ):\n super().__init__()\n self.mixing = nn.Parameter(torch.FloatTensor([0.5,0.5,0.5,0.5]))\n self.mlp = nn.ModuleList([\n nn.Sequential(\n nn.Conv2d(dim, decoder_dim, 1, padding= 0, bias=False), #follow mmseg to use conv-bn-relu\n #nn.Conv2d(dim, decoder_dim, 3, padding= 1, bias=False),\n nn.BatchNorm2d(decoder_dim),\n nn.ReLU(inplace=True),\n MixUpSample(2**i) if i!=0 else nn.Identity(),\n ) for i, dim in enumerate(encoder_dim)])\n\n self.fuse = nn.Sequential(\n #nn.Conv2d(len(encoder_dim) * decoder_dim, decoder_dim, 1, padding=0, bias=False),\n nn.Conv2d(len(encoder_dim) * decoder_dim, decoder_dim, 3, padding=1, bias=False),\n nn.BatchNorm2d(decoder_dim),\n nn.ReLU(inplace=True),\n #nn.Conv2d(decoder_dim, decoder_dim, 3, padding=1, bias=False),\n #nn.BatchNorm2d(decoder_dim),\n #nn.ReLU(inplace=True),\n )\n\n def forward(self, feature):\n\n out = []\n for i,f in enumerate(feature):\n f = self.mlp[i](f)\n out.append(f)\n\n x = self.fuse(torch.cat(out, dim = 1))\n return x, out\n\n\n#\n# do a reverse mit upsize + conv + Mix FFN\n\n#################################################################\n\nclass RGB(nn.Module):\n IMAGE_RGB_MEAN = [0.485, 0.456, 0.406] #[0.5, 0.5, 0.5]\n IMAGE_RGB_STD = [0.229, 0.224, 0.225] #[0.5, 0.5, 0.5]\n\n def __init__(self,):\n super(RGB, self).__init__()\n self.register_buffer('mean', torch.zeros(1,3,1,1))\n self.register_buffer('std', torch.ones(1,3,1,1))\n self.mean.data = torch.FloatTensor(self.IMAGE_RGB_MEAN).view(self.mean.shape)\n self.std.data = torch.FloatTensor(self.IMAGE_RGB_STD).view(self.std.shape)\n\n def forward(self, x):\n x = (x-self.mean)/self.std\n return x\n\n\nclass Net(nn.Module):\n def load_pretrain( self,):\n checkpoint = cfg[self.arch]['checkpoint']\n print('load %s'%checkpoint)\n checkpoint = torch.load(checkpoint, map_location=lambda storage, loc: storage) #True\n #checkpoint = torch.load('./pytorch_model.bin', map_location=lambda storage, loc: storage) #True\n print(self.encoder.load_state_dict(checkpoint,strict=False)) #True\n\n\n def __init__( self,):\n super(Net, self).__init__()\n self.output_type = ['inference', 'loss']\n self.rgb = RGB()\n self.dropout = nn.Dropout(0.1)\n\n self.arch = 'pvt_v2_b4'\n self.encoder = cfg[self.arch]['builder']()\n encoder_dim = self.encoder.embed_dims\n #[64, 128, 320, 512]\n\n self.decoder = to3x3_Decoder(\n encoder_dim = encoder_dim,\n decoder_dim = 320,\n )\n self.logit = nn.Sequential(\n nn.Conv2d(320, 1, kernel_size=1, padding=0),\n )\n self.aux = nn.ModuleList([\n nn.Conv2d(encoder_dim[i], 1, kernel_size=1, padding=0) for i in range(4)\n ])\n\n\n def forward(self, batch):\n\n x = batch['image']\n x = self.rgb(x)\n\n B,C,H,W = x.shape\n encoder = self.encoder(x)\n #print([f.shape for f in encoder])\n\n last, decoder = self.decoder(encoder)\n last = self.dropout(last)\n logit = self.logit(last)\n logit = F.interpolate(logit, size=None, scale_factor=4, mode='bilinear', align_corners=False)\n #print(logit.shape)\n\n output = {}\n if 'loss' in self.output_type:\n output['bce_loss'] = F.binary_cross_entropy_with_logits(logit,batch['mask'])\n for i in range(4):\n output['aux%d_loss'%i] = criterion_aux_loss(self.aux[i](encoder[i]),batch['mask'])\n\n if 'inference' in self.output_type:\n output['probability'] = torch.sigmoid(logit)\n\n return output\n\ndef criterion_aux_loss(logit, mask):\n mask = F.interpolate(mask,size=logit.shape[-2:], mode='nearest')\n loss = F.binary_cross_entropy_with_logits(logit,mask)\n return loss\n\n'''\n\nx.shape\ntorch.Size([4, 3, 320, 320])\n\n\nprint([f.shape for f in feature])\n[\ntorch.Size([4, 128, 80, 80]),\ntorch.Size([4, 256, 40, 40]),\ntorch.Size([4, 512, 20, 20]),\ntorch.Size([4, 1024, 10, 10])\n]\n\n\n'''\ndef run_check_net():\n batch_size = 4\n image_size = 512\n\n #---\n batch = {\n 'image' : torch.from_numpy( np.random.uniform(-1,1,(batch_size,3,image_size,image_size)) ).float(),\n 'mask' : torch.from_numpy( np.random.choice(2,(batch_size,1,image_size,image_size)) ).float(),\n 'organ' : torch.from_numpy( np.random.choice(5,(batch_size)) ).long(),\n }\n batch = {k:v.cuda() for k,v in batch.items()}\n\n\n\n net = Net().cuda()\n # torch.save({ 'state_dict': net.state_dict() }, 'model.pth' )\n # exit(0)\n net.load_pretrain()\n\n with torch.no_grad():\n with torch.cuda.amp.autocast(enabled=True):\n output = net(batch)\n\n print('batch')\n for k,v in batch.items():\n print('%32s :'%k, v.shape)\n\n print('output')\n for k,v in output.items():\n if 'loss' not in k:\n print('%32s :'%k, v.shape)\n for k,v in output.items():\n if 'loss' in k:\n print('%32s :'%k, v.item())\n\n'''\nclass mit_b0(MixVisionTransformer):\n def __init__(self, **kwargs):\n super(mit_b0, self).__init__(\n patch_size=4, embed_dims=[32, 64, 160, 256], num_heads=[1, 2, 5, 8], mlp_ratios=[4, 4, 4, 4],\n qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[2, 2, 2, 2], sr_ratios=[8, 4, 2, 1],\n drop_rate=0.0, drop_path_rate=0.1)\n\n\n'''\ndef run_check_mit():\n batch_size = 4\n image_size = 512\n\n #checkpoint = '/root/Downloads/mit_b0.pth'\n #---\n image = torch.from_numpy( np.random.uniform(-1,1,(batch_size,3,image_size,image_size)) ).float()\n image = image.cuda()\n\n net = pvt_v2_b4()\n\n state_dict = torch.load(checkpoint, map_location=lambda storage, loc: storage)\n net.load_state_dict(state_dict,strict=False) #True\n #Unexpected key(s) in state_dict: \"head.weight\", \"head.bias\".\n\n net = net.cuda()\n feature = net(image)\n print([f.shape for f in feature])\n\n# main #################################################################\n#if __name__ == '__main__':\n #run_check_mit()\n #run_check_net()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-22T08:08:34.594578Z","iopub.execute_input":"2022-08-22T08:08:34.594933Z","iopub.status.idle":"2022-08-22T08:08:34.629438Z","shell.execute_reply.started":"2022-08-22T08:08:34.594903Z","shell.execute_reply":"2022-08-22T08:08:34.628396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom functools import partial\n\n#from timm.models.layers import DropPath, to_2tuple, trunc_normal_\n#from timm.models.registry import register_model\n#from timm.models.vision_transformer import _cfg\n\nimport math\n\nclass DWConv3(nn.Module):\n def __init__(self, dim=768):\n super(DWConv3, self).__init__()\n self.dwconv = nn.Conv2d(dim, dim, 3, 1, 1, bias=True, groups=dim)\n\n def forward(self, x, H, W):\n B, N, C = x.shape\n x = x.transpose(1, 2).view(B, C, H, W)\n x = self.dwconv(x)\n x = x.flatten(2).transpose(1, 2)\n\n return x\n\n\nclass Mlp3(nn.Module):\n def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):\n super().__init__()\n out_features = out_features or in_features\n hidden_features = hidden_features or in_features\n self.fc1 = nn.Linear(in_features, hidden_features)\n self.dwconv = DWConv3(hidden_features)\n self.act = act_layer()\n self.fc2 = nn.Linear(hidden_features, out_features)\n self.drop = nn.Dropout(drop)\n\n self.apply(self._init_weights)\n\n def _init_weights(self, m):\n if isinstance(m, nn.Linear):\n trunc_normal_(m.weight, std=.02)\n if isinstance(m, nn.Linear) and m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.LayerNorm):\n nn.init.constant_(m.bias, 0)\n nn.init.constant_(m.weight, 1.0)\n elif isinstance(m, nn.Conv2d):\n fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n fan_out //= m.groups\n m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n if m.bias is not None:\n m.bias.data.zero_()\n\n def forward(self, x, H, W):\n x = self.fc1(x)\n x = self.dwconv(x, H, W)\n x = self.act(x)\n x = self.drop(x)\n x = self.fc2(x)\n x = self.drop(x)\n return x\n\n\nclass Attention3(nn.Module):\n def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., sr_ratio=1):\n super().__init__()\n assert dim % num_heads == 0, f\"dim {dim} should be divided by num_heads {num_heads}.\"\n\n self.dim = dim\n self.num_heads = num_heads\n head_dim = dim // num_heads\n self.scale = qk_scale or head_dim ** -0.5\n\n self.q = nn.Linear(dim, dim, bias=qkv_bias)\n self.kv = nn.Linear(dim, dim * 2, bias=qkv_bias)\n self.attn_drop = nn.Dropout(attn_drop)\n self.proj = nn.Linear(dim, dim)\n self.proj_drop = nn.Dropout(proj_drop)\n\n self.sr_ratio = sr_ratio\n if sr_ratio > 1:\n self.sr = nn.Conv2d(dim, dim, kernel_size=sr_ratio, stride=sr_ratio)\n self.norm = nn.LayerNorm(dim)\n\n self.apply(self._init_weights)\n\n def _init_weights(self, m):\n if isinstance(m, nn.Linear):\n trunc_normal_(m.weight, std=.02)\n if isinstance(m, nn.Linear) and m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.LayerNorm):\n nn.init.constant_(m.bias, 0)\n nn.init.constant_(m.weight, 1.0)\n elif isinstance(m, nn.Conv2d):\n fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n fan_out //= m.groups\n m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n if m.bias is not None:\n m.bias.data.zero_()\n\n def forward(self, x, H, W):\n B, N, C = x.shape\n q = self.q(x).reshape(B, N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3)\n\n if self.sr_ratio > 1:\n x_ = x.permute(0, 2, 1).reshape(B, C, H, W)\n x_ = self.sr(x_).reshape(B, C, -1).permute(0, 2, 1)\n x_ = self.norm(x_)\n kv = self.kv(x_).reshape(B, -1, 2, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)\n else:\n kv = self.kv(x).reshape(B, -1, 2, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)\n k, v = kv[0], kv[1]\n\n attn = (q @ k.transpose(-2, -1)) * self.scale\n attn = attn.softmax(dim=-1)\n attn = self.attn_drop(attn)\n\n x = (attn @ v).transpose(1, 2).reshape(B, N, C)\n x = self.proj(x)\n x = self.proj_drop(x)\n\n return x\n\n\nclass Block3(nn.Module):\n\n def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,\n drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, sr_ratio=1):\n super().__init__()\n self.norm1 = norm_layer(dim)\n self.attn = Attention3(\n dim,\n num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale,\n attn_drop=attn_drop, proj_drop=drop, sr_ratio=sr_ratio)\n # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here\n self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()\n self.norm2 = norm_layer(dim)\n mlp_hidden_dim = int(dim * mlp_ratio)\n self.mlp = Mlp3(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)\n\n self.apply(self._init_weights)\n\n def _init_weights(self, m):\n if isinstance(m, nn.Linear):\n trunc_normal_(m.weight, std=.02)\n if isinstance(m, nn.Linear) and m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.LayerNorm):\n nn.init.constant_(m.bias, 0)\n nn.init.constant_(m.weight, 1.0)\n elif isinstance(m, nn.Conv2d):\n fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n fan_out //= m.groups\n m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n if m.bias is not None:\n m.bias.data.zero_()\n\n def forward(self, x, H, W):\n x = x + self.drop_path(self.attn(self.norm1(x), H, W))\n x = x + self.drop_path(self.mlp(self.norm2(x), H, W))\n\n return x\n\n\nclass OverlapPatchEmbed(nn.Module):\n \"\"\" Image to Patch Embedding\n \"\"\"\n\n def __init__(self, img_size=224, patch_size=7, stride=4, in_chans=3, embed_dim=768):\n super().__init__()\n img_size = to_2tuple(img_size)\n patch_size = to_2tuple(patch_size)\n\n self.img_size = img_size\n self.patch_size = patch_size\n self.H, self.W = img_size[0] // patch_size[0], img_size[1] // patch_size[1]\n self.num_patches = self.H * self.W\n self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=stride,\n padding=(patch_size[0] // 2, patch_size[1] // 2))\n self.norm = nn.LayerNorm(embed_dim)\n\n self.apply(self._init_weights)\n\n def _init_weights(self, m):\n if isinstance(m, nn.Linear):\n trunc_normal_(m.weight, std=.02)\n if isinstance(m, nn.Linear) and m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.LayerNorm):\n nn.init.constant_(m.bias, 0)\n nn.init.constant_(m.weight, 1.0)\n elif isinstance(m, nn.Conv2d):\n fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n fan_out //= m.groups\n m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n if m.bias is not None:\n m.bias.data.zero_()\n\n def forward(self, x):\n x = self.proj(x)\n _, _, H, W = x.shape\n x = x.flatten(2).transpose(1, 2)\n x = self.norm(x)\n\n return x, H, W\n\n\nclass MixVisionTransformer(nn.Module):\n def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dims=[64, 128, 256, 512],\n num_heads=[1, 2, 4, 8], mlp_ratios=[4, 4, 4, 4], qkv_bias=False, qk_scale=None, drop_rate=0.,\n attn_drop_rate=0., drop_path_rate=0., norm_layer=nn.LayerNorm,\n depths=[3, 4, 6, 3], sr_ratios=[8, 4, 2, 1]):\n super().__init__()\n self.num_classes = num_classes\n self.depths = depths\n self.embed_dims = embed_dims\n\n # patch_embed\n self.patch_embed1 = OverlapPatchEmbed(img_size=img_size, patch_size=7, stride=4, in_chans=in_chans,\n embed_dim=embed_dims[0])\n self.patch_embed2 = OverlapPatchEmbed(img_size=img_size // 4, patch_size=3, stride=2, in_chans=embed_dims[0],\n embed_dim=embed_dims[1])\n self.patch_embed3 = OverlapPatchEmbed(img_size=img_size // 8, patch_size=3, stride=2, in_chans=embed_dims[1],\n embed_dim=embed_dims[2])\n self.patch_embed4 = OverlapPatchEmbed(img_size=img_size // 16, patch_size=3, stride=2, in_chans=embed_dims[2],\n embed_dim=embed_dims[3])\n\n # transformer encoder\n dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] # stochastic depth decay rule\n cur = 0\n self.block1 = nn.ModuleList([Block3(\n dim=embed_dims[0], num_heads=num_heads[0], mlp_ratio=mlp_ratios[0], qkv_bias=qkv_bias, qk_scale=qk_scale,\n drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[cur + i], norm_layer=norm_layer,\n sr_ratio=sr_ratios[0])\n for i in range(depths[0])])\n self.norm1 = norm_layer(embed_dims[0])\n\n cur += depths[0]\n self.block2 = nn.ModuleList([Block3(\n dim=embed_dims[1], num_heads=num_heads[1], mlp_ratio=mlp_ratios[1], qkv_bias=qkv_bias, qk_scale=qk_scale,\n drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[cur + i], norm_layer=norm_layer,\n sr_ratio=sr_ratios[1])\n for i in range(depths[1])])\n self.norm2 = norm_layer(embed_dims[1])\n\n cur += depths[1]\n self.block3 = nn.ModuleList([Block3(\n dim=embed_dims[2], num_heads=num_heads[2], mlp_ratio=mlp_ratios[2], qkv_bias=qkv_bias, qk_scale=qk_scale,\n drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[cur + i], norm_layer=norm_layer,\n sr_ratio=sr_ratios[2])\n for i in range(depths[2])])\n self.norm3 = norm_layer(embed_dims[2])\n\n cur += depths[2]\n self.block4 = nn.ModuleList([Block3(\n dim=embed_dims[3], num_heads=num_heads[3], mlp_ratio=mlp_ratios[3], qkv_bias=qkv_bias, qk_scale=qk_scale,\n drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[cur + i], norm_layer=norm_layer,\n sr_ratio=sr_ratios[3])\n for i in range(depths[3])])\n self.norm4 = norm_layer(embed_dims[3])\n\n # classification head\n # self.head = nn.Linear(embed_dims[3], num_classes) if num_classes > 0 else nn.Identity()\n\n self.apply(self._init_weights)\n\n def _init_weights(self, m):\n if isinstance(m, nn.Linear):\n trunc_normal_(m.weight, std=.02)\n if isinstance(m, nn.Linear) and m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.LayerNorm):\n nn.init.constant_(m.bias, 0)\n nn.init.constant_(m.weight, 1.0)\n elif isinstance(m, nn.Conv2d):\n fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n fan_out //= m.groups\n m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))\n if m.bias is not None:\n m.bias.data.zero_()\n\n def init_weights(self, pretrained=None):\n pass\n # if isinstance(pretrained, str):\n # \tlogger = get_root_logger()\n # \tload_checkpoint(self, pretrained, map_location='cpu', strict=False, logger=logger)\n\n def reset_drop_path(self, drop_path_rate):\n dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(self.depths))]\n cur = 0\n for i in range(self.depths[0]):\n self.block1[i].drop_path.drop_prob = dpr[cur + i]\n\n cur += self.depths[0]\n for i in range(self.depths[1]):\n self.block2[i].drop_path.drop_prob = dpr[cur + i]\n\n cur += self.depths[1]\n for i in range(self.depths[2]):\n self.block3[i].drop_path.drop_prob = dpr[cur + i]\n\n cur += self.depths[2]\n for i in range(self.depths[3]):\n self.block4[i].drop_path.drop_prob = dpr[cur + i]\n\n def freeze_patch_emb(self):\n self.patch_embed1.requires_grad = False\n\n @torch.jit.ignore\n def no_weight_decay(self):\n return {'pos_embed1', 'pos_embed2', 'pos_embed3', 'pos_embed4', 'cls_token'} # has pos_embed may be better\n\n def get_classifier(self):\n return self.head\n\n def reset_classifier(self, num_classes, global_pool=''):\n self.num_classes = num_classes\n self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity()\n\n def forward_features(self, x):\n B = x.shape[0]\n outs = []\n\n # stage 1\n x, H, W = self.patch_embed1(x)\n for i, blk in enumerate(self.block1):\n x = blk(x, H, W)\n x = self.norm1(x)\n x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous()\n outs.append(x)\n\n # stage 2\n x, H, W = self.patch_embed2(x)\n for i, blk in enumerate(self.block2):\n x = blk(x, H, W)\n x = self.norm2(x)\n x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous()\n outs.append(x)\n\n # stage 3\n x, H, W = self.patch_embed3(x)\n for i, blk in enumerate(self.block3):\n x = blk(x, H, W)\n x = self.norm3(x)\n x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous()\n outs.append(x)\n\n # stage 4\n x, H, W = self.patch_embed4(x)\n for i, blk in enumerate(self.block4):\n x = blk(x, H, W)\n x = self.norm4(x)\n x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous()\n outs.append(x)\n\n return outs\n\n def forward(self, x):\n x = self.forward_features(x)\n # x = self.head(x)\n\n return x\n\n\n\n\nclass mit_b0(MixVisionTransformer):\n def __init__(self, **kwargs):\n super(mit_b0, self).__init__(\n patch_size=4, embed_dims=[32, 64, 160, 256], num_heads=[1, 2, 5, 8], mlp_ratios=[4, 4, 4, 4],\n qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[2, 2, 2, 2], sr_ratios=[8, 4, 2, 1],\n drop_rate=0.0, drop_path_rate=0.1)\n\n\nclass mit_b1(MixVisionTransformer):\n def __init__(self, **kwargs):\n super(mit_b1, self).__init__(\n patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[4, 4, 4, 4],\n qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[2, 2, 2, 2], sr_ratios=[8, 4, 2, 1],\n drop_rate=0.0, drop_path_rate=0.1)\n\n\nclass mit_b2(MixVisionTransformer):\n def __init__(self, **kwargs):\n super(mit_b2, self).__init__(\n patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[4, 4, 4, 4],\n qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 4, 6, 3], sr_ratios=[8, 4, 2, 1],\n drop_rate=0.0, drop_path_rate=0.1)\n\n\nclass mit_b3(MixVisionTransformer):\n def __init__(self, **kwargs):\n super(mit_b3, self).__init__(\n patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[4, 4, 4, 4],\n qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 4, 18, 3], sr_ratios=[8, 4, 2, 1],\n drop_rate=0.0, drop_path_rate=0.1)\n\n\nclass mit_b4(MixVisionTransformer):\n def __init__(self, **kwargs):\n super(mit_b4, self).__init__(\n patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[4, 4, 4, 4],\n qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 8, 27, 3], sr_ratios=[8, 4, 2, 1],\n drop_rate=0.0, drop_path_rate=0.1)\n\n\nclass mit_b5(MixVisionTransformer):\n def __init__(self, **kwargs):\n super(mit_b5, self).__init__(\n patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[4, 4, 4, 4],\n qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 6, 40, 3], sr_ratios=[8, 4, 2, 1],\n drop_rate=0.0, drop_path_rate=0.1)\n\n\npretrain_dir ='.'\nif 1:\n cfg3 = dict(\n mit_b0 = dict(\n checkpoint = pretrain_dir + '/mit_b0.pth',\n builder = mit_b0,\n ),\n mit_b1 = dict(\n checkpoint = pretrain_dir + '/mit_b1.pth',\n builder = mit_b1,\n ),\n mit_b2 = dict(\n checkpoint = pretrain_dir + '/mit_b2.pth',\n builder = mit_b2,\n ),\n mit_b3 = dict(\n checkpoint = pretrain_dir + '/mit_b3.pth',\n builder = mit_b3,\n ),\n\n )\n\nif 0:\n m = mit_b3()\n print(m)\n x=torch.rand(1,3,224,224)\n y=m(x)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RGB3(nn.Module):\n IMAGE_RGB_MEAN = [0.485, 0.456, 0.406] # [0.5, 0.5, 0.5]\n IMAGE_RGB_STD = [0.229, 0.224, 0.225] # [0.5, 0.5, 0.5]\n\n def __init__(self, ):\n super(RGB3, self).__init__()\n self.register_buffer('mean', torch.zeros(1, 3, 1, 1))\n self.register_buffer('std', torch.ones(1, 3, 1, 1))\n self.mean.data = torch.FloatTensor(self.IMAGE_RGB_MEAN).view(self.mean.shape)\n self.std.data = torch.FloatTensor(self.IMAGE_RGB_STD).view(self.std.shape)\n\n def forward(self, x):\n x = (x - self.mean) / self.std\n return x\n\nclass Net3(nn.Module):\n\n def load_pretrain(self, ):\n checkpoint = cfg3[self.arch]['checkpoint']\n print('load %s' % checkpoint)\n checkpoint = torch.load(checkpoint, map_location=lambda storage, loc: storage) #True\n #checkpoint = torch.load('./pytorch_model.bin', map_location=lambda storage, loc: storage) #True\n print(self.encoder.load_state_dict(checkpoint,strict=False)) #True\n\n def __init__(self,\n encoder=None,\n decoder=None,\n encoder_cfg=dict(),\n decoder_cfg=dict(),\n ):\n \n super(Net3, self).__init__()\n encoder = cfg3['mit_b3']['builder']()\n decoder_dim = [256, 128, 64, 32, 16]\n\n # ----\n self.output_type = ['inference', 'loss']\n self.rgb = RGB3()\n\n #https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/resnet.py\n conv_dim = 32\n self.conv = nn.Sequential(\n nn.Conv2d(3, 32, kernel_size=3, stride=2, padding=1, bias=False),\n nn.BatchNorm2d(32),\n nn.ReLU(inplace=True),\n nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1, bias=False),\n nn.BatchNorm2d(32),\n nn.ReLU(inplace=True),\n nn.Conv2d(32, conv_dim, kernel_size=3, stride=1, padding=1, bias=False)\n )\n\n\n #self.encoder = encoder(\n #**encoder_cfg\n #)\n self.arch = 'mit_b3'\n self.encoder = cfg3[self.arch]['builder']() \n encoder_dim = self.encoder.embed_dims\n # [64, 128, 320, 512]\n\n self.decoder = UnetDecoder(\n encoder_channels=[0, conv_dim] + encoder_dim,\n decoder_channels=decoder_dim,\n n_blocks=5,\n use_batchnorm=True,\n center=False,\n attention_type=None,\n )\n\n self.logit = nn.Sequential(\n nn.Conv2d(decoder_dim[-1], 1, kernel_size=1),\n )\n self.aux = nn.ModuleList([\n nn.Conv2d(encoder_dim[i], 1, kernel_size=1, padding=0) for i in range(len(encoder_dim))\n ])\n\n def forward(self, batch):\n\n x = batch['image']\n x = self.rgb(x)\n\n B, C, H, W = x.shape\n encoder = self.encoder(x)\n ##print('encoder', [f.shape for f in encoder])\n\n conv = self.conv(x)\n ##print('conv', conv.shape)\n\n # ---------------------------------\n if 1:\n feature = encoder[::-1] # reverse channels to start from head of encoder\n head = feature[0]\n skip = feature[1:] + [conv, None]\n d = self.decoder.center(head)\n\n decoder = []\n for i, decoder_block in enumerate(self.decoder.blocks):\n # print(i, d.shape, skip[i].shape if skip[i] is not None else 'none')\n # print(decoder_block.conv1[0])\n # print('')\n s = skip[i]\n d = decoder_block(d, s)\n decoder.append(d)\n last = d\n print('decoder',[f.shape for f in decoder])\n # ---------------------------------------------------------\n\n\n #---\n logit = self.logit(last)\n ##print('logit',logit.shape)\n\n output = {}\n if 'loss' in self.output_type:\n output['bce_loss'] = F.binary_cross_entropy_with_logits(logit, batch['mask'])\n #output['label_loss'] = criterion_binary_cross_entropy(logit, batch['mask'])\n\n for i in range(len(self.aux)):\n output['aux%d_loss' % i] = criterion_aux_loss(self.aux[i](encoder[i]), batch['mask'])\n\n if 'inference' in self.output_type:\n #probability_from_logit = torch.softmax(logit,1)\n probability_from_logit = torch.sigmoid(logit)\n output['probability_from_logit'] = probability_from_logit\n output['probability'] = probability_from_logit\n\n return output\n\n\ndef criterion_dice_loss(logit, mask):\n smooth = 1.\n\n batch_size, C, H, W = logit.shape\n p = torch.sigmoid(logit).reshape(batch_size, -1)\n t = mask.reshape(batch_size, -1)\n\n intersection = (p * t).sum(-1)\n union = p.sum(-1) + t.sum(-1)\n dice = 1 - (2 * intersection + smooth) / (union + smooth)\n dice = dice.mean()\n return dice\n\n\n# http://jck.bio/pytorch_onehot/\ndef criterion_multi_binary_cross_entropy(logit, mask, organ):\n logit = F.interpolate(logit, size=None, scale_factor=4, mode='bilinear', align_corners=False)\n batch_size, C, H, W = logit.shape\n\n label = mask.long() * organ.reshape(batch_size, 1, 1, 1)\n onehot = torch.zeros((batch_size, num_organ + 1, H, W)).to(mask)\n onehot = onehot.scatter(1, label, 1)\n # onehot[:,0] = 1-onehot[:,0]\n\n loss = F.binary_cross_entropy_with_logits(logit, onehot)\n return loss\n\ndef criterion_binary_cross_entropy(logit, mask):\n logit = F.interpolate(logit, size=None, scale_factor=4, mode='bilinear', align_corners=False)\n loss = F.binary_cross_entropy_with_logits(logit, mask)\n return loss\n\ndef criterion_cross_entropy(logit, mask, organ):\n logit = F.interpolate(logit, size=None, scale_factor=4, mode='bilinear', align_corners=False)\n batch_size, C, H, W = logit.shape\n\n label = mask.long() * organ.reshape(batch_size, 1, 1, 1)\n\n #https://github.com/pytorch/pytorch/blob/master/torch/nn/modules/loss.py\n log_softmax = F.log_softmax(logit,1)\n loss = F.nll_loss(log_softmax,label.squeeze(1))\n return loss\n\n\n\ndef criterion_aux_loss(logit, mask):\n mask = F.interpolate(mask, size=logit.shape[-2:], mode='nearest')\n loss = F.binary_cross_entropy_with_logits(logit, mask)\n return loss\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nclass Conv2dReLU1(nn.Sequential):\n def __init__(\n self,\n in_channels,\n out_channels,\n kernel_size,\n padding=0,\n stride=1,\n use_batchnorm=True,\n ):\n\n if use_batchnorm == \"inplace\" and InPlaceABN is None:\n raise RuntimeError(\n \"In order to use `use_batchnorm='inplace'` inplace_abn package must be installed. \"\n + \"To install see: https://github.com/mapillary/inplace_abn\"\n )\n\n conv = nn.Conv2d(\n in_channels,\n out_channels,\n kernel_size,\n stride=stride,\n padding=padding,\n bias=not (use_batchnorm),\n )\n relu = nn.ReLU(inplace=True)\n\n if use_batchnorm == \"inplace\":\n bn = InPlaceABN(out_channels, activation=\"leaky_relu\", activation_param=0.0)\n relu = nn.Identity()\n\n elif use_batchnorm and use_batchnorm != \"inplace\":\n bn = nn.BatchNorm2d(out_channels)\n\n else:\n bn = nn.Identity()\n\n super(Conv2dReLU1, self).__init__(conv, bn, relu)\n\n\nclass SCSEModule(nn.Module):\n def __init__(self, in_channels, reduction=16):\n super().__init__()\n self.cSE = nn.Sequential(\n nn.AdaptiveAvgPool2d(1),\n nn.Conv2d(in_channels, in_channels // reduction, 1),\n nn.ReLU(inplace=True),\n nn.Conv2d(in_channels // reduction, in_channels, 1),\n nn.Sigmoid(),\n )\n self.sSE = nn.Sequential(nn.Conv2d(in_channels, 1, 1), nn.Sigmoid())\n\n def forward(self, x):\n return x * self.cSE(x) + x * self.sSE(x)\n\n\nclass ArgMax(nn.Module):\n def __init__(self, dim=None):\n super().__init__()\n self.dim = dim\n\n def forward(self, x):\n return torch.argmax(x, dim=self.dim)\n\n\nclass Clamp(nn.Module):\n def __init__(self, min=0, max=1):\n super().__init__()\n self.min, self.max = min, max\n\n def forward(self, x):\n return torch.clamp(x, self.min, self.max)\n\n\nclass Activation(nn.Module):\n def __init__(self, name, **params):\n\n super().__init__()\n\n if name is None or name == \"identity\":\n self.activation = nn.Identity(**params)\n elif name == \"sigmoid\":\n self.activation = nn.Sigmoid()\n elif name == \"softmax2d\":\n self.activation = nn.Softmax(dim=1, **params)\n elif name == \"softmax\":\n self.activation = nn.Softmax(**params)\n elif name == \"logsoftmax\":\n self.activation = nn.LogSoftmax(**params)\n elif name == \"tanh\":\n self.activation = nn.Tanh()\n elif name == \"argmax\":\n self.activation = ArgMax(**params)\n elif name == \"argmax2d\":\n self.activation = ArgMax(dim=1, **params)\n elif name == \"clamp\":\n self.activation = Clamp(**params)\n elif callable(name):\n self.activation = name(**params)\n else:\n raise ValueError(\n f\"Activation should be callable/sigmoid/softmax/logsoftmax/tanh/\"\n f\"argmax/argmax2d/clamp/None; got {name}\"\n )\n\n def forward(self, x):\n return self.activation(x)\n\n\nclass Attention33(nn.Module):\n def __init__(self, name, **params):\n super().__init__()\n\n if name is None:\n self.attention = nn.Identity(**params)\n elif name == \"scse\":\n self.attention = SCSEModule(**params)\n else:\n raise ValueError(\"Attention {} is not implemented\".format(name))\n\n def forward(self, x):\n return self.attention(x)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass DecoderBlock(nn.Module):\n def __init__(\n self,\n in_channels,\n skip_channels,\n out_channels,\n use_batchnorm=True,\n attention_type=None,\n ):\n super().__init__()\n self.conv1 = Conv2dReLU1(\n in_channels + skip_channels,\n out_channels,\n kernel_size=3,\n padding=1,\n use_batchnorm=use_batchnorm,\n )\n self.attention1 = Attention33(attention_type, in_channels=in_channels + skip_channels)\n self.conv2 = Conv2dReLU1(\n out_channels,\n out_channels,\n kernel_size=3,\n padding=1,\n use_batchnorm=use_batchnorm,\n )\n self.attention2 = Attention33(attention_type, in_channels=out_channels)\n\n def forward(self, x, skip=None):\n x = F.interpolate(x, scale_factor=2, mode=\"nearest\")\n if skip is not None:\n x = torch.cat([x, skip], dim=1)\n x = self.attention1(x)\n x = self.conv1(x)\n x = self.conv2(x)\n x = self.attention2(x)\n return x\n\n\nclass CenterBlock(nn.Sequential):\n def __init__(self, in_channels, out_channels, use_batchnorm=True):\n conv1 = Conv2dReLU1(\n in_channels,\n out_channels,\n kernel_size=3,\n padding=1,\n use_batchnorm=use_batchnorm,\n )\n conv2 = Conv2dReLU1(\n out_channels,\n out_channels,\n kernel_size=3,\n padding=1,\n use_batchnorm=use_batchnorm,\n )\n super().__init__(conv1, conv2)\n\n\nclass UnetDecoder(nn.Module):\n def __init__(\n self,\n encoder_channels,\n decoder_channels,\n n_blocks=5,\n use_batchnorm=True,\n attention_type=None,\n center=False,\n ):\n super().__init__()\n\n if n_blocks != len(decoder_channels):\n raise ValueError(\n \"Model depth is {}, but you provide `decoder_channels` for {} blocks.\".format(\n n_blocks, len(decoder_channels)\n )\n )\n\n # remove first skip with same spatial resolution\n encoder_channels = encoder_channels[1:]\n # reverse channels to start from head of encoder\n encoder_channels = encoder_channels[::-1]\n\n # computing blocks input and output channels\n head_channels = encoder_channels[0]\n in_channels = [head_channels] + list(decoder_channels[:-1])\n skip_channels = list(encoder_channels[1:]) + [0]\n out_channels = decoder_channels\n\n if center:\n self.center = CenterBlock(head_channels, head_channels, use_batchnorm=use_batchnorm)\n else:\n self.center = nn.Identity()\n\n # combine decoder keyword arguments\n kwargs = dict(use_batchnorm=use_batchnorm, attention_type=attention_type)\n blocks = [\n DecoderBlock(in_ch, skip_ch, out_ch, **kwargs)\n for in_ch, skip_ch, out_ch in zip(in_channels, skip_channels, out_channels)\n ]\n self.blocks = nn.ModuleList(blocks)\n\n def forward(self, *features):\n\n features = features[1:] # remove first skip with same spatial resolution\n features = features[::-1] # reverse channels to start from head of encoder\n\n head = features[0]\n skips = features[1:]\n\n x = self.center(head)\n for i, decoder_block in enumerate(self.blocks):\n skip = skips[i] if i < len(skips) else None\n x = decoder_block(x, skip)\n\n return x","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\n\nclass MixUpSample1(nn.Module):\n def __init__( self, scale_factor=2):\n super().__init__()\n assert(scale_factor!=1)\n\n self.mixing = nn.Parameter(torch.tensor(0.5))\n self.scale_factor = scale_factor\n\n def forward(self, x):\n x = self.mixing *F.interpolate(x, scale_factor=self.scale_factor, mode='bilinear', align_corners=False) \\\n + (1-self.mixing )*F.interpolate(x, scale_factor=self.scale_factor, mode='nearest')\n return x\n\n#https://github.com/lhoyer/DAFormer/blob/master/mmseg/models/decode_heads/daformer_head.py\ndef Conv2dBnReLU(in_channel, out_channel, kernel_size=3, padding=1,stride=1, dilation=1):\n return nn.Sequential(\n nn.Conv2d(in_channel, out_channel, kernel_size=kernel_size, padding=padding, stride=stride, dilation=dilation, bias=False),\n nn.BatchNorm2d(out_channel),\n nn.ReLU(inplace=True),\n )\n\nclass ASPP(nn.Module):\n\n def __init__(self,\n in_channel,\n channel,\n dilation,\n ):\n super(ASPP, self).__init__()\n\n self.conv = nn.ModuleList()\n for d in dilation:\n self.conv.append(\n Conv2dBnReLU(\n in_channel,\n channel,\n kernel_size=1 if d == 1 else 3,\n dilation=d,\n padding=0 if d == 1 else d,\n )\n )\n\n self.out = Conv2dBnReLU(\n len(dilation) * channel,\n channel,\n kernel_size=3,\n padding=1,\n )\n\n def forward(self, x):\n aspp = []\n for conv in self.conv:\n aspp.append(conv(x))\n aspp = torch.cat(aspp, dim=1)\n out = self.out(aspp)\n return out\n\n#DepthwiseSeparable\nclass DSConv2d(nn.Module):\n def __init__(self,\n in_channel,\n out_channel,\n kernel_size,\n stride = 1,\n padding = 0,\n dilation = 1\n ):\n super().__init__()\n\n self.depthwise = nn.Sequential(\n nn.Conv2d( in_channel, in_channel, kernel_size, stride=stride, padding=padding, dilation=dilation),\n nn.BatchNorm2d(in_channel),\n nn.ReLU(inplace=True)\n )\n\n self.pointwise = nn.Sequential(\n nn.Conv2d( in_channel, out_channel, kernel_size=1, stride=1, padding=0),\n nn.BatchNorm2d(out_channel),\n nn.ReLU(inplace=True)\n )\n\n def forward(self, x):\n x = self.depthwise(x)\n x = self.pointwise(x)\n return x\n\nclass DSASPP(nn.Module):\n\n def __init__(self,\n in_channel,\n channel,\n dilation,\n ):\n super(DSASPP, self).__init__()\n\n self.conv = nn.ModuleList()\n for d in dilation:\n if d == 1:\n self.conv.append(\n Conv2dBnReLU(\n in_channel,\n channel,\n kernel_size=1 if d == 1 else 3,\n dilation=d,\n padding=0 if d == 1 else d,\n )\n )\n else:\n self.conv.append(\n DSConv2d(\n in_channel,\n channel,\n kernel_size=3,\n dilation=d,\n padding=d,\n )\n )\n\n self.out = Conv2dBnReLU(\n len(dilation) * channel,\n channel,\n kernel_size=3,\n padding=1,\n )\n\n def forward(self, x):\n aspp = []\n for conv in self.conv:\n aspp.append(conv(x))\n aspp = torch.cat(aspp, dim=1)\n out = self.out(aspp)\n return out\n\n\n##############################################################################################33\n\nclass DaformerDecoder(nn.Module):\n def __init__(\n self,\n encoder_dim = [32, 64, 160, 256],\n decoder_dim = 256,\n dilation = [1, 6, 12, 18],\n use_bn_mlp = True,\n fuse = 'conv1x1',\n ):\n super().__init__()\n self.mlp = nn.ModuleList([\n nn.Sequential(\n # Conv2dBnReLU(dim, decoder_dim, 1, padding=0), #follow mmseg to use conv-bn-relu\n *(\n ( nn.Conv2d(dim, decoder_dim, 1, padding= 0, bias=False),\n nn.BatchNorm2d(decoder_dim),\n nn.ReLU(inplace=True),\n )if use_bn_mlp else\n ( nn.Conv2d(dim, decoder_dim, 1, padding= 0, bias=True),)\n ),\n\n MixUpSample1(2**i) if i!=0 else nn.Identity(),\n ) for i, dim in enumerate(encoder_dim)])\n\n if fuse=='conv1x1':\n self.fuse = nn.Sequential(\n nn.Conv2d(len(encoder_dim) * decoder_dim, decoder_dim, 1, padding=0, bias=False),\n nn.BatchNorm2d(decoder_dim),\n nn.ReLU(inplace=True),\n )\n\n if fuse=='conv3x3':\n self.fuse = nn.Sequential(\n nn.Conv2d(len(encoder_dim) * decoder_dim, decoder_dim, 3, padding=1, bias=False),\n nn.BatchNorm2d(decoder_dim),\n nn.ReLU(inplace=True),\n )\n\n if fuse=='aspp':\n self.fuse = ASPP(\n decoder_dim*len(encoder_dim),\n decoder_dim,\n dilation,\n )\n\n if fuse=='ds-aspp':\n self.fuse = DSASPP(\n decoder_dim*len(encoder_dim),\n decoder_dim,\n dilation,\n )\n\n\n def forward(self, feature):\n\n out = []\n for i,f in enumerate(feature):\n f = self.mlp[i](f)\n out.append(f)\n #print(f.shape)\n x = self.fuse(torch.cat(out, dim = 1))\n return x, out\n\n\nclass daformer_conv3x3 (DaformerDecoder):\n def __init__(self, **kwargs):\n super(daformer_conv3x3, self).__init__(\n fuse = 'conv3x3',\n **kwargs\n )\nclass daformer_conv1x1 (DaformerDecoder):\n def __init__(self, **kwargs):\n super(daformer_conv1x1, self).__init__(\n fuse = 'conv1x1',\n **kwargs\n )\n\nclass daformer_aspp (DaformerDecoder):\n def __init__(self, **kwargs):\n super(daformer_aspp, self).__init__(\n fuse = 'aspp',\n **kwargs\n )\n\nif 0:\n m = daformer_conv1x1(\n encoder_dim = [32, 64, 160, 256],\n decoder_dim = 256,\n dilation = None,\n\n )\n print(m)\n feature = [\n torch.rand(1,c,64//(2**i),64//(2**i)) for i,c in enumerate([32, 64, 160, 256])\n ]\n y, out = m(feature)\n print(y.shape)\n \nclass Net_daformer(nn.Module):\n def load_pretrain( self,):\n checkpoint = cfg[self.arch]['checkpoint']\n print('load %s'%checkpoint)\n checkpoint = torch.load(checkpoint, map_location=lambda storage, loc: storage) #True\n #checkpoint = torch.load('./pytorch_model.bin', map_location=lambda storage, loc: storage) #True\n print(self.encoder.load_state_dict(checkpoint,strict=False)) #True\n\n\n def __init__( self,):\n super(Net_daformer, self).__init__()\n self.output_type = ['inference', 'loss']\n self.rgb = RGB()\n self.dropout = nn.Dropout(0.1)\n\n self.arch = 'pvt_v2_b4'\n self.encoder = cfg[self.arch]['builder']()\n encoder_dim = self.encoder.embed_dims\n #[64, 128, 320, 512]\n\n self.decoder = daformer_conv1x1(\n encoder_dim=encoder_dim,\n decoder_dim=320,\n )\n self.logit = nn.Sequential(\n nn.Conv2d(320, 1, kernel_size=1, padding=0),\n )\n self.aux = nn.ModuleList([\n nn.Conv2d(encoder_dim[i], 1, kernel_size=1, padding=0) for i in range(4)\n ])\n\n\n def forward(self, batch):\n\n x = batch['image']\n x = self.rgb(x)\n\n B,C,H,W = x.shape\n encoder = self.encoder(x)\n #print([f.shape for f in encoder])\n\n last, decoder = self.decoder(encoder)\n last = self.dropout(last)\n logit = self.logit(last)\n logit = F.interpolate(logit, size=None, scale_factor=4, mode='bilinear', align_corners=False)\n #print(logit.shape)\n\n output = {}\n if 'loss' in self.output_type:\n output['bce_loss'] = F.binary_cross_entropy_with_logits(logit,batch['mask'])\n for i in range(4):\n output['aux%d_loss'%i] = criterion_aux_loss(self.aux[i](encoder[i]),batch['mask'])\n\n if 'inference' in self.output_type:\n output['probability'] = torch.sigmoid(logit)\n\n return output\n\ndef criterion_aux_loss(logit, mask):\n mask = F.interpolate(mask,size=logit.shape[-2:], mode='nearest')\n loss = F.binary_cross_entropy_with_logits(logit,mask)\n return loss\n","metadata":{"execution":{"iopub.status.busy":"2022-09-20T01:36:37.288286Z","iopub.execute_input":"2022-09-20T01:36:37.289022Z","iopub.status.idle":"2022-09-20T01:36:39.403043Z","shell.execute_reply.started":"2022-09-20T01:36:37.288927Z","shell.execute_reply":"2022-09-20T01:36:39.401074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimage_size = 768\n\norgan_threshold = {\n 'kidney': 0.40,\n 'prostate': 0.35,\n 'largeintestine': 0.50,\n 'spleen': 0.40,\n 'lung': 0.10,\n}\n\norgan_threshold = {\n 'Hubmap': {\n 'kidney' : 0.45, \n 'prostate' : 0.40, #0.40 best single\n 'largeintestine': 0.30, #0.30 BEST ENSEMBLE\n 'spleen' : 0.30, #0.30 BEST ENSEMBLE\n 'lung' : 0.06, #0.06 best ensemble(5 ensemvle), 0.07 best ensemble(4 ensemvle)\n },\n 'HPA': {\n 'kidney' : 0.50,\n 'prostate' : 0.50,\n 'largeintestine': 0.50,\n 'spleen' : 0.50,\n 'lung' : 0.10,\n },\n}\n\n#net1 = Net().cuda()\n#state_dict = swa\n\n# image_size 392\n#pretrained = torch.load(\"../input/swin-small/00033180.model.pth\", map_location=lambda storage, loc: storage)['state_dict'] #0.66\n#pretrained = torch.load(\"../input/swins-small3/00019950.model.pth\", map_location=lambda storage, loc: storage)['state_dict'] #0.65\n#pretrained = torch.load(\"../input/swin-small4/00039690.model.pth\", map_location=lambda storage, loc: storage)['state_dict'] # 0.69\n\n# image_size 672\n#pretrained = torch.load(\"../input/swin-small-672/00017640.model.pth\", map_location=lambda storage, loc: storage)['state_dict'] #0.69\n\n#pretrained2 = torch.load(\"../input/swin-small-672-2/00011760.model.pth\", map_location=lambda storage, loc: storage)['state_dict'] #0.71 0.5 thr\n#pretrained2_2 = torch.load(\"../input/swin-small-fold2-672/00017640.model.pth\", map_location=lambda storage, loc: storage)['state_dict'] #0.71 0.5 thr\n\n#pretrained1 = torch.load('../input/newweight/00022599.model.pth', map_location=lambda storage, loc: storage)['state_dict'] #Hubmap 0.53\n# best combination Hubmap 0.56\n###########################################################################################################################3\n#pretrained1 = torch.load('../input/pvt-v2-v4-480-fold3/00009240.model.pth', map_location=lambda storage, loc: storage)['state_dict'] #Hubmap 0.54\n#pretrained1_aug = torch.load('../input/pvt-v2-b4-aug-fold3-480/00006300.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \n##############################################################################################################################\n#Hubmap 0.57\n#pretrained1 = torch.load('../input/pvtv2-b4-768-fold3-4/00018900.model.pth', map_location=lambda storage, loc: storage)['state_dict'] #Hubmap 0.57\n#pretrained1_aug = torch.load('../input/pvt-v2-b4-aug-fold3-480/00006300.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \n##########################################################################################################################################\n#pretrained1 = torch.load('../input/pvtv2-b4-combined-768/00016380.model.pth', map_location=lambda storage, loc: storage)['state_dict'] #Hubmap 0.57\n#pretrained1_aug = torch.load('../input/pvtv2-b4-768-fold3-4/00018900.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \n\n####################################################################\n#pretrained2 = torch.load('../input/d/chris666/pvtv2-b4-768-combined/00017220.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n#0.80 v29 best\n#pretrained3 = torch.load('../input/d/chris666/pvtv2-b4-768-combined/00010920.model.pth', map_location=lambda storage, loc: storage)['state_dict']\npretrained1 = torch.load('../input/pvtv2-b4-768-fold3-4/00018900.model.pth', map_location=lambda storage, loc: storage)['state_dict'] #Hubmap 0.57\npretrained4 = torch.load('../input/pvtv2-768-spleen/00003780.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n#pretrained_spleen = torch.load('../input/d/chris666/pvtv2-daf-spleen/00004860.model.pth', map_location=lambda storage, loc: storage)['state_dict']\npretrained2 = torch.load('../input/d/chris666/pvtv2-b4-768-combined/00017220.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n#0.80 v29 best\npretrained3 = torch.load('../input/d/chris666/pvtv2-b4-768-combined/00010920.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n#v4 0.80 score\n#pretrained_spleen = torch.load('../input/pvtv2-768-spleen/00003780.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n\n\n#pretrained1_2 = torch.load('../input/newweight/00023436.model.pth', map_location=lambda storage, loc: storage)['state_dict'] #Hubmap 0.53\n#pretrained1_3 = torch.load('../input/newweight/00025389.model.pth', map_location=lambda storage, loc: storage)['state_dict'] #Hubmap 0.53\n\n#ensemble_weights1=[pretrained1]\n#ensemble_weights2=[pretrained2, pretrained2_2]\n\n#pretrained = torch.load('./result/fold-0/checkpoint/' + '00040110.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n\n\n#pretrained = torch.load(pretrain_dir + '/' + '00007665.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n#net1.load_state_dict(pretrained1, strict=False) # True\n#net2.load_state_dict(pretrained2, strict=False) \n\nnet1 = Net().cuda()\nnet1.load_state_dict(pretrained1, strict=False) # True\nnet1 = net1.eval()\nnet1.output_type = ['inference']\n\nnet2 = Net().cuda()\nnet2.load_state_dict(pretrained2, strict=False) # True\nnet2 = net2.eval()\nnet2.output_type = ['inference']\n\nnet3 = Net().cuda()\nnet3.load_state_dict(pretrained3, strict=False) # True\nnet3 = net3.eval()\nnet3.output_type = ['inference']\n\nnet4 = Net().cuda()\nnet4.load_state_dict(pretrained4, strict=False) # True\nnet4 = net4.eval()\nnet4.output_type = ['inference']\n\n#net5 = Net_daformer().cuda()\n#net5.load_state_dict(pretrained_spleen, strict=False) # True\n#net5 = net5.eval()\n#net5.output_type = ['inference']\n# 0.80 v29 best\nensemble_mit = [net1, net2, net3, net4]\n#ensemble_mit = [net1, net2, net3, net4, net5]\n#ensemble_mit = [net1, net4, net5]\n#ensemble_mit = [net1, net2, net4]\ndel net1, net2, net3, net4, pretrained1, pretrained2, pretrained3, pretrained4\n\n\n#net_spleen = Net().cuda()\n#net_spleen.load_state_dict(pretrained_spleen, strict=False) # True\n#net_spleen = net_spleen.eval()\n#net_spleen.output_type = ['inference']\n\n#ensemble_spleen = [net1, net_spleen]\n#del net1, net_spleen\n# ----\nresult = {\n 'id': [],\n 'probability': [],\n 'rle': [],\n}\n\n#start_timer = timer()\n\n#not_prostate = test_df[~((test_df['organ']=='spleen') & (test_df['data_source']=='Hubmap'))]\nspleen_df = test_df[((test_df['organ']=='spleen') & (test_df['data_source']=='Hubmap'))]\nprostate_df = test_df[((test_df['organ']=='prostate') & (test_df['data_source']=='Hubmap'))]\nlung_df = test_df[((test_df['organ']=='lung') & (test_df['data_source']=='Hubmap'))]\nNet_daformer\nother_df = test_df[~(((test_df['organ']=='spleen') & (test_df['data_source']=='Hubmap'))|((test_df['organ']=='prostate') & (test_df['data_source']=='Hubmap'))|((test_df['organ']=='lung') & (test_df['data_source']=='Hubmap')))]\n\n\n#largeintestine_df = test_df[((test_df['organ']=='largeintestine') & (test_df['data_source']=='Hubmap'))]\n#prostate_df = test_df[((test_df['organ']=='prostate') & (test_df['data_source']=='Hubmap'))]\n#other_df = test_df[~(((test_df['organ']=='prostate') & (test_df['data_source']=='Hubmap'))|((test_df['organ']=='largeintestine') & (test_df['data_source']=='Hubmap')))]\n\n\nfor person_id in spleen_df['id'].unique(): \n pid_src = spleen_df[spleen_df['id'] == person_id]['data_source'].values[0]\n organ_cat = spleen_df[spleen_df['id'] == person_id]['organ'].values[0]\n image = cv2.imread('../input/hubmap-organ-segmentation/test_images/%d.tiff' % person_id,\n cv2.IMREAD_COLOR)\n image = image.astype(np.float32) / 255\n H, W, _ = image.shape\n \n image_size = 768 #768\n s = spleen_df[spleen_df['id'] == person_id]['pixel_size'].values[0] / 0.4 * (image_size / 3000)\n h = int(np.ceil(int(H*s)/32)*32)\n w = int(np.ceil(int(W*s)/32)*32) \n image = cv2.resize(image,dsize=(w,h),interpolation=cv2.INTER_LINEAR) \n #image = cv2.resize(image,dsize=(image_size,image_size),interpolation=cv2.INTER_LINEAR) \n \n ######################################################################################### \n \n new_image = image_to_tensor(image)\n new_image = new_image.cuda()\n batch = {\n 'image':\n torch.stack([\n new_image,\n torch.flip(new_image, [1]),\n torch.flip(new_image, [2]),\n ]), # simple TTA\n }\n \n\n prob_list = []\n for nettt in ensemble_mit: \n\n probability = 0\n p = 0\n with torch.no_grad():\n with amp.autocast(enabled=is_amp):\n output = nettt(batch) # data_parallel(net, batch) # \n # probability += output['probability']\n probability += F.interpolate(\n output['probability'], size=(H, W), mode='bilinear', align_corners=False)\n\n # undo TTA\n probability[1] = torch.flip(probability[1], [1])\n probability[2] = torch.flip(probability[2], [2])\n probability = probability.float().data.cpu().numpy().mean(0)[0]\n prob_list.append(probability)\n\n \n\n #p = probability > organ_threshold[pid_src][organ_cat]\n\n # weight are [0.5,0.25,0.25]\n #probability = []\n #probability = prob_list2[0]\n #probability = prob_list[0]*0.5 + prob_list[1]*0.5\n #probability = prob_list[0]*0.4 + prob_list[1]*0.3 + prob_list[2]*0.3\n #0.80 v29 best\n probability = prob_list[0]*0.25 + prob_list[1]*0.25 + prob_list[2]*0.25 + prob_list[3]*0.25\n #probability = prob_list[0]*0.22 + prob_list[1]*0.22 + prob_list[2]*0.22 + prob_list[3]*0.22 + prob_list[4]*0.12\n #probability = prob_list[0]*0.2 + prob_list[1]*0.2 + prob_list[2]*0.2 + prob_list[3]*0.2 + prob_list[4]*0.2\n\n #probability = prob_list[0]\n del prob_list\n \n if pid_src == 'Hubmap':\n p = probability > organ_threshold['Hubmap'][organ_cat]\n if pid_src == 'HPA':\n p = probability > organ_threshold['HPA'][organ_cat]\n \n #p = probability > organ_threshold[organ_cat]\n #widthhh=test_df[test_df['id'] == person_id]['img_width'].values[0]\n #heighttt=test_df[test_df['id'] == person_id]['img_height'].values[0]\n mask = cv2.resize(1*p, (H,W),interpolation=cv2.INTER_NEAREST)\n \n ##################################################################################################################################\n \n plt.imshow(mask)\n del p\n rle = rle_encode_less_memory(mask)\n \n del mask\n \n if pid_src != 'HPA':\n result['rle'].append(rle)\n\n result['id'].append(person_id)\n\n\n else: \n #for only Hubmap score test\n #result['rle'].append('') \n #result['id'].append(person_id) \n # all score test\n result['rle'].append(rle) \n result['id'].append(person_id) \n\ndel ensemble_mit\n \n#################################################################################################################################### \n\n#################################################################################\n#net1 = Net().cuda() \n#net1.load_state_dict(pretrained1, strict=False)\n#net1 = net1.eval()\n#net1.output_type = ['inference']\n\npretrained1 = torch.load('../input/pvtv2-b4-768-fold3-4/00018900.model.pth', map_location=lambda storage, loc: storage)['state_dict'] #Hubmap 0.57\n#pretrained_spleen = torch.load('../input/pvtv2-768-spleen/00003780.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n#pretrained2 = torch.load('../input/pvtv2-768-prostate-newset/00005250.model.pth', map_location=lambda storage, loc: storage)['state_dict'] #Hubmap 0.57\n#pretrained3 = torch.load('../input/pvtv2-768-prostate-newset/00007140.model.pth', map_location=lambda storage, loc: storage)['state_dict'] #Hubmap 0.57 \n \nnet1 = Net().cuda()\nnet1.load_state_dict(pretrained1, strict=False) # True\nnet1 = net1.eval()\nnet1.output_type = ['inference']\n\nensemble_spleen = [net1]\ndel net1, pretrained1\n#ensemble_spleen = [net1, net2, net3]\n#del net1, net2, net3, pretrained1, pretrained2, pretrained3\n\n\n \nfor person_id in prostate_df['id'].unique():\n#for person_id in other_df['id'].unique():\n pid_src = prostate_df[prostate_df['id'] == person_id]['data_source'].values[0]\n organ_cat = prostate_df[prostate_df['id'] == person_id]['organ'].values[0]\n \n #pid_src = other_df[other_df['id'] == person_id]['data_source'].values[0]\n #organ_cat = other_df[other_df['id'] == person_id]['organ'].values[0]\n image = cv2.imread('../input/hubmap-organ-segmentation/test_images/%d.tiff' % person_id,\n cv2.IMREAD_COLOR)\n image = image.astype(np.float32) / 255\n H, W, _ = image.shape \n #net1.load_state_dict(pretrained1, strict=False) # True\n\n #net1 = net1.eval()\n #net1.output_type = ['inference']\n \n \n #if organ_cat == 'prostate':\n #image_size = 672\n #s = test_df[test_df['id'] == person_id]['pixel_size'].values[0] / 0.4 * (image_size / 3000)\n #image = cv2.resize(image, dsize=None, fx=s, fy=s, interpolation=cv2.INTER_LINEAR)\n #else:\n \n image_size = 768 #480\n s = prostate_df[prostate_df['id'] == person_id]['pixel_size'].values[0] / 0.4 * (image_size / 3000)\n #s = other_df[other_df['id'] == person_id]['pixel_size'].values[0] / 0.4 * (image_size / 3000)\n h = int(np.ceil(int(H*s)/32)*32)\n w = int(np.ceil(int(W*s)/32)*32) \n image = cv2.resize(image,dsize=(w,h),interpolation=cv2.INTER_LINEAR) \n\n new_image = image_to_tensor(image)\n new_image = new_image.cuda()\n batch = {\n 'image':\n torch.stack([\n new_image,\n torch.flip(new_image, [1]),\n torch.flip(new_image, [2]),\n ]), # simple TTA\n }\n\n prob_list = []\n for nettt in ensemble_spleen:\n probability = 0\n p = 0\n with torch.no_grad():\n with amp.autocast(enabled=is_amp):\n output = nettt(batch) # data_parallel(net, batch) # \n # probability += output['probability']\n probability += F.interpolate(\n output['probability'], size=(H, W), mode='bilinear', align_corners=False)\n\n # undo TTA\n probability[1] = torch.flip(probability[1], [1])\n probability[2] = torch.flip(probability[2], [2])\n probability = probability.float().data.cpu().numpy().mean(0)[0]\n prob_list.append(probability)\n\n \n\n #p = probability > organ_threshold[pid_src][organ_cat]\n\n # weight are [0.5,0.25,0.25]\n #probability = []\n #0.80 v18\n probability = prob_list[0]\n #probability = prob_list[0]*0.45 + prob_list[1]*0.275 + prob_list[2]*0.275\n del prob_list\n if pid_src == 'Hubmap':\n p = probability > organ_threshold['Hubmap'][organ_cat]\n if pid_src == 'HPA':\n p = probability > organ_threshold['HPA'][organ_cat]\n \n #p = probability > organ_threshold[organ_cat]\n #widthhh=test_df[test_df['id'] == person_id]['img_width'].values[0]\n #heighttt=test_df[test_df['id'] == person_id]['img_height'].values[0]\n mask = cv2.resize(1*p, (H,W),interpolation=cv2.INTER_NEAREST)\n \n ##################################################################################################################################\n \n #plt.imshow(mask)\n del p\n \n #rle = rle_encode_less_memory(1*p)\n rle = rle_encode_less_memory(mask)\n \n del mask\n \n if pid_src != 'HPA':\n result['rle'].append(rle)\n #result['probability'].append(probability)\n result['id'].append(person_id)\n #print('\\r', t, end='', flush=True)\n\n else: \n #for only Hubmap score test\n #result['rle'].append('') \n #result['id'].append(person_id) \n # all score test\n result['rle'].append(rle) \n result['id'].append(person_id) \ndel ensemble_spleen\n\n#pretrained1 = torch.load('../input/pvtv2-b4-768-fold3-4/00018900.model.pth', map_location=lambda storage, loc: storage)['state_dict'] #Hubmap 0.57\n# 0.80 v3\n#pretrained_lung = torch.load('../input/pvtv2-768-lung/00002592.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n#pretrained_lung2 = torch.load('../input/pvtv2-768-lung2/00003024.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \n#pretrained_lung2 = torch.load('../input/pvtv2-768-lunggg/00005292.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \n# 0.80 v3\n#pretrained_lung3 = torch.load('../input/pvtv2-768-lung3/00003348.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n#pretrained_lung3 = torch.load('../input/pvtv768-lungg4/00003780.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n\n# 0.80 v18 best\npretrained1 = torch.load('../input/pvtv2-b4-768-fold3-4/00018900.model.pth', map_location=lambda storage, loc: storage)['state_dict'] #Hubmap 0.57\n# 0.80 v18 best\n#pretrained_lung = torch.load('../input/pvtv2-768-lung/00002592.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n# 0.80 v29 best\npretrained_lung = torch.load('../input/pvtv2-daf-lung/00003780.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n#pretrained_lung = torch.load('../input/pvtv2-768-2-lung/00003888.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n\n#pretrained_lung2 = torch.load('../input/pvtv2-768-lunggg/00005292.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \n#0.80 v45 best\n#pretrained_lung2 = torch.load('../input/pvtv2-daf-lung-fold2/00004590.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \npretrained_lung2 = torch.load('../input/pvtv2-daf-lung-fold2-new/00005292.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \n\n#pretrained_lung2 = torch.load('../input/d/chris666/pvtv2-lung-allstain/00004050.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \n\n\n#pretrained_lung3 = torch.load('../input/pvtv768-lungg4/00003780.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n#0.80 v47 best\npretrained_lung3 = torch.load('../input/d/chris666/pvtv2-lung-allstain/00004590.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \n\n# 0.80 v47 best\npretrained_lung4 = torch.load('../input/pvtv2-lunggg-2/00004752.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n#pretrained_lung4 = torch.load('../input/pvtv2-daf-fold1-lung/00005562.model.pth', map_location=lambda storage, loc: storage)['state_dict']\n\n#pretrained_lung4 = torch.load('../input/pvtv2-768-lunggg/00007128.model.pth', map_location=lambda storage, loc: storage)['state_dict'] \n\nnet1 = Net().cuda()\nnet1.load_state_dict(pretrained1, strict=False) # True\nnet1 = net1.eval()\nnet1.output_type = ['inference']\n\n#net2 = Net().cuda()\n#net2.load_state_dict(pretrained_lung, strict=False) # True\n#net2 = net2.eval()\n#net2.output_type = ['inference']\n# 0.80 v29 best\nnet2 = Net_daformer().cuda()\nnet2.load_state_dict(pretrained_lung, strict=False) # True\nnet2 = net2.eval()\nnet2.output_type = ['inference']\n\n# add\n#net3 = Net().cuda()\n#net3.load_state_dict(pretrained_lung2, strict=False) # True\n#net3 = net3.eval()\n#net3.output_type = ['inference']\nnet3 = Net_daformer().cuda()\nnet3.load_state_dict(pretrained_lung2, strict=False) # True\nnet3 = net3.eval()\nnet3.output_type = ['inference']\n\n\nnet4 = Net().cuda()\nnet4.load_state_dict(pretrained_lung3, strict=False) # True\nnet4 = net4.eval()\nnet4.output_type = ['inference']\n#net4 = Net_daformer().cuda()\n#net4.load_state_dict(pretrained_lung3, strict=False) # True\n#net4 = net4.eval()\n#net4.output_type = ['inference']\n\nnet5 = Net().cuda()\nnet5.load_state_dict(pretrained_lung4, strict=False) # True\nnet5 = net5.eval()\nnet5.output_type = ['inference']\n#net5 = Net_daformer().cuda()\n#net5.load_state_dict(pretrained_lung4, strict=False) # True\n#net5 = net4.eval()\n#net5.output_type = ['inference']\n\n#ensemble_lung = [net1, net2, net3, net4]\n#del net1, net2, net3, net4, pretrained1, pretrained_lung, pretrained_lung2, pretrained_lung3 \n# 0.80 v27 best\nensemble_lung = [net1, net2, net3, net4, net5]\n# 0.80 v29 best\n#ensemble_lung = [net1, net2, net5]\n# 0.80 v45 best\n#ensemble_lung = [net1, net2, net3, net5]\ndel net1, net2, net3, net4, net5, pretrained1, pretrained_lung, pretrained_lung2, pretrained_lung3, pretrained_lung4 \n\nfor person_id in lung_df['id'].unique():\n#for person_id in other_df['id'].unique():\n pid_src = lung_df[lung_df['id'] == person_id]['data_source'].values[0]\n organ_cat = lung_df[lung_df['id'] == person_id]['organ'].values[0]\n \n #pid_src = other_df[other_df['id'] == person_id]['data_source'].values[0]\n #organ_cat = other_df[other_df['id'] == person_id]['organ'].values[0]\n image = cv2.imread('../input/hubmap-organ-segmentation/test_images/%d.tiff' % person_id,\n cv2.IMREAD_COLOR)\n image = image.astype(np.float32) / 255\n H, W, _ = image.shape \n #net1.load_state_dict(pretrained1, strict=False) # True\n\n #net1 = net1.eval()\n #net1.output_type = ['inference']\n \n \n #if organ_cat == 'prostate':\n #image_size = 672\n #s = test_df[test_df['id'] == person_id]['pixel_size'].values[0] / 0.4 * (image_size / 3000)\n #image = cv2.resize(image, dsize=None, fx=s, fy=s, interpolation=cv2.INTER_LINEAR)\n #else:\n \n image_size = 768 #480\n s = lung_df[lung_df['id'] == person_id]['pixel_size'].values[0] / 0.4 * (image_size / 3000)\n #s = other_df[other_df['id'] == person_id]['pixel_size'].values[0] / 0.4 * (image_size / 3000)\n h = int(np.ceil(int(H*s)/32)*32)\n w = int(np.ceil(int(W*s)/32)*32) \n image = cv2.resize(image,dsize=(w,h),interpolation=cv2.INTER_LINEAR) \n\n new_image = image_to_tensor(image)\n new_image = new_image.cuda()\n batch = {\n 'image':\n torch.stack([\n new_image,\n torch.flip(new_image, [1]),\n torch.flip(new_image, [2]),\n ]), # simple TTA\n }\n\n prob_list = []\n for nettt in ensemble_lung:\n probability = 0\n p = 0\n with torch.no_grad():\n with amp.autocast(enabled=is_amp):\n output = nettt(batch) # data_parallel(net, batch) # \n # probability += output['probability']\n probability += F.interpolate(\n output['probability'], size=(H, W), mode='bilinear', align_corners=False)\n\n # undo TTA\n probability[1] = torch.flip(probability[1], [1])\n probability[2] = torch.flip(probability[2], [2])\n probability = probability.float().data.cpu().numpy().mean(0)[0]\n prob_list.append(probability)\n\n \n\n #p = probability > organ_threshold[pid_src][organ_cat]\n\n # weight are [0.5,0.25,0.25]\n #probability = prob_list[0]*0.6 + prob_list[1]*0.4\n ##### 0.8 v29 best\n #probability = prob_list[0]*0.4 + prob_list[1]*0.3 +prob_list[2]*0.3\n ######0.8 v45 best\n #probability = prob_list[0]*0.31 + prob_list[1]*0.23 +prob_list[2]*0.23 + prob_list[3]*0.23\n probability = prob_list[0]*0.28 + prob_list[1]*0.18 +prob_list[2]*0.18 + prob_list[3]*0.18 + prob_list[4]*0.18\n del prob_list\n if pid_src == 'Hubmap':\n p = probability > organ_threshold['Hubmap'][organ_cat]\n if pid_src == 'HPA':\n p = probability > organ_threshold['HPA'][organ_cat]\n \n #p = probability > organ_threshold[organ_cat]\n #widthhh=test_df[test_df['id'] == person_id]['img_width'].values[0]\n #heighttt=test_df[test_df['id'] == person_id]['img_height'].values[0]\n mask = cv2.resize(1*p, (H,W),interpolation=cv2.INTER_NEAREST)\n \n ##################################################################################################################################\n \n #plt.imshow(mask)\n del p\n \n #rle = rle_encode_less_memory(1*p)\n rle = rle_encode_less_memory(mask)\n \n del mask\n \n if pid_src != 'HPA':\n result['rle'].append(rle)\n #result['probability'].append(probability)\n result['id'].append(person_id)\n #print('\\r', t, end='', flush=True)\n\n else: \n #for only Hubmap score test\n #result['rle'].append('') \n #result['id'].append(person_id) \n # all score test\n result['rle'].append(rle) \n result['id'].append(person_id) \ndel ensemble_lung\n\npretrained2 = torch.load('../input/d/chris666/pvtv2-b4-768-combined/00017220.model.pth', map_location=lambda storage, loc: storage)['state_dict']\npretrained3 = torch.load('../input/d/chris666/pvtv2-b4-768-combined/00010920.model.pth', map_location=lambda storage, loc: storage)['state_dict']\npretrained1 = torch.load('../input/pvtv2-b4-768-fold3-4/00018900.model.pth', map_location=lambda storage, loc: storage)['state_dict'] #Hubmap 0.57\n\nnet1 = Net().cuda()\nnet1.load_state_dict(pretrained1, strict=False) # True\nnet1 = net1.eval()\nnet1.output_type = ['inference']\n\nnet2 = Net().cuda()\nnet2.load_state_dict(pretrained2, strict=False) # True\nnet2 = net2.eval()\nnet2.output_type = ['inference']\n\nnet3 = Net().cuda()\nnet3.load_state_dict(pretrained3, strict=False) # True\nnet3 = net3.eval()\nnet3.output_type = ['inference']\n\nensemble_weight = [net1, net2, net3]\ndel net1, net2, net3, pretrained1, pretrained2, pretrained3\n#ensemble_weight = [net1]\n#del net1, pretrained1\n\nfor person_id in other_df['id'].unique():\n#for person_id in other_df['id'].unique():\n pid_src = other_df[other_df['id'] == person_id]['data_source'].values[0]\n organ_cat = other_df[other_df['id'] == person_id]['organ'].values[0]\n \n #pid_src = other_df[other_df['id'] == person_id]['data_source'].values[0]\n #organ_cat = other_df[other_df['id'] == person_id]['organ'].values[0]\n image = cv2.imread('../input/hubmap-organ-segmentation/test_images/%d.tiff' % person_id,\n cv2.IMREAD_COLOR)\n image = image.astype(np.float32) / 255\n H, W, _ = image.shape \n #net1.load_state_dict(pretrained1, strict=False) # True\n\n #net1 = net1.eval()\n #net1.output_type = ['inference']\n \n \n #if organ_cat == 'prostate':\n #image_size = 672\n #s = test_df[test_df['id'] == person_id]['pixel_size'].values[0] / 0.4 * (image_size / 3000)\n #image = cv2.resize(image, dsize=None, fx=s, fy=s, interpolation=cv2.INTER_LINEAR)\n #else:\n \n image_size = 768 #480\n s = other_df[other_df['id'] == person_id]['pixel_size'].values[0] / 0.4 * (image_size / 3000)\n #s = other_df[other_df['id'] == person_id]['pixel_size'].values[0] / 0.4 * (image_size / 3000)\n h = int(np.ceil(int(H*s)/32)*32)\n w = int(np.ceil(int(W*s)/32)*32) \n image = cv2.resize(image,dsize=(w,h),interpolation=cv2.INTER_LINEAR) \n\n new_image = image_to_tensor(image)\n new_image = new_image.cuda()\n batch = {\n 'image':\n torch.stack([\n new_image,\n torch.flip(new_image, [1]),\n torch.flip(new_image, [2]),\n ]), # simple TTA\n }\n\n prob_list = []\n for nettt in ensemble_weight:\n probability = 0\n p = 0\n with torch.no_grad():\n with amp.autocast(enabled=is_amp):\n output = nettt(batch) # data_parallel(net, batch) # \n # probability += output['probability']\n probability += F.interpolate(\n output['probability'], size=(H, W), mode='bilinear', align_corners=False)\n\n # undo TTA\n probability[1] = torch.flip(probability[1], [1])\n probability[2] = torch.flip(probability[2], [2])\n probability = probability.float().data.cpu().numpy().mean(0)[0]\n prob_list.append(probability)\n\n \n\n #p = probability > organ_threshold[pid_src][organ_cat]\n\n # weight are [0.5,0.25,0.25]\n #probability = []\n #probability = prob_list[0]\n probability = prob_list[0]*0.65 + prob_list[1]*0.3 + prob_list[2]*0.05\n del prob_list\n if pid_src == 'Hubmap':\n p = probability > organ_threshold['Hubmap'][organ_cat]\n if pid_src == 'HPA':\n p = probability > organ_threshold['HPA'][organ_cat]\n \n #p = probability > organ_threshold[organ_cat]\n #widthhh=test_df[test_df['id'] == person_id]['img_width'].values[0]\n #heighttt=test_df[test_df['id'] == person_id]['img_height'].values[0]\n mask = cv2.resize(1*p, (H,W),interpolation=cv2.INTER_NEAREST)\n \n ##################################################################################################################################\n \n #plt.imshow(mask)\n del p\n \n #rle = rle_encode_less_memory(1*p)\n rle = rle_encode_less_memory(mask)\n \n del mask\n \n if pid_src != 'HPA':\n result['rle'].append(rle)\n #result['probability'].append(probability)\n result['id'].append(person_id)\n #print('\\r', t, end='', flush=True)\n\n else: \n #for only Hubmap score test\n #result['rle'].append('') \n #result['id'].append(person_id) \n # all score test\n result['rle'].append(rle) \n result['id'].append(person_id) \ndel ensemble_weight\n\n\n\nprint('')\n# ---\nsubmit_df = pd.DataFrame({'id': result['id'], 'rle': result['rle']})\ntmp_list = []\nfor i in test_df['id'].values:\n tmp_list.append(submit_df[submit_df['id']==i].values[0])\ndel submit_df\nsubmit_df = pd.DataFrame(tmp_list)\ndel tmp_list\nsubmit_df.columns = ['id','rle']\n#submit_df=new_df\n\nprint(submit_df)\nprint('submit_df ok!')\nprint('')\n\nsubmit_df.to_csv('submission.csv', index=False)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}