{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"DEBUG = False","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:26:13.937426Z","iopub.execute_input":"2022-02-18T14:26:13.937842Z","iopub.status.idle":"2022-02-18T14:26:13.97499Z","shell.execute_reply.started":"2022-02-18T14:26:13.937733Z","shell.execute_reply":"2022-02-18T14:26:13.974165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\nconfig = {\n    'split_seed_list':[0],\n    'FOLD_LIST':[0,1,2,3], \n    'model_path':'../input/hubmap-new-03-03/',\n    'model_name':'seresnext101_ctrans',\n    \n    'num_classes':1,\n    'resolution':1024, #(1024,1024),(512,512),\n    'input_resolution':320, #(320,320), #(256,256), #(512,512), #(384,384)\n    'deepsupervision':False, # always false for inference\n    'clfhead':False,\n    'clf_threshold':0.5,\n    'small_mask_threshold':0, #256*256*0.03, #512*512*0.03,\n    'mask_threshold':0.5,\n    'pad_size':256, #(64,64), #(256,256), #(128,128)\n    \n    'tta':3,\n    'test_batch_size':12,\n    \n    'FP16':False,\n    'num_workers':4,\n    'device':torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"),\n}\n\ndevice = config['device']","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:26:13.979607Z","iopub.execute_input":"2022-02-18T14:26:13.981802Z","iopub.status.idle":"2022-02-18T14:26:15.432612Z","shell.execute_reply.started":"2022-02-18T14:26:13.981763Z","shell.execute_reply":"2022-02-18T14:26:15.431759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Libraries and Data","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\npd.get_option(\"display.max_columns\")\npd.set_option('display.max_columns', 300)\npd.get_option(\"display.max_rows\")\npd.set_option('display.max_rows', 300)\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport sys\nimport os\nfrom os.path import join as opj\nimport gc\n\nimport cv2\nimport rasterio\nfrom rasterio.windows import Window\n\nINPUT_PATH = '../input/hubmap-kidney-segmentation'","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:26:15.434425Z","iopub.execute_input":"2022-02-18T14:26:15.434862Z","iopub.status.idle":"2022-02-18T14:26:16.079521Z","shell.execute_reply.started":"2022-02-18T14:26:15.434821Z","shell.execute_reply":"2022-02-18T14:26:16.078788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Python        : ' + sys.version.split('\\n')[0])\nprint('Numpy         : ' + np.__version__)\nprint('Pandas        : ' + pd.__version__)\nprint('Rasterio      : ' + rasterio.__version__)\nprint('OpenCV        : ' + cv2.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:26:16.082223Z","iopub.execute_input":"2022-02-18T14:26:16.082635Z","iopub.status.idle":"2022-02-18T14:26:16.0895Z","shell.execute_reply.started":"2022-02-18T14:26:16.082596Z","shell.execute_reply":"2022-02-18T14:26:16.088657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(opj(INPUT_PATH, 'train.csv'))\ninfo_df  = pd.read_csv(opj(INPUT_PATH,'HuBMAP-20-dataset_information.csv'))\nsub_df = pd.read_csv(opj(INPUT_PATH, 'sample_submission.csv'))\n\nprint('train_df.shape = ', train_df.shape)\nprint('info_df.shape  = ', info_df.shape)\nprint('sub_df.shape = ', sub_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:26:16.092491Z","iopub.execute_input":"2022-02-18T14:26:16.092765Z","iopub.status.idle":"2022-02-18T14:26:16.474691Z","shell.execute_reply.started":"2022-02-18T14:26:16.092736Z","shell.execute_reply":"2022-02-18T14:26:16.473871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sub_df['predicted'] = '1 1'\n#sub_df.to_csv('submission.csv', index=False)\n\nif len(sub_df) == 5:\n    if DEBUG:\n        sub_df = sub_df[:]\n    else:\n        sub_df = sub_df[:1]","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:26:16.476101Z","iopub.execute_input":"2022-02-18T14:26:16.476363Z","iopub.status.idle":"2022-02-18T14:26:16.480652Z","shell.execute_reply.started":"2022-02-18T14:26:16.476329Z","shell.execute_reply":"2022-02-18T14:26:16.479955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"import random\nimport torch\nimport numpy as np\nimport os\nimport time\n\ndef fix_seed(seed):\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef elapsed_time(start_time):\n    return time.time() - start_time\n\ndef count_parameters(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\nfix_seed(2020)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:26:16.482337Z","iopub.execute_input":"2022-02-18T14:26:16.482856Z","iopub.status.idle":"2022-02-18T14:26:16.493941Z","shell.execute_reply.started":"2022-02-18T14:26:16.482819Z","shell.execute_reply":"2022-02-18T14:26:16.493243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\n\ndef rle2mask(rle, shape):\n    '''\n    mask_rle: run-length as string formatted (start length)\n    shape: (height, width) of array to return \n    Returns numpy array <- 1(mask), 0(background)\n    '''\n    s = rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape, order='F')  # Needed to align to RLE direction\n\n\ndef mask2rle(img, shape, small_mask_threshold):\n    '''\n    Convert mask to rle.\n    img: numpy array <- 1(mask), 0(background)\n    Returns run length as string formated\n    \n    pixels = np.array([1,1,1,0,0,1,0,1,1]) #-> rle = '1 3 6 1 8 2'\n    pixels = np.concatenate([[0], pixels, [0]]) #[0,1,1,1,0,0,1,0,1,1,0]\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1 #[ 1  4  6  7  8 10] bit change points\n    print(runs[1::2]) #[4 7 10]\n    print(runs[::2]) #[1 6 8]\n    runs[1::2] -= runs[::2]\n    print(runs) #[1 3 6 1 8 2]\n    '''\n    if img.shape != shape:\n        h,w = shape\n        img = cv2.resize(img, dsize=(w,h), interpolation=cv2.INTER_LINEAR)\n    img = img.astype(np.int8) \n    pixels = img.T.flatten()\n    #pixels = np.concatenate([[0], pixels, [0]])\n    pixels = np.pad(pixels, ((1, 1), ))\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    if runs[1::2].sum() <= small_mask_threshold:\n        return ''\n    else:\n        return ' '.join(str(x) for x in runs)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:26:16.495435Z","iopub.execute_input":"2022-02-18T14:26:16.49583Z","iopub.status.idle":"2022-02-18T14:26:16.50884Z","shell.execute_reply.started":"2022-02-18T14:26:16.495791Z","shell.execute_reply":"2022-02-18T14:26:16.50789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"# -*- coding: utf-8 -*-\n# @Time    : 2021/6/19 2:44 下午\n# @Author  : Haonan Wang\n# @File    : Config.py\n# @Software: PyCharm\n#import os\n#import torch\n#import time\npackage_dir = \"../input/ml-collection/ml_collections-master/\"\nsys.path.insert(0, package_dir)\nimport ml_collections\n\n## PARAMETERS OF THE MODEL\n#save_model = True\n#tensorboard = True\n#os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n#use_cuda = torch.cuda.is_available()\n#seed = 666\n#os.environ['PYTHONHASHSEED'] = str(seed)\n\n#cosineLR = True # whether use cosineLR or not\nn_channels = 3\n#n_labels = 1\n#epochs = 2000\n#img_size = 224\n#print_frequency = 1\n#save_frequency = 5000\n#vis_frequency = 10\n#early_stopping_patience = 50\n\n#pretrain = False\n#task_name = 'MoNuSeg' # GlaS MoNuSeg\n## task_name = 'GlaS'\n#learning_rate = 1e-3\n#batch_size = 4\n\n\n## model_name = 'UCTransNet'\n#model_name = 'UCTransNet_pretrain'\n\n#train_dataset = './datasets/'+ task_name+ '/Train_Folder/'\n#val_dataset = './datasets/'+ task_name+ '/Val_Folder/'\n#test_dataset = './datasets/'+ task_name+ '/Test_Folder/'\n#session_name       = 'Test_session' + '_' + time.strftime('%m.%d_%Hh%M')\n#save_path          = task_name +'/'+ model_name +'/' + session_name + '/'\n#model_path         = save_path + 'models/'\n#tensorboard_folder = save_path + 'tensorboard_logs/'\n#logger_path        = save_path + session_name + \".log\"\n#visualize_path     = save_path + 'visualize_val/'\n\n\n##########################################################################\n# CTrans configs\n# #########################################################################\ndef get_CTranS_config():\n    config = ml_collections.ConfigDict()\n    config.transformer = ml_collections.ConfigDict()\n    config.KV_size = 3840  # KV_size = Q1 + Q2 + Q3 + Q4\n    config.transformer.num_heads  = 2\n    config.transformer.num_layers = 2\n    config.expand_ratio           = 4  # MLP channel dimension expand ratio\n    config.transformer.embeddings_dropout_rate = 0.1\n    config.transformer.attention_dropout_rate = 0.1\n    config.transformer.dropout_rate = 0\n    config.patch_sizes = [16,8,4,2]\n    config.base_channel = 64 # base channel of U-Net\n    config.n_classes = 1\n    return config\n\n\n\n\n## used in testing phase, copy the session name in training phase\n#test_session = \"Test_session_07.03_20h39\"\n","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:26:16.510614Z","iopub.execute_input":"2022-02-18T14:26:16.511302Z","iopub.status.idle":"2022-02-18T14:26:16.612182Z","shell.execute_reply.started":"2022-02-18T14:26:16.511266Z","shell.execute_reply":"2022-02-18T14:26:16.611478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# -*- coding: utf-8 -*-\n# @Author  : Haonan Wang\n# @File    : CTrans.py\n# @Software: PyCharm\n# coding=utf-8\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nimport copy\nimport logging\nimport math\nimport torch\nimport torch.nn as nn\nimport numpy as np\nfrom torch.nn import Dropout, Softmax, Conv2d, LayerNorm\nfrom torch.nn.modules.utils import _pair\n\n\nlogger = logging.getLogger(__name__)\n\nclass Channel_Embeddings(nn.Module):\n    \"\"\"Construct the embeddings from patch, position embeddings.\n    \"\"\"\n    def __init__(self,config, patchsize, img_size, in_channels):\n        super().__init__()\n        img_size = _pair(img_size)\n        patch_size = _pair(patchsize)\n        n_patches = (img_size[0] // patch_size[0]) * (img_size[1] // patch_size[1])\n\n        self.patch_embeddings = Conv2d(in_channels=in_channels,\n                                       out_channels=in_channels,\n                                       kernel_size=patch_size,\n                                       stride=patch_size)\n        self.position_embeddings = nn.Parameter(torch.zeros(1, n_patches, in_channels))\n        self.dropout = Dropout(config.transformer[\"embeddings_dropout_rate\"])\n\n    def forward(self, x):\n        if x is None:\n            return None\n        x = self.patch_embeddings(x)  # (B, hidden. n_patches^(1/2), n_patches^(1/2))\n        x = x.flatten(2)\n        x = x.transpose(-1, -2)  # (B, n_patches, hidden)\n        embeddings = x + self.position_embeddings\n        embeddings = self.dropout(embeddings)\n        return embeddings\n\nclass Reconstruct(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size, scale_factor):\n        super(Reconstruct, self).__init__()\n        if kernel_size == 3:\n            padding = 1\n        else:\n            padding = 0\n        self.conv = nn.Conv2d(in_channels, out_channels,kernel_size=kernel_size, padding=padding)\n        self.norm = nn.BatchNorm2d(out_channels)\n        self.activation = nn.ReLU(inplace=True)\n        self.scale_factor = scale_factor\n\n    def forward(self, x):\n        if x is None:\n            return None\n\n        B, n_patch, hidden = x.size()  # reshape from (B, n_patch, hidden) to (B, h, w, hidden)\n        h, w = int(np.sqrt(n_patch)), int(np.sqrt(n_patch))\n        x = x.permute(0, 2, 1)\n        x = x.contiguous().view(B, hidden, h, w)\n        x = nn.Upsample(scale_factor=self.scale_factor)(x)\n\n        out = self.conv(x)\n        out = self.norm(out)\n        out = self.activation(out)\n        return out\n\nclass Attention_org(nn.Module):\n    def __init__(self, config, vis,channel_num):\n        super(Attention_org, self).__init__()\n        self.vis = vis\n        self.KV_size = config.KV_size\n        self.channel_num = channel_num\n        self.num_attention_heads = config.transformer[\"num_heads\"]\n\n        self.query1 = nn.ModuleList()\n        self.query2 = nn.ModuleList()\n        self.query3 = nn.ModuleList()\n        self.query4 = nn.ModuleList()\n        self.key = nn.ModuleList()\n        self.value = nn.ModuleList()\n\n        for _ in range(config.transformer[\"num_heads\"]):\n            query1 = nn.Linear(channel_num[0], channel_num[0], bias=False)\n            query2 = nn.Linear(channel_num[1], channel_num[1], bias=False)\n            query3 = nn.Linear(channel_num[2], channel_num[2], bias=False)\n            query4 = nn.Linear(channel_num[3], channel_num[3], bias=False)\n            key = nn.Linear( self.KV_size,  self.KV_size, bias=False)\n            value = nn.Linear(self.KV_size,  self.KV_size, bias=False)\n            self.query1.append(copy.deepcopy(query1))\n            self.query2.append(copy.deepcopy(query2))\n            self.query3.append(copy.deepcopy(query3))\n            self.query4.append(copy.deepcopy(query4))\n            self.key.append(copy.deepcopy(key))\n            self.value.append(copy.deepcopy(value))\n        self.psi = nn.InstanceNorm2d(self.num_attention_heads)\n        self.softmax = Softmax(dim=3)\n        self.out1 = nn.Linear(channel_num[0], channel_num[0], bias=False)\n        self.out2 = nn.Linear(channel_num[1], channel_num[1], bias=False)\n        self.out3 = nn.Linear(channel_num[2], channel_num[2], bias=False)\n        self.out4 = nn.Linear(channel_num[3], channel_num[3], bias=False)\n        self.attn_dropout = Dropout(config.transformer[\"attention_dropout_rate\"])\n        self.proj_dropout = Dropout(config.transformer[\"attention_dropout_rate\"])\n\n\n\n    def forward(self, emb1,emb2,emb3,emb4, emb_all):\n        multi_head_Q1_list = []\n        multi_head_Q2_list = []\n        multi_head_Q3_list = []\n        multi_head_Q4_list = []\n        multi_head_K_list = []\n        multi_head_V_list = []\n        if emb1 is not None:\n            for query1 in self.query1:\n                Q1 = query1(emb1)\n                multi_head_Q1_list.append(Q1)\n        if emb2 is not None:\n            for query2 in self.query2:\n                Q2 = query2(emb2)\n                multi_head_Q2_list.append(Q2)\n        if emb3 is not None:\n            for query3 in self.query3:\n                Q3 = query3(emb3)\n                multi_head_Q3_list.append(Q3)\n        if emb4 is not None:\n            for query4 in self.query4:\n                Q4 = query4(emb4)\n                multi_head_Q4_list.append(Q4)\n        for key in self.key:\n            K = key(emb_all)\n            multi_head_K_list.append(K)\n        for value in self.value:\n            V = value(emb_all)\n            multi_head_V_list.append(V)\n        # print(len(multi_head_Q4_list))\n\n        multi_head_Q1 = torch.stack(multi_head_Q1_list, dim=1) if emb1 is not None else None\n        multi_head_Q2 = torch.stack(multi_head_Q2_list, dim=1) if emb2 is not None else None\n        multi_head_Q3 = torch.stack(multi_head_Q3_list, dim=1) if emb3 is not None else None\n        multi_head_Q4 = torch.stack(multi_head_Q4_list, dim=1) if emb4 is not None else None\n        multi_head_K = torch.stack(multi_head_K_list, dim=1)\n        multi_head_V = torch.stack(multi_head_V_list, dim=1)\n\n        multi_head_Q1 = multi_head_Q1.transpose(-1, -2) if emb1 is not None else None\n        multi_head_Q2 = multi_head_Q2.transpose(-1, -2) if emb2 is not None else None\n        multi_head_Q3 = multi_head_Q3.transpose(-1, -2) if emb3 is not None else None\n        multi_head_Q4 = multi_head_Q4.transpose(-1, -2) if emb4 is not None else None\n\n        attention_scores1 = torch.matmul(multi_head_Q1, multi_head_K) if emb1 is not None else None\n        attention_scores2 = torch.matmul(multi_head_Q2, multi_head_K) if emb2 is not None else None\n        attention_scores3 = torch.matmul(multi_head_Q3, multi_head_K) if emb3 is not None else None\n        attention_scores4 = torch.matmul(multi_head_Q4, multi_head_K) if emb4 is not None else None\n\n        attention_scores1 = attention_scores1 / math.sqrt(self.KV_size) if emb1 is not None else None\n        attention_scores2 = attention_scores2 / math.sqrt(self.KV_size) if emb2 is not None else None\n        attention_scores3 = attention_scores3 / math.sqrt(self.KV_size) if emb3 is not None else None\n        attention_scores4 = attention_scores4 / math.sqrt(self.KV_size) if emb4 is not None else None\n\n        attention_probs1 = self.softmax(self.psi(attention_scores1)) if emb1 is not None else None\n        attention_probs2 = self.softmax(self.psi(attention_scores2)) if emb2 is not None else None\n        attention_probs3 = self.softmax(self.psi(attention_scores3)) if emb3 is not None else None\n        attention_probs4 = self.softmax(self.psi(attention_scores4)) if emb4 is not None else None\n        # print(attention_probs4.size())\n\n        if self.vis:\n            weights =  []\n            weights.append(attention_probs1.mean(1))\n            weights.append(attention_probs2.mean(1))\n            weights.append(attention_probs3.mean(1))\n            weights.append(attention_probs4.mean(1))\n        else: weights=None\n\n        attention_probs1 = self.attn_dropout(attention_probs1) if emb1 is not None else None\n        attention_probs2 = self.attn_dropout(attention_probs2) if emb2 is not None else None\n        attention_probs3 = self.attn_dropout(attention_probs3) if emb3 is not None else None\n        attention_probs4 = self.attn_dropout(attention_probs4) if emb4 is not None else None\n\n        multi_head_V = multi_head_V.transpose(-1, -2)\n        context_layer1 = torch.matmul(attention_probs1, multi_head_V) if emb1 is not None else None\n        context_layer2 = torch.matmul(attention_probs2, multi_head_V) if emb2 is not None else None\n        context_layer3 = torch.matmul(attention_probs3, multi_head_V) if emb3 is not None else None\n        context_layer4 = torch.matmul(attention_probs4, multi_head_V) if emb4 is not None else None\n\n        context_layer1 = context_layer1.permute(0, 3, 2, 1).contiguous() if emb1 is not None else None\n        context_layer2 = context_layer2.permute(0, 3, 2, 1).contiguous() if emb2 is not None else None\n        context_layer3 = context_layer3.permute(0, 3, 2, 1).contiguous() if emb3 is not None else None\n        context_layer4 = context_layer4.permute(0, 3, 2, 1).contiguous() if emb4 is not None else None\n        context_layer1 = context_layer1.mean(dim=3) if emb1 is not None else None\n        context_layer2 = context_layer2.mean(dim=3) if emb2 is not None else None\n        context_layer3 = context_layer3.mean(dim=3) if emb3 is not None else None\n        context_layer4 = context_layer4.mean(dim=3) if emb4 is not None else None\n\n        O1 = self.out1(context_layer1) if emb1 is not None else None\n        O2 = self.out2(context_layer2) if emb2 is not None else None\n        O3 = self.out3(context_layer3) if emb3 is not None else None\n        O4 = self.out4(context_layer4) if emb4 is not None else None\n        O1 = self.proj_dropout(O1) if emb1 is not None else None\n        O2 = self.proj_dropout(O2) if emb2 is not None else None\n        O3 = self.proj_dropout(O3) if emb3 is not None else None\n        O4 = self.proj_dropout(O4) if emb4 is not None else None\n        return O1,O2,O3,O4, weights\n\n\n\n\nclass Mlp(nn.Module):\n    def __init__(self,config, in_channel, mlp_channel):\n        super(Mlp, self).__init__()\n        self.fc1 = nn.Linear(in_channel, mlp_channel)\n        self.fc2 = nn.Linear(mlp_channel, in_channel)\n        self.act_fn = nn.GELU()\n        self.dropout = Dropout(config.transformer[\"dropout_rate\"])\n        self._init_weights()\n\n    def _init_weights(self):\n        nn.init.xavier_uniform_(self.fc1.weight)\n        nn.init.xavier_uniform_(self.fc2.weight)\n        nn.init.normal_(self.fc1.bias, std=1e-6)\n        nn.init.normal_(self.fc2.bias, std=1e-6)\n\n    def forward(self, x):\n        x = self.fc1(x)\n        x = self.act_fn(x)\n        x = self.dropout(x)\n        x = self.fc2(x)\n        x = self.dropout(x)\n        return x\n\nclass Block_ViT(nn.Module):\n    def __init__(self, config, vis, channel_num):\n        super(Block_ViT, self).__init__()\n        expand_ratio = config.expand_ratio\n        self.attn_norm1 = LayerNorm(channel_num[0],eps=1e-6)\n        self.attn_norm2 = LayerNorm(channel_num[1],eps=1e-6)\n        self.attn_norm3 = LayerNorm(channel_num[2],eps=1e-6)\n        self.attn_norm4 = LayerNorm(channel_num[3],eps=1e-6)\n        self.attn_norm =  LayerNorm(config.KV_size,eps=1e-6)\n        self.channel_attn = Attention_org(config, vis, channel_num)\n\n        self.ffn_norm1 = LayerNorm(channel_num[0],eps=1e-6)\n        self.ffn_norm2 = LayerNorm(channel_num[1],eps=1e-6)\n        self.ffn_norm3 = LayerNorm(channel_num[2],eps=1e-6)\n        self.ffn_norm4 = LayerNorm(channel_num[3],eps=1e-6)\n        self.ffn1 = Mlp(config,channel_num[0],channel_num[0]*expand_ratio)\n        self.ffn2 = Mlp(config,channel_num[1],channel_num[1]*expand_ratio)\n        self.ffn3 = Mlp(config,channel_num[2],channel_num[2]*expand_ratio)\n        self.ffn4 = Mlp(config,channel_num[3],channel_num[3]*expand_ratio)\n\n\n    def forward(self, emb1,emb2,emb3,emb4):\n        embcat = []\n        org1 = emb1\n        org2 = emb2\n        org3 = emb3\n        org4 = emb4\n        for i in range(4):\n            var_name = \"emb\"+str(i+1)\n            tmp_var = locals()[var_name]\n            if tmp_var is not None:\n                embcat.append(tmp_var)\n\n        emb_all = torch.cat(embcat,dim=2)\n        cx1 = self.attn_norm1(emb1) if emb1 is not None else None\n        cx2 = self.attn_norm2(emb2) if emb2 is not None else None\n        cx3 = self.attn_norm3(emb3) if emb3 is not None else None\n        cx4 = self.attn_norm4(emb4) if emb4 is not None else None\n        emb_all = self.attn_norm(emb_all)\n        cx1,cx2,cx3,cx4, weights = self.channel_attn(cx1,cx2,cx3,cx4,emb_all)\n        cx1 = org1 + cx1 if emb1 is not None else None\n        cx2 = org2 + cx2 if emb2 is not None else None\n        cx3 = org3 + cx3 if emb3 is not None else None\n        cx4 = org4 + cx4 if emb4 is not None else None\n\n        org1 = cx1\n        org2 = cx2\n        org3 = cx3\n        org4 = cx4\n        x1 = self.ffn_norm1(cx1) if emb1 is not None else None\n        x2 = self.ffn_norm2(cx2) if emb2 is not None else None\n        x3 = self.ffn_norm3(cx3) if emb3 is not None else None\n        x4 = self.ffn_norm4(cx4) if emb4 is not None else None\n        x1 = self.ffn1(x1) if emb1 is not None else None\n        x2 = self.ffn2(x2) if emb2 is not None else None\n        x3 = self.ffn3(x3) if emb3 is not None else None\n        x4 = self.ffn4(x4) if emb4 is not None else None\n        x1 = x1 + org1 if emb1 is not None else None\n        x2 = x2 + org2 if emb2 is not None else None\n        x3 = x3 + org3 if emb3 is not None else None\n        x4 = x4 + org4 if emb4 is not None else None\n\n        return x1, x2, x3, x4, weights\n\n\nclass Encoder(nn.Module):\n    def __init__(self, config, vis, channel_num):\n        super(Encoder, self).__init__()\n        self.vis = vis\n        self.layer = nn.ModuleList()\n        self.encoder_norm1 = LayerNorm(channel_num[0],eps=1e-6)\n        self.encoder_norm2 = LayerNorm(channel_num[1],eps=1e-6)\n        self.encoder_norm3 = LayerNorm(channel_num[2],eps=1e-6)\n        self.encoder_norm4 = LayerNorm(channel_num[3],eps=1e-6)\n        for _ in range(config.transformer[\"num_layers\"]):\n            layer = Block_ViT(config, vis, channel_num)\n            self.layer.append(copy.deepcopy(layer))\n\n    def forward(self, emb1,emb2,emb3,emb4):\n        attn_weights = []\n        for layer_block in self.layer:\n            emb1,emb2,emb3,emb4, weights = layer_block(emb1,emb2,emb3,emb4)\n            if self.vis:\n                attn_weights.append(weights)\n        emb1 = self.encoder_norm1(emb1) if emb1 is not None else None\n        emb2 = self.encoder_norm2(emb2) if emb2 is not None else None\n        emb3 = self.encoder_norm3(emb3) if emb3 is not None else None\n        emb4 = self.encoder_norm4(emb4) if emb4 is not None else None\n        return emb1,emb2,emb3,emb4, attn_weights\n\n\nclass ChannelTransformer(nn.Module):\n    def __init__(self, config, vis, img_size, channel_num=[64, 128, 256, 512], patchSize=[32, 16, 8, 4]):\n        super().__init__()\n\n        self.patchSize_1 = patchSize[0]\n        self.patchSize_2 = patchSize[1]\n        self.patchSize_3 = patchSize[2]\n        self.patchSize_4 = patchSize[3]\n        self.embeddings_1 = Channel_Embeddings(config,self.patchSize_1, img_size=img_size//4,    in_channels=channel_num[0])\n        self.embeddings_2 = Channel_Embeddings(config,self.patchSize_2, img_size=img_size//8, in_channels=channel_num[1])\n        self.embeddings_3 = Channel_Embeddings(config,self.patchSize_3, img_size=img_size//16, in_channels=channel_num[2])\n        self.embeddings_4 = Channel_Embeddings(config,self.patchSize_4, img_size=img_size//32, in_channels=channel_num[3])\n        self.encoder = Encoder(config, vis, channel_num)\n\n        self.reconstruct_1 = Reconstruct(channel_num[0], channel_num[0], kernel_size=1,scale_factor=(self.patchSize_1,self.patchSize_1))\n        self.reconstruct_2 = Reconstruct(channel_num[1], channel_num[1], kernel_size=1,scale_factor=(self.patchSize_2,self.patchSize_2))\n        self.reconstruct_3 = Reconstruct(channel_num[2], channel_num[2], kernel_size=1,scale_factor=(self.patchSize_3,self.patchSize_3))\n        self.reconstruct_4 = Reconstruct(channel_num[3], channel_num[3], kernel_size=1,scale_factor=(self.patchSize_4,self.patchSize_4))\n\n    def forward(self,en1,en2,en3,en4):\n\n        emb1 = self.embeddings_1(en1)\n        emb2 = self.embeddings_2(en2)\n        emb3 = self.embeddings_3(en3)\n        emb4 = self.embeddings_4(en4)\n\n        encoded1, encoded2, encoded3, encoded4, attn_weights = self.encoder(emb1,emb2,emb3,emb4)  # (B, n_patch, hidden)\n        x1 = self.reconstruct_1(encoded1) if en1 is not None else None\n        x2 = self.reconstruct_2(encoded2) if en2 is not None else None\n        x3 = self.reconstruct_3(encoded3) if en3 is not None else None\n        x4 = self.reconstruct_4(encoded4) if en4 is not None else None\n\n        x1 = x1 + en1  if en1 is not None else None\n        x2 = x2 + en2  if en2 is not None else None\n        x3 = x3 + en3  if en3 is not None else None\n        x4 = x4 + en4  if en4 is not None else None\n\n        return x1, x2, x3, x4, attn_weights\n","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:26:16.614352Z","iopub.execute_input":"2022-02-18T14:26:16.614833Z","iopub.status.idle":"2022-02-18T14:26:16.700256Z","shell.execute_reply.started":"2022-02-18T14:26:16.614796Z","shell.execute_reply":"2022-02-18T14:26:16.69947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport torch\nfrom torch import nn, optim\nimport torch.nn.functional as F\nimport sys\npackage_dir = \"../input/pretrainedmodels/pretrained-models.pytorch-master/\"\nsys.path.insert(0, package_dir)\nimport pretrainedmodels\n# from CTrans_mf2 import ChannelTransformer\n# import Config_mf3\n# from torchsummary import summary\n\ndef conv3x3(in_channel, out_channel): #not change resolusion\n    return nn.Conv2d(in_channel,out_channel,\n                      kernel_size=3,stride=1,padding=1,dilation=1,bias=False)\n\ndef conv1x1(in_channel, out_channel): #not change resolution\n    return nn.Conv2d(in_channel,out_channel,\n                      kernel_size=1,stride=1,padding=0,dilation=1,bias=False)\n\ndef init_weight(m):\n    classname = m.__class__.__name__\n    if classname.find('Conv') != -1:\n        #nn.init.xavier_uniform_(m.weight, gain=1)\n        #nn.init.xavier_normal_(m.weight, gain=1)\n        #nn.init.kaiming_uniform_(m.weight, mode='fan_in', nonlinearity='relu')\n        nn.init.kaiming_normal_(m.weight, mode='fan_in', nonlinearity='relu')\n        #nn.init.orthogonal_(m.weight, gain=1)\n        if m.bias is not None:\n            m.bias.data.zero_()\n    elif classname.find('Batch') != -1:\n        m.weight.data.normal_(1,0.02)\n        m.bias.data.zero_()\n    elif classname.find('Linear') != -1:\n        nn.init.orthogonal_(m.weight, gain=1)\n        if m.bias is not None:\n            m.bias.data.zero_()\n    elif classname.find('Embedding') != -1:\n        nn.init.orthogonal_(m.weight, gain=1)\n\n\nclass ChannelAttentionModule(nn.Module):\n    def __init__(self, in_channel, reduction):\n        super().__init__()\n        self.global_maxpool = nn.AdaptiveMaxPool2d(1)\n        self.global_avgpool = nn.AdaptiveAvgPool2d(1) \n        self.fc = nn.Sequential(\n            conv1x1(in_channel, in_channel//reduction).apply(init_weight),\n            nn.ReLU(True),\n            conv1x1(in_channel//reduction, in_channel).apply(init_weight)\n        )\n        \n    def forward(self, inputs):\n        x1 = self.global_maxpool(inputs)\n        x2 = self.global_avgpool(inputs)\n        x1 = self.fc(x1)\n        x2 = self.fc(x2)\n        x  = torch.sigmoid(x1 + x2)\n        return x\n\n\nclass SpatialAttentionModule(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv3x3 = conv3x3(2,1).apply(init_weight)\n        \n    def forward(self, inputs):\n        x1,_ = torch.max(inputs, dim=1, keepdim=True)\n        x2 = torch.mean(inputs, dim=1, keepdim=True)\n        x  = torch.cat([x1,x2], dim=1)\n        x  = self.conv3x3(x)\n        x  = torch.sigmoid(x)\n        return x\n\n\nclass CBAM(nn.Module):\n    def __init__(self, in_channel, reduction):\n        super().__init__()\n        self.channel_attention = ChannelAttentionModule(in_channel, reduction)\n        self.spatial_attention = SpatialAttentionModule()\n        \n    def forward(self, inputs):\n        x = inputs * self.channel_attention(inputs)\n        x = x * self.spatial_attention(x)\n        return x\n\n\nclass CenterBlock(nn.Module):\n    def __init__(self, in_channel, out_channel):\n        super().__init__()\n        self.conv = conv3x3(in_channel, out_channel).apply(init_weight)\n        \n    def forward(self, inputs):\n        x = self.conv(inputs)\n        return x\n\n\nclass DecodeBlock(nn.Module):\n    def __init__(self, in_channel, out_channel, upsample):\n        super().__init__()\n        self.bn1 = nn.BatchNorm2d(in_channel).apply(init_weight)\n        self.upsample = nn.Sequential()\n        if upsample:\n            self.upsample.add_module('upsample',nn.Upsample(scale_factor=2, mode='nearest'))\n        self.conv3x3_1 = conv3x3(in_channel, in_channel).apply(init_weight)\n        self.bn2 = nn.BatchNorm2d(in_channel).apply(init_weight)\n        self.conv3x3_2 = conv3x3(in_channel, out_channel).apply(init_weight)\n        self.cbam = CBAM(out_channel, reduction=16)\n        self.conv1x1   = conv1x1(in_channel, out_channel).apply(init_weight)\n        \n    def forward(self, inputs):\n        x  = F.relu(self.bn1(inputs))\n        x  = self.upsample(x)\n        x  = self.conv3x3_1(x)\n        x  = self.conv3x3_2(F.relu(self.bn2(x)))\n        x  = self.cbam(x)\n        x += self.conv1x1(self.upsample(inputs)) #shortcut\n        return x\n\n\n#U-Net SeResNext101 + CBAM + hypercolumns + deepsupervision\nclass UNET_SERESNEXT101(nn.Module):\n    def __init__(self, resolution, deepsupervision, clfhead, load_weights=True):\n        super().__init__()\n        h,w = resolution\n        self.deepsupervision = deepsupervision\n        self.clfhead = clfhead\n#         self.clf_threshold = clf_threshold\n        \n        #encoder\n        model_name = 'se_resnext101_32x4d'\n#         if load_weights:\n#             seresnext101 = pretrainedmodels.__dict__[model_name](pretrained='imagenet')\n#         else:\n#             seresnext101 = pretrainedmodels.__dict__[model_name](pretrained=None)\n        seresnext101 = pretrainedmodels.__dict__[model_name](pretrained=None)\n        if load_weights:\n            seresnext101.load_state_dict(torch.load(f'../../../pretrainedmodels_weight/{model_name}.pth'))\n        \n        self.encoder0 = nn.Sequential(\n            seresnext101.layer0.conv1, #(*,3,h,w)->(*,64,h/2,w/2)\n            seresnext101.layer0.bn1,\n            seresnext101.layer0.relu1,\n        )\n        self.encoder1 = nn.Sequential(\n            seresnext101.layer0.pool, #->(*,64,h/4,w/4)\n            seresnext101.layer1 #->(*,256,h/4,w/4)\n        )\n        self.encoder2 = seresnext101.layer2 #->(*,512,h/8,w/8)\n        self.encoder3 = seresnext101.layer3 #->(*,1024,h/16,w/16)\n        self.encoder4 = seresnext101.layer4 #->(*,2048,h/32,w/32)\n        \n        #center\n        self.center  = CenterBlock(2048,512) #->(*,512,h/32,w/32)\n        \n        #decoder\n        self.decoder4 = DecodeBlock(512+2048,64, upsample=True) #->(*,64,h/16,w/16)\n        self.decoder3 = DecodeBlock(64+1024,64, upsample=True) #->(*,64,h/8,w/8)\n        self.decoder2 = DecodeBlock(64+512,64,  upsample=True) #->(*,64,h/4,w/4) \n        self.decoder1 = DecodeBlock(64+256,64,   upsample=True) #->(*,64,h/2,w/2) \n        self.decoder0 = DecodeBlock(64,64, upsample=True) #->(*,64,h,w) \n        \n        #upsample\n        self.upsample4 = nn.Upsample(scale_factor=16, mode='bilinear', align_corners=True)\n        self.upsample3 = nn.Upsample(scale_factor=8, mode='bilinear', align_corners=True)\n        self.upsample2 = nn.Upsample(scale_factor=4, mode='bilinear', align_corners=True)\n        self.upsample1 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        \n        #deep supervision\n        self.deep4 = conv1x1(64,1).apply(init_weight)\n        self.deep3 = conv1x1(64,1).apply(init_weight)\n        self.deep2 = conv1x1(64,1).apply(init_weight)\n        self.deep1 = conv1x1(64,1).apply(init_weight)\n        \n        #final conv\n        self.final_conv = nn.Sequential(\n            conv3x3(320,64).apply(init_weight),\n            nn.ELU(True),\n            conv1x1(64,1).apply(init_weight)\n        )\n        \n        #clf head\n        self.avgpool = nn.AdaptiveAvgPool2d(1)\n        self.clf = nn.Sequential(\n            nn.BatchNorm1d(2048).apply(init_weight),\n            nn.Linear(2048,512).apply(init_weight),\n            nn.ELU(True),\n            nn.BatchNorm1d(512).apply(init_weight),\n            nn.Linear(512,1).apply(init_weight)\n        )\n        \n    def forward(self, inputs):\n        #encoder\n        x0 = self.encoder0(inputs) #->(*,64,h/2,w/2)\n        x1 = self.encoder1(x0) #->(*,256,h/4,w/4)\n        x2 = self.encoder2(x1) #->(*,512,h/8,w/8)\n        x3 = self.encoder3(x2) #->(*,1024,h/16,w/16)\n        x4 = self.encoder4(x3) #->(*,2048,h/32,w/32)\n        \n        #clf head\n        logits_clf = self.clf(self.avgpool(x4).squeeze(-1).squeeze(-1)) #->(*,1)\n        if config['clf_threshold'] is not None:\n            if (torch.sigmoid(logits_clf)>config['clf_threshold']).sum().item()==0:\n                bs,_,h,w = inputs.shape\n                logits = torch.zeros((bs,1,h,w))\n                if self.clfhead:\n                    if self.deepsupervision:\n                        return logits,_,_\n                    else:\n                        return logits,_\n                else:\n                    if self.deepsupervision:\n                        return logits,_\n                    else:\n                        return logits\n        \n        #center\n        y5 = self.center(x4) #->(*,320,h/32,w/32)\n        \n        #decoder\n        y4 = self.decoder4(torch.cat([x4,y5], dim=1)) #->(*,64,h/16,w/16)\n        y3 = self.decoder3(torch.cat([x3,y4], dim=1)) #->(*,64,h/8,w/8)\n        y2 = self.decoder2(torch.cat([x2,y3], dim=1)) #->(*,64,h/4,w/4)\n        y1 = self.decoder1(torch.cat([x1,y2], dim=1)) #->(*,64,h/2,w/2) \n        y0 = self.decoder0(y1) #->(*,64,h,w)\n        \n        #hypercolumns\n        y4 = self.upsample4(y4) #->(*,64,h,w)\n        y3 = self.upsample3(y3) #->(*,64,h,w)\n        y2 = self.upsample2(y2) #->(*,64,h,w)\n        y1 = self.upsample1(y1) #->(*,64,h,w)\n        hypercol = torch.cat([y0,y1,y2,y3,y4], dim=1)\n        \n        #final conv\n        logits = self.final_conv(hypercol) #->(*,1,h,w)\n        \n        #clf head\n        logits_clf = self.clf(self.avgpool(x4).squeeze(-1).squeeze(-1)) #->(*,1)\n        \n        if self.clfhead:\n            if self.deepsupervision:\n                s4 = self.deep4(y4)\n                s3 = self.deep3(y3)\n                s2 = self.deep2(y2)\n                s1 = self.deep1(y1)\n                logits_deeps = [s4,s3,s2,s1]\n                return logits, logits_deeps, logits_clf\n            else:\n                return logits, logits_clf\n        else:\n            if self.deepsupervision:\n                s4 = self.deep4(y4)\n                s3 = self.deep3(y3)\n                s2 = self.deep2(y2)\n                s1 = self.deep1(y1)\n                logits_deeps = [s4,s3,s2,s1]\n                return logits, logits_deeps\n            else:\n                return logits\n\nclass Flatten(nn.Module):\n    def forward(self, x):\n        return x.view(x.size(0), -1)\n\nclass CCA(nn.Module):\n    \"\"\"\n    CCA Block\n    \"\"\"\n    def __init__(self, F_g, F_x):\n        super().__init__()\n        self.mlp_x = nn.Sequential(\n            Flatten(),\n            nn.Linear(F_x, F_x))\n        self.mlp_g = nn.Sequential(\n            Flatten(),\n            nn.Linear(F_g, F_x))\n        self.relu = nn.ReLU(inplace=True)\n\n    def forward(self, g, x):\n        # channel-wise attention\n        avg_pool_x = F.avg_pool2d( x, (x.size(2), x.size(3)), stride=(x.size(2), x.size(3)))\n        channel_att_x = self.mlp_x(avg_pool_x)\n        avg_pool_g = F.avg_pool2d( g, (g.size(2), g.size(3)), stride=(g.size(2), g.size(3)))\n        channel_att_g = self.mlp_g(avg_pool_g)\n        channel_att_sum = (channel_att_x + channel_att_g)/2.0\n        scale = torch.sigmoid(channel_att_sum).unsqueeze(2).unsqueeze(3).expand_as(x)\n        x_after_channel = x * scale\n        out = self.relu(x_after_channel)\n        return out\n\nclass DecodeBlock_CTrans(nn.Module):\n    def __init__(self, in_channel, out_channel, upsample):\n        super().__init__()\n        self.bn1 = nn.BatchNorm2d(in_channel).apply(init_weight)\n        self.upsample = nn.Sequential()\n        if upsample:\n            self.upsample.add_module('upsample',nn.Upsample(scale_factor=2, mode='nearest'))\n        self.conv3x3_1 = conv3x3(in_channel, in_channel).apply(init_weight)\n        self.bn2 = nn.BatchNorm2d(in_channel).apply(init_weight)\n        self.conv3x3_2 = conv3x3(in_channel, out_channel).apply(init_weight)\n        #self.cbam = CBAM(out_channel, reduction=16)\n        self.conv1x1   = conv1x1(in_channel, out_channel).apply(init_weight)\n        \n    def forward(self, inputs):\n        x  = F.relu(self.bn1(inputs))\n        x  = self.upsample(x)\n        x  = self.conv3x3_1(x)\n        x  = self.conv3x3_2(F.relu(self.bn2(x)))\n        #x  = self.cbam(x)\n        x += self.conv1x1(self.upsample(inputs)) #shortcut\n        return x\n\nclass UpBlock_attention(nn.Module):\n    def __init__(self, skip_channels, up_channels=64, out_channels=64, activation='ReLU'):\n        super().__init__()\n        #self.up = nn.Upsample(scale_factor=2)\n        self.coatt = CCA(F_g=up_channels, F_x=skip_channels)\n        #self.nConvs = _make_nConv(in_channels, out_channels, nb_Conv, activation)\n        self.nConvs = DecodeBlock_CTrans(skip_channels + up_channels, out_channels, upsample=True)\n\n\n    def forward(self, x, skip_x):\n        #up = self.up(x)\n        skip_x_att = self.coatt(g=x, x=skip_x)\n        x = torch.cat([skip_x_att, x], dim=1)  # dim 1 is the channel dimension\n\n        return self.nConvs(x)\n\nclass UNET_SERESNEXT101_CTrans(nn.Module):\n    def __init__(self, config, deepsupervision, clfhead, load_weights=True, img_size=224, vis=False):\n        super().__init__()\n        #h,w = resolution\n        self.deepsupervision = deepsupervision\n        self.clfhead = clfhead\n#         self.clf_threshold = clf_threshold\n        self.vis = vis\n\n        #encoder\n        model_name = 'se_resnext101_32x4d'\n        seresnext101 = pretrainedmodels.__dict__[model_name](pretrained=None)\n        if load_weights:\n            seresnext101.load_state_dict(torch.load(f'../../../pretrainedmodels_weight/{model_name}.pth'))\n        self.mtc = ChannelTransformer(config, vis, img_size,\n                                     channel_num=[256, 512, 1024, 2048],\n                                     patchSize=config.patch_sizes)\n        self.encoder0 = nn.Sequential(\n            seresnext101.layer0.conv1, #(*,3,h,w)->(*,64,h/2,w/2)\n            seresnext101.layer0.bn1,\n            seresnext101.layer0.relu1,\n        )\n        self.encoder1 = nn.Sequential(\n            seresnext101.layer0.pool, #->(*,64,h/4,w/4)\n            seresnext101.layer1 #->(*,256,h/4,w/4)\n        )\n        self.encoder2 = seresnext101.layer2 #->(*,512,h/8,w/8)\n        self.encoder3 = seresnext101.layer3 #->(*,1024,h/16,w/16)\n        self.encoder4 = seresnext101.layer4 #->(*,2048,h/32,w/32)\n        \n        #center\n        self.center  = CenterBlock(2048,512) #->(*,512,h/32,w/32)\n        \n        #decoder\n        #self.decoder4 = DecodeBlock(512+2048,64, upsample=True) #->(*,64,h/16,w/16)\n        #self.decoder3 = DecodeBlock(64+1024,64, upsample=True) #->(*,64,h/8,w/8)\n        #self.decoder2 = DecodeBlock(64+512,64,  upsample=True) #->(*,64,h/4,w/4) \n        #self.decoder1 = DecodeBlock(64+256,64,   upsample=True) #->(*,64,h/2,w/2)\n        self.decoder4 = UpBlock_attention(skip_channels=2048, up_channels = 512, out_channels = 64)\n        self.decoder3 = UpBlock_attention(skip_channels=1024, up_channels = 64, out_channels = 64)\n        self.decoder2 = UpBlock_attention(skip_channels=512, up_channels = 64, out_channels = 64)\n        self.decoder1 = UpBlock_attention(skip_channels=256, up_channels = 64, out_channels = 64)\n        self.decoder0 = DecodeBlock(64,64, upsample=True) #->(*,64,h,w) \n        \n        #upsample\n        self.upsample4 = nn.Upsample(scale_factor=16, mode='bilinear', align_corners=True)\n        self.upsample3 = nn.Upsample(scale_factor=8, mode='bilinear', align_corners=True)\n        self.upsample2 = nn.Upsample(scale_factor=4, mode='bilinear', align_corners=True)\n        self.upsample1 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        \n        #deep supervision\n        self.deep4 = conv1x1(64,1).apply(init_weight)\n        self.deep3 = conv1x1(64,1).apply(init_weight)\n        self.deep2 = conv1x1(64,1).apply(init_weight)\n        self.deep1 = conv1x1(64,1).apply(init_weight)\n        \n        #final conv\n        self.final_conv = nn.Sequential(\n            conv3x3(320,64).apply(init_weight),\n            nn.ELU(True),\n            conv1x1(64,1).apply(init_weight)\n        )\n        \n        #clf head\n        self.avgpool = nn.AdaptiveAvgPool2d(1)\n        self.clf = nn.Sequential(\n            nn.BatchNorm1d(2048).apply(init_weight),\n            nn.Linear(2048,512).apply(init_weight),\n            nn.ELU(True),\n            nn.BatchNorm1d(512).apply(init_weight),\n            nn.Linear(512,1).apply(init_weight)\n        )\n        \n    def forward(self, inputs):\n        #encoder\n        x0 = self.encoder0(inputs) #->(*,64,h/2,w/2)\n        x1 = self.encoder1(x0) #->(*,256,h/4,w/4)\n        x2 = self.encoder2(x1) #->(*,512,h/8,w/8)\n        x3 = self.encoder3(x2) #->(*,1024,h/16,w/16)\n        x4 = self.encoder4(x3) #->(*,2048,h/32,w/32)\n        \n        #clf head\n        logits_clf = self.clf(self.avgpool(x4).squeeze(-1).squeeze(-1)) #->(*,1)\n        if config['clf_threshold'] is not None:\n            if (torch.sigmoid(logits_clf)>config['clf_threshold']).sum().item()==0:\n                bs,_,h,w = inputs.shape\n                logits = torch.zeros((bs,1,h,w))\n                if self.clfhead:\n                    if self.deepsupervision:\n                        return logits,_,_\n                    else:\n                        return logits,_\n                else:\n                    if self.deepsupervision:\n                        return logits,_\n                    else:\n                        return logits\n        \n        #center\n        y5 = self.center(x4) #->(*,512,h/32,w/32)\n\n        #CCT\n        x1, x2, x3, x4, _ = self.mtc(x1, x2, x3, x4)\n\n        #decoder\n        #y4 = self.decoder4(torch.cat([x4,y5], dim=1)) #->(*,64,h/16,w/16)\n        #y3 = self.decoder3(torch.cat([x3,y4], dim=1)) #->(*,64,h/8,w/8)\n        #y2 = self.decoder2(torch.cat([x2,y3], dim=1)) #->(*,64,h/4,w/4)\n        #y1 = self.decoder1(torch.cat([x1,y2], dim=1)) #->(*,64,h/2,w/2) \n        y4 = self.decoder4(y5, x4)\n        y3 = self.decoder3(y4, x3)\n        y2 = self.decoder2(y3, x2)\n        y1 = self.decoder1(y2, x1)\n        y0 = self.decoder0(y1) #->(*,64,h,w)\n        \n        #hypercolumns\n        y4 = self.upsample4(y4) #->(*,64,h,w)\n        y3 = self.upsample3(y3) #->(*,64,h,w)\n        y2 = self.upsample2(y2) #->(*,64,h,w)\n        y1 = self.upsample1(y1) #->(*,64,h,w)\n        hypercol = torch.cat([y0,y1,y2,y3,y4], dim=1)\n        \n        #final conv\n        logits = self.final_conv(hypercol) #->(*,1,h,w)\n        \n        #clf head\n        logits_clf = self.clf(self.avgpool(x4).squeeze(-1).squeeze(-1)) #->(*,1)\n        \n        if self.clfhead:\n            if self.deepsupervision:\n                s4 = self.deep4(y4)\n                s3 = self.deep3(y3)\n                s2 = self.deep2(y2)\n                s1 = self.deep1(y1)\n                logits_deeps = [s4,s3,s2,s1]\n                return logits, logits_deeps, logits_clf\n            else:\n                return logits, logits_clf\n        else:\n            if self.deepsupervision:\n                s4 = self.deep4(y4)\n                s3 = self.deep3(y3)\n                s2 = self.deep2(y2)\n                s1 = self.deep1(y1)\n                logits_deeps = [s4,s3,s2,s1]\n                return logits, logits_deeps\n            else:\n                return logits\n\n\ndef build_model(resolution, deepsupervision, clfhead, load_weights):\n    model_name = config['model_name']\n    if model_name=='seresnext101':\n        model = UNET_SERESNEXT101(resolution, deepsupervision, clfhead, load_weights)\n    elif model_name == 'seresnext101_ctrans':\n        model = UNET_SERESNEXT101_CTrans(get_CTranS_config(), deepsupervision, clfhead, load_weights, img_size = resolution[0])\n        \n    return model\n\n\n# if __name__ == '__main__':\n#     def get_parameter_number(model):\n#         total_num = sum(p.numel() for p in model.parameters())\n#         trainable_num = sum(p.numel() for p in model.parameters() if p.requires_grad)\n#         return {'Total' : total_num, 'Trainable' : trainable_num}\n#     device = \"cpu\"\n#     model = build_model(model_name='seresnext101_ctrans',\n#                         resolution=(320, 320),\n#                         deepsupervision=False,\n#                         clfhead=False,\n#                         clf_threshold=None,\n#                         load_weights=False).to(device, torch.float32)\n#     result=get_parameter_number(model)\n#     print('total: ', result['Total'], ', Trainable: ', result['Trainable'])\n","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:26:16.701736Z","iopub.execute_input":"2022-02-18T14:26:16.702064Z","iopub.status.idle":"2022-02-18T14:26:18.403269Z","shell.execute_reply.started":"2022-02-18T14:26:16.702027Z","shell.execute_reply":"2022-02-18T14:26:18.402502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"#from models import build_model\n\nLOAD_LOCAL_WEIGHT_PATH_LIST = {}\nfor seed in config['split_seed_list']:\n    LOAD_LOCAL_WEIGHT_PATH_LIST[seed] = []\n    for fold in config['FOLD_LIST']:\n        LOAD_LOCAL_WEIGHT_PATH_LIST[seed].append(opj(config['model_path'],f'model_seed{seed}_fold{fold}_bestscore.pth'))\n        #LOAD_LOCAL_WEIGHT_PATH_LIST[seed].append(opj(config['model_path'],f'model_seed{seed}_fold{fold}_swa.pth'))\n\nmodel_list = {}\nfor seed in config['split_seed_list']:\n    model_list[seed] = []\n    for path in LOAD_LOCAL_WEIGHT_PATH_LIST[seed]:\n        print(\"Loading weights from %s\" % path)\n        \n        model = build_model(resolution=(320,320), #config['resolution'], \n                            deepsupervision=config['deepsupervision'], \n                            clfhead=config['clfhead'],\n                            load_weights=False).to(device)\n        \n        model.load_state_dict(torch.load(path))\n        model.eval()\n        model_list[seed].append(model) ","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:26:18.406266Z","iopub.execute_input":"2022-02-18T14:26:18.40678Z","iopub.status.idle":"2022-02-18T14:27:53.699152Z","shell.execute_reply.started":"2022-02-18T14:26:18.406739Z","shell.execute_reply":"2022-02-18T14:27:53.698358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom albumentations import (Compose, HorizontalFlip, VerticalFlip, Rotate, RandomRotate90,\n                            ShiftScaleRotate, ElasticTransform,\n                            GridDistortion, RandomSizedCrop, RandomCrop, CenterCrop,\n                            RandomBrightnessContrast, HueSaturationValue, IAASharpen,\n                            RandomGamma, RandomBrightness, RandomBrightnessContrast,\n                            GaussianBlur,CLAHE,\n                            Cutout, CoarseDropout, GaussNoise, ChannelShuffle, ToGray, OpticalDistortion,\n                            Normalize, OneOf, NoOp)\nfrom albumentations.pytorch import ToTensorV2#, ToTensor\n#from get_config import *\n#config = get_config()\n\nMEAN = np.array([0.485, 0.456, 0.406])\nSTD  = np.array([0.229, 0.224, 0.225])\n\ndef get_transforms_test():\n    transforms = Compose([\n        Normalize(mean=(MEAN[0], MEAN[1], MEAN[2]), \n                  std=(STD[0], STD[1], STD[2])),\n        ToTensorV2(),\n    ] )\n    return transforms\n\ndef denormalize(z, mean=MEAN.reshape(-1,1,1), std=STD.reshape(-1,1,1)):\n    return std*z + mean","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:27:53.70059Z","iopub.execute_input":"2022-02-18T14:27:53.70084Z","iopub.status.idle":"2022-02-18T14:27:55.413824Z","shell.execute_reply.started":"2022-02-18T14:27:53.700806Z","shell.execute_reply":"2022-02-18T14:27:55.413094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset\n\nclass HuBMAPDataset(Dataset):\n    def __init__(self, idx, df):\n        super().__init__()\n        filename = df.loc[idx, 'id']+'.tiff'\n        path = opj(INPUT_PATH,'test',filename)\n        self.data = rasterio.open(path)\n        if self.data.count != 3:\n            subdatasets = self.data.subdatasets\n            self.layers = []\n            if len(subdatasets) > 0:\n                for i,subdataset in enumerate(subdatasets,0):\n                    self.layers.append(rasterio.open(subdataset))\n        self.h, self.w = self.data.height, self.data.width\n        self.input_sz = config['input_resolution']\n        self.sz = config['resolution']\n        self.pad_sz = config['pad_size'] # add to each input tile\n        self.pred_sz = self.sz - 2*self.pad_sz\n        self.pad_h = self.pred_sz - self.h % self.pred_sz # add to whole slide\n        self.pad_w = self.pred_sz - self.w % self.pred_sz # add to whole slide\n        self.num_h = (self.h + self.pad_h) // self.pred_sz\n        self.num_w = (self.w + self.pad_w) // self.pred_sz\n        self.transforms = get_transforms_test()\n        \n    def __len__(self):\n        return self.num_h * self.num_w\n    \n    def __getitem__(self, idx): # idx = i_h * self.num_w + i_w\n        # prepare coordinates for rasterio\n        i_h = idx // self.num_w\n        i_w = idx % self.num_w\n        y = i_h*self.pred_sz \n        x = i_w*self.pred_sz\n        py0,py1 = max(0,y), min(y+self.pred_sz, self.h)\n        px0,px1 = max(0,x), min(x+self.pred_sz, self.w)\n        \n        # padding coordinate for rasterio\n        qy0,qy1 = max(0,y-self.pad_sz), min(y+self.pred_sz+self.pad_sz, self.h)\n        qx0,qx1 = max(0,x-self.pad_sz), min(x+self.pred_sz+self.pad_sz, self.w)\n        \n        # placeholder for input tile (before resize)\n        img = np.zeros((self.sz,self.sz,3), np.uint8)\n        \n        # replace the value\n        if self.data.count == 3:\n            img[0:qy1-qy0, 0:qx1-qx0] =\\\n                np.moveaxis(self.data.read([1,2,3], window=Window.from_slices((qy0,qy1),(qx0,qx1))), 0,-1)\n        else:\n            for i,layer in enumerate(self.layers):\n                img[0:qy1-qy0, 0:qx1-qx0, i] =\\\n                    layer.read(1,window=Window.from_slices((qy0,qy1),(qx0,qx1)))\n        if self.sz != self.input_sz:\n            img = cv2.resize(img, (self.input_sz, self.input_sz), interpolation=cv2.INTER_AREA)\n        img = self.transforms(image=img)['image'] # to normalized tensor\n        return {'img':img, 'p':[py0,py1,px0,px1], 'q':[qy0,qy1,qx0,qx1]}","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:27:55.416822Z","iopub.execute_input":"2022-02-18T14:27:55.417091Z","iopub.status.idle":"2022-02-18T14:27:55.435242Z","shell.execute_reply.started":"2022-02-18T14:27:55.417045Z","shell.execute_reply":"2022-02-18T14:27:55.433746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader\nfrom tqdm.notebook import tqdm\nimport gc\nimport math\n\n\ndef my_collate_fn(batch):\n    img = []\n    p = []\n    q = []\n    for sample in batch:\n        img.append(sample['img'])\n        p.append(sample['p'])\n        q.append(sample['q'])\n    img = torch.stack(img)\n    return {'img':img, 'p':p, 'q':q}\n\n\nseed = 0\n\ndef get_pred_mask(idx, df, model_list):\n    ds = HuBMAPDataset(idx, df)\n    #rasterio cannot be used with multiple workers\n    dl = DataLoader(ds,batch_size=config['test_batch_size'],\n                    num_workers=0,shuffle=False,pin_memory=True,\n                    collate_fn=my_collate_fn) \n    \n    pred_mask = np.zeros((len(ds),ds.pred_sz,ds.pred_sz), dtype=np.uint8)\n    \n    i_data = 0\n    for data in tqdm(dl):\n        bs = data['img'].shape[0]\n        img_patch = data['img'] # (bs,3,input_res,input_res)\n        pred_mask_float = 0\n        for model in model_list[seed]:\n            with torch.no_grad():\n                if config['tta']>0:\n                    pred_mask_float += torch.sigmoid(model(img_patch.to(device, torch.float32, non_blocking=True))).detach().cpu().numpy()[:,0,:,:] #.squeeze()\n                if config['tta']>1:\n                    # h-flip\n                    _pred_mask_float = torch.sigmoid(model(img_patch.flip([-1]).to(device, torch.float32, non_blocking=True))).detach().cpu().numpy()[:,0,:,:] #.squeeze()\n                    pred_mask_float += _pred_mask_float[:,:,::-1]\n                if config['tta']>2:\n                    # v-flip\n                    _pred_mask_float = torch.sigmoid(model(img_patch.flip([-2]).to(device, torch.float32, non_blocking=True))).detach().cpu().numpy()[:,0,:,:] #.squeeze()\n                    pred_mask_float += _pred_mask_float[:,::-1,:]\n                if config['tta']>3:\n                    # h-v-flip\n                    _pred_mask_float = torch.sigmoid(model(img_patch.flip([-1,-2]).to(device, torch.float32, non_blocking=True))).detach().cpu().numpy()[:,0,:,:] #.squeeze()\n                    pred_mask_float += _pred_mask_float[:,::-1,::-1]\n        pred_mask_float = pred_mask_float / min(config['tta'],4) / len(model_list[seed]) # (bs,input_res,input_res)\n        \n        # resize\n        pred_mask_float = np.vstack([cv2.resize(_mask.astype(np.float32), (ds.sz,ds.sz))[None] for _mask in pred_mask_float])\n        \n        # float to uint8\n        pred_mask_int = (pred_mask_float>config['mask_threshold']).astype(np.uint8)\n        \n        # replace the values\n        for j in range(bs):\n            py0,py1,px0,px1 = data['p'][j]\n            qy0,qy1,qx0,qx1 = data['q'][j]\n            pred_mask[i_data+j,0:py1-py0, 0:px1-px0] = pred_mask_int[j, py0-qy0:py1-qy0, px0-qx0:px1-qx0] # (pred_sz,pred_sz)\n        i_data += bs\n    \n    pred_mask = pred_mask.reshape(ds.num_h*ds.num_w, ds.pred_sz, ds.pred_sz).reshape(ds.num_h, ds.num_w, ds.pred_sz, ds.pred_sz)\n    pred_mask = pred_mask.transpose(0,2,1,3).reshape(ds.num_h*ds.pred_sz, ds.num_w*ds.pred_sz)\n    pred_mask = pred_mask[:ds.h,:ds.w] # back to the original slide size\n    non_zero_ratio = (pred_mask).sum() / (ds.h*ds.w)\n    print('non_zero_ratio = {:.4f}'.format(non_zero_ratio))\n    return pred_mask,ds.h,ds.w\n\ndef get_rle(y_preds, h,w):\n    rle = mask2rle(y_preds, shape=(h,w), small_mask_threshold=config['small_mask_threshold'])\n    return rle","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:27:55.436892Z","iopub.execute_input":"2022-02-18T14:27:55.437218Z","iopub.status.idle":"2022-02-18T14:27:55.460858Z","shell.execute_reply.started":"2022-02-18T14:27:55.437168Z","shell.execute_reply":"2022-02-18T14:27:55.459921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfor idx in range(len(sub_df)): \n    print('idx = ', idx)\n    pred_mask,h,w = get_pred_mask(idx, sub_df, model_list)\n    rle = get_rle(pred_mask,h,w)\n    sub_df.loc[idx,'predicted'] = rle","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:27:55.462269Z","iopub.execute_input":"2022-02-18T14:27:55.463028Z","iopub.status.idle":"2022-02-18T14:57:50.309323Z","shell.execute_reply.started":"2022-02-18T14:27:55.46299Z","shell.execute_reply":"2022-02-18T14:57:50.308562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"sub_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:57:50.310807Z","iopub.execute_input":"2022-02-18T14:57:50.311068Z","iopub.status.idle":"2022-02-18T14:57:50.369774Z","shell.execute_reply.started":"2022-02-18T14:57:50.311033Z","shell.execute_reply":"2022-02-18T14:57:50.36903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2022-02-18T14:57:50.370915Z","iopub.execute_input":"2022-02-18T14:57:50.371663Z","iopub.status.idle":"2022-02-18T14:57:50.396705Z","shell.execute_reply.started":"2022-02-18T14:57:50.371623Z","shell.execute_reply":"2022-02-18T14:57:50.396059Z"},"trusted":true},"execution_count":null,"outputs":[]}]}