{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nfrom sklearn.metrics import roc_auc_score, accuracy_score, f1_score, log_loss\nimport pickle\nfrom torch.utils.data import DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nimport warnings\nimport sys\nimport pandas as pd\nimport os\nimport gc\nimport sys\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\nimport cv2\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport argparse\nimport importlib\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD, AdamW\n\nimport datetime\nimport wandb","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:08.817094Z","iopub.execute_input":"2023-06-11T01:02:08.817774Z","iopub.status.idle":"2023-06-11T01:02:13.642255Z","shell.execute_reply.started":"2023-06-11T01:02:08.817726Z","shell.execute_reply":"2023-06-11T01:02:13.640859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport hashlib\n\ndata_root = Path(\"/kaggle/input/vesuvius-challenge-ink-detection\")\ntest_root = data_root.joinpath(\"test\")\ntest_mask = sorted(test_root.glob(\"**/*.png\"))[0]\n\nwith test_mask.open(\"rb\") as f:\n    hash_md5 = hashlib.md5(f.read()).hexdigest()\n\nfast_sub = hash_md5 == \"0b0fffdc0e88be226673846a143bb3e0\"\nfast_sub","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:13.649622Z","iopub.execute_input":"2023-06-11T01:02:13.652188Z","iopub.status.idle":"2023-06-11T01:02:13.713403Z","shell.execute_reply.started":"2023-06-11T01:02:13.652138Z","shell.execute_reply":"2023-06-11T01:02:13.712345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if fast_sub:\n#     exit()","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:13.715773Z","iopub.execute_input":"2023-06-11T01:02:13.716938Z","iopub.status.idle":"2023-06-11T01:02:13.724159Z","shell.execute_reply.started":"2023-06-11T01:02:13.716895Z","shell.execute_reply":"2023-06-11T01:02:13.722253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append('/kaggle/input/pretrainedmodels/pretrainedmodels-0.7.4')\nsys.path.append('/kaggle/input/efficientnet-pytorch/EfficientNet-PyTorch-master')\nsys.path.append('/kaggle/input/timm-pytorch-image-models/pytorch-image-models-master')\nsys.path.append('/kaggle/input/segmentation-models-pytorch/segmentation_models.pytorch-master')\n\nimport segmentation_models_pytorch as smp","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:13.729105Z","iopub.execute_input":"2023-06-11T01:02:13.729439Z","iopub.status.idle":"2023-06-11T01:02:16.311831Z","shell.execute_reply.started":"2023-06-11T01:02:13.729409Z","shell.execute_reply":"2023-06-11T01:02:16.310731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append('/kaggle/input/ttach-lib/ttach-master')\nimport ttach as tta","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:16.315258Z","iopub.execute_input":"2023-06-11T01:02:16.316410Z","iopub.status.idle":"2023-06-11T01:02:16.390108Z","shell.execute_reply.started":"2023-06-11T01:02:16.316366Z","shell.execute_reply":"2023-06-11T01:02:16.389069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom torch.utils.data import DataLoader, Dataset\nimport cv2\nimport torch\nimport os\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:16.391696Z","iopub.execute_input":"2023-06-11T01:02:16.392150Z","iopub.status.idle":"2023-06-11T01:02:17.317435Z","shell.execute_reply.started":"2023-06-11T01:02:16.392109Z","shell.execute_reply":"2023-06-11T01:02:17.316334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## config","metadata":{}},{"cell_type":"code","source":"import os\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nclass CFG:\n    # ============== comp exp name =============\n    comp_name = 'vesuvius'  # comp名 モデルの出力先フォルダ名とか\n\n    # comp_dir_path = './'\n    comp_dir_path = '/kaggle/input/'\n    comp_folder_name = 'vesuvius-challenge-ink-detection'  # compのデータセットのフォルダ名\n    # comp_dataset_path = f'{comp_dir_path}datasets/{comp_folder_name}/'\n    comp_dataset_path = f'{comp_dir_path}{comp_folder_name}/'\n    \n    # exp_name = 'vesuvius_2d_1dpool_5fold_exp001'\n    # exp_name = 'vesuvius_2d_1dpool_5fold_exp004'\n    \n    # ============== pred target =============\n    target_size = 1\n\n    # ============== model cfg =============\n    \n    in_chans = 15*2 # 65\n    idxs = list(range(25-9, 40+6))\n\n    # ============== training cfg =============\n    size = 224\n    tile_size = 224\n    stride = tile_size // 4\n\n    batch_size = 16 # 32\n\n    # ============== ほぼ固定 =============\n\n    print_freq = 50\n    num_workers = 2\n\n    seed = 42\n\n    # ============== augmentation =============\n\n\n    valid_aug_list = [\n        A.Resize(size, size),\n        A.Normalize(\n            mean= [0] * in_chans,\n            std= [1] * in_chans\n        ),\n        ToTensorV2(transpose_mask=True),\n    ]\n","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:17.319031Z","iopub.execute_input":"2023-06-11T01:02:17.319430Z","iopub.status.idle":"2023-06-11T01:02:17.330079Z","shell.execute_reply.started":"2023-06-11T01:02:17.319390Z","shell.execute_reply":"2023-06-11T01:02:17.328795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IS_DEBUG = False\n# mode = 'train' if IS_DEBUG else 'test'\nTH = 0.5","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:17.332116Z","iopub.execute_input":"2023-06-11T01:02:17.332986Z","iopub.status.idle":"2023-06-11T01:02:17.344569Z","shell.execute_reply.started":"2023-06-11T01:02:17.332945Z","shell.execute_reply":"2023-06-11T01:02:17.343195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:17.346083Z","iopub.execute_input":"2023-06-11T01:02:17.347162Z","iopub.status.idle":"2023-06-11T01:02:17.421794Z","shell.execute_reply.started":"2023-06-11T01:02:17.347120Z","shell.execute_reply":"2023-06-11T01:02:17.420605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## helper","metadata":{}},{"cell_type":"code","source":"# ref.: https://www.kaggle.com/stainsby/fast-tested-rle\ndef rle(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.flatten()\n    # pixels = (pixels >= thr).astype(int)\n    \n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:17.428273Z","iopub.execute_input":"2023-06-11T01:02:17.428996Z","iopub.status.idle":"2023-06-11T01:02:17.437155Z","shell.execute_reply.started":"2023-06-11T01:02:17.428960Z","shell.execute_reply":"2023-06-11T01:02:17.435954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## dataset","metadata":{}},{"cell_type":"code","source":"def read_image(fragment_id):\n    images = []\n\n    # idxs = range(65)\n    mid = 65 // 2\n    start = mid - CFG.in_chans // 2\n    end = mid + CFG.in_chans // 2\n    if CFG.in_chans%2==1:\n        end+=1\n    idxs = range(start, end)\n\n    if CFG.idxs is not None:\n        idxs = CFG.idxs\n    \n    print(idxs)\n\n    for i in tqdm(idxs):\n        \n        image = cv2.imread(CFG.comp_dataset_path + f\"test/{fragment_id}/surface_volume/{i:02}.tif\", 0)\n\n        pad0 = (CFG.tile_size - image.shape[0] % CFG.tile_size)\n        pad1 = (CFG.tile_size - image.shape[1] % CFG.tile_size)\n\n        image = np.pad(image, [(0, pad0), (0, pad1)], constant_values=0)\n\n        images.append(image)\n    images = np.stack(images, axis=2)\n    \n    return images","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:17.438548Z","iopub.execute_input":"2023-06-11T01:02:17.438958Z","iopub.status.idle":"2023-06-11T01:02:17.450959Z","shell.execute_reply.started":"2023-06-11T01:02:17.438929Z","shell.execute_reply":"2023-06-11T01:02:17.449721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transforms(data, cfg):\n    if data == 'train':\n        aug = A.Compose(cfg.train_aug_list)\n    elif data == 'valid':\n        aug = A.Compose(cfg.valid_aug_list)\n\n    # print(aug)\n    return aug\n\nclass CustomDataset(Dataset):\n    def __init__(self, images, cfg, labels=None, transform=None):\n        self.images = images\n        self.cfg = cfg\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        # return len(self.xyxys)\n        return len(self.images)\n\n    def __getitem__(self, idx):\n        # x1, y1, x2, y2 = self.xyxys[idx]\n        image = self.images[idx]\n        data = self.transform(image=image)\n        image = data['image']\n\n        start_idx = 6\n        img_num = 21\n        end_idx = start_idx+img_num\n        image2 = image[start_idx:end_idx, ...]\n\n        return image, image2\n","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:17.452912Z","iopub.execute_input":"2023-06-11T01:02:17.453696Z","iopub.status.idle":"2023-06-11T01:02:17.464429Z","shell.execute_reply.started":"2023-06-11T01:02:17.453658Z","shell.execute_reply":"2023-06-11T01:02:17.463560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_test_dataset(fragment_id):\n    test_images = read_image(fragment_id)\n    \n    x1_list = list(range(0, test_images.shape[1]-CFG.tile_size+1, CFG.stride))\n    y1_list = list(range(0, test_images.shape[0]-CFG.tile_size+1, CFG.stride))\n    \n    test_images_list = []\n    xyxys = []\n    for y1 in y1_list:\n        for x1 in x1_list:\n            y2 = y1 + CFG.tile_size\n            x2 = x1 + CFG.tile_size\n            \n            if test_images[y1:y2, x1:x2].sum() != 0:\n                test_images_list.append(test_images[y1:y2, x1:x2])\n                xyxys.append((x1, y1, x2, y2))\n                \n    xyxys = np.stack(xyxys)\n            \n    test_dataset = CustomDataset(test_images_list, CFG, transform=get_transforms(data='valid', cfg=CFG))\n    \n    test_loader = DataLoader(test_dataset,\n                          batch_size=CFG.batch_size,\n                          shuffle=False,\n                          num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n    \n    return test_loader, xyxys","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:17.466027Z","iopub.execute_input":"2023-06-11T01:02:17.466726Z","iopub.status.idle":"2023-06-11T01:02:17.477976Z","shell.execute_reply.started":"2023-06-11T01:02:17.466664Z","shell.execute_reply":"2023-06-11T01:02:17.476962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## model","metadata":{}},{"cell_type":"code","source":"from typing import Optional, Union, List\n# sys.path.append('libs/segmentation_models.pytorch-master')\n\nimport segmentation_models_pytorch as smp\nfrom segmentation_models_pytorch.encoders import get_encoder\nfrom segmentation_models_pytorch.base import (\n    SegmentationModel,\n    SegmentationHead,\n    ClassificationHead,\n)\n# from .decoder import UnetDecoder\nfrom segmentation_models_pytorch.decoders.unet.decoder import UnetDecoder\nimport segmentation_models_pytorch.base.initialization as init\n\nclass CustomUnet(nn.Module):\n    \"\"\"Unet_ is a fully convolution neural network for image semantic segmentation. Consist of *encoder*\n    and *decoder* parts connected with *skip connections*. Encoder extract features of different spatial\n    resolution (skip connections) which are used by decoder to define accurate segmentation mask. Use *concatenation*\n    for fusing decoder blocks with skip connections.\n    Args:\n        encoder_name: Name of the classification model that will be used as an encoder (a.k.a backbone)\n            to extract features of different spatial resolution\n        encoder_depth: A number of stages used in encoder in range [3, 5]. Each stage generate features\n            two times smaller in spatial dimensions than previous one (e.g. for depth 0 we will have features\n            with shapes [(N, C, H, W),], for depth 1 - [(N, C, H, W), (N, C, H // 2, W // 2)] and so on).\n            Default is 5\n        encoder_weights: One of **None** (random initialization), **\"imagenet\"** (pre-training on ImageNet) and\n            other pretrained weights (see table with available weights for each encoder_name)\n        decoder_channels: List of integers which specify **in_channels** parameter for convolutions used in decoder.\n            Length of the list should be the same as **encoder_depth**\n        decoder_use_batchnorm: If **True**, BatchNorm2d layer between Conv2D and Activation layers\n            is used. If **\"inplace\"** InplaceABN will be used, allows to decrease memory consumption.\n            Available options are **True, False, \"inplace\"**\n        decoder_attention_type: Attention module used in decoder of the model. Available options are\n            **None** and **scse** (https://arxiv.org/abs/1808.08127).\n        in_channels: A number of input channels for the model, default is 3 (RGB images)\n        classes: A number of classes for output mask (or you can think as a number of channels of output mask)\n        activation: An activation function to apply after the final convolution layer.\n            Available options are **\"sigmoid\"**, **\"softmax\"**, **\"logsoftmax\"**, **\"tanh\"**, **\"identity\"**,\n                **callable** and **None**.\n            Default is **None**\n        aux_params: Dictionary with parameters of the auxiliary output (classification head). Auxiliary output is build\n            on top of encoder if **aux_params** is not **None** (default). Supported params:\n                - classes (int): A number of classes\n                - pooling (str): One of \"max\", \"avg\". Default is \"avg\"\n                - dropout (float): Dropout factor in [0, 1)\n                - activation (str): An activation function to apply \"sigmoid\"/\"softmax\"\n                    (could be **None** to return logits)\n    Returns:\n        ``torch.nn.Module``: Unet\n    .. _Unet:\n        https://arxiv.org/abs/1505.04597\n    \"\"\"\n\n    def __init__(\n        self,\n        encoder_name: str = \"resnet34\",\n        encoder_depth: int = 5,\n        encoder_weights: Optional[str] = \"imagenet\",\n        decoder_use_batchnorm: bool = True,\n        decoder_channels: List[int] = (256, 128, 64, 32, 16),\n        decoder_attention_type: Optional[str] = None,\n        in_channels: int = 3,\n        classes: int = 1,\n        activation: Optional[Union[str, callable]] = None,\n        aux_params: Optional[dict] = None,\n        img_total_num = None, # add\n        img_size = None, # add\n        seq_pool_type = 'encoder' # add\n    ):\n        super().__init__()\n\n        self.encoder = get_encoder(\n            encoder_name,\n            in_channels=in_channels,\n            depth=encoder_depth,\n            weights=encoder_weights,\n        )\n\n        self.decoder = UnetDecoder(\n            encoder_channels=self.encoder.out_channels,\n            decoder_channels=decoder_channels,\n            n_blocks=encoder_depth,\n            use_batchnorm=decoder_use_batchnorm,\n            center=True if encoder_name.startswith(\"vgg\") else False,\n            attention_type=decoder_attention_type,\n        )\n\n        self.segmentation_head = SegmentationHead(\n            in_channels=decoder_channels[-1], # avg+maxのときは、*2\n            out_channels=classes,\n            activation=activation,\n            kernel_size=3,\n        )\n\n        if aux_params is not None:\n            self.classification_head = ClassificationHead(in_channels=self.encoder.out_channels[-1], **aux_params)\n        else:\n            self.classification_head = None\n\n        self.name = \"u-{}\".format(encoder_name)\n        self.initialize()\n\n        # add\n        self.in_chans = in_channels\n        self.head_in_chans = decoder_channels[-1]\n        self.avg3d = nn.AdaptiveAvgPool3d(output_size=(None, None, 1))\n\n        self.img_total_num = img_total_num\n        self.img_size = img_size\n        self.seq_pool_type = seq_pool_type\n\n    def initialize(self):\n        init.initialize_decoder(self.decoder)\n        init.initialize_head(self.segmentation_head)\n        if self.classification_head is not None:\n            init.initialize_head(self.classification_head)\n\n    def check_input_shape(self, x):\n\n        h, w = x.shape[-2:]\n        output_stride = self.encoder.output_stride\n        if h % output_stride != 0 or w % output_stride != 0:\n            new_h = (h // output_stride + 1) * output_stride if h % output_stride != 0 else h\n            new_w = (w // output_stride + 1) * output_stride if w % output_stride != 0 else w\n            raise RuntimeError(\n                f\"Wrong input shape height={h}, width={w}. Expected image height and width \"\n                f\"divisible by {output_stride}. Consider pad your images to shape ({new_h}, {new_w}).\"\n            )\n\n    def forward(self, x):\n        \"\"\"Sequentially pass `x` trough model`s encoder, decoder and heads\"\"\"\n\n        self.check_input_shape(x)\n\n        # add\n        bs = x.shape[0]\n        seq_len = self.img_total_num // self.in_chans\n        x = x.view(bs * seq_len, self.in_chans, self.img_size, self.img_size)\n\n        features = self.encoder(x)\n\n        # add\n        if self.seq_pool_type == 'encoder':\n            features = [_feature.view(bs, seq_len, *_feature.shape[1:]).mean(1) for _feature in features]\n\n        decoder_output = self.decoder(*features)\n        \n        \n        if self.seq_pool_type == 'decoder':\n            decoder_output = decoder_output.view(bs, seq_len, self.head_in_chans, self.img_size, self.img_size)\n            # decoder_output = decoder_output.permute(0, 2, 3, 4, 1)\n            # decoder_output = self.avg3d(decoder_output).squeeze(-1)\n            decoder_output = decoder_output.mean(1)\n\n        masks = self.segmentation_head(decoder_output)\n\n        if self.classification_head is not None:\n            labels = self.classification_head(features[-1])\n            return masks, labels\n\n        return masks\n\n    @torch.no_grad()\n    def predict(self, x):\n        \"\"\"Inference method. Switch model to `eval` mode, call `.forward(x)` with `torch.no_grad()`\n        Args:\n            x: 4D torch tensor with shape (batch_size, channels, height, width)\n        Return:\n            prediction: 4D torch tensor with shape (batch_size, classes, height, width)\n        \"\"\"\n        if self.training:\n            self.eval()\n\n        x = self.forward(x)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:17.481851Z","iopub.execute_input":"2023-06-11T01:02:17.482172Z","iopub.status.idle":"2023-06-11T01:02:17.514367Z","shell.execute_reply.started":"2023-06-11T01:02:17.482142Z","shell.execute_reply":"2023-06-11T01:02:17.513287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ConvSilu(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size=3):\n        super(ConvSilu, self).__init__()\n        self.layer = nn.Sequential(\n            nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, padding=1),\n            nn.SiLU(inplace=True)\n        )\n    def forward(self, x):\n        return self.layer(x)\n\n\n\n\"\"\"\nhttps://www.kaggle.com/code/victorsd/2nd-place-inference/notebook?scriptVersionId=106240458\n\"\"\"\nimport torch.nn.functional as F\nimport timm\n\nclass UnetConvNeXt(nn.Module):\n    def __init__(self, name='resnet34', pretrained=True, in_chans=3, otp_size=1, decoder_filters=[32, 48, 64, 96, 128], \n                    img_total_num = None, # add\n                    img_size = None, # add\n                    seq_pool_type = 'decoder' # add\n                 ):\n        super(UnetConvNeXt, self).__init__()\n\n        encoder = timm.create_model(name, features_only=True, pretrained=pretrained, in_chans=in_chans)\n\n        encoder_filters = [f['num_chs'] for f in encoder.feature_info]\n\n        decoder_filters = decoder_filters\n\n        self.conv6 = ConvSilu(encoder_filters[-1], decoder_filters[-1])\n        self.conv6_2 = ConvSilu(decoder_filters[-1] + encoder_filters[-2], decoder_filters[-1])\n        self.conv7 = ConvSilu(decoder_filters[-1], decoder_filters[-2])\n        self.conv7_2 = ConvSilu(decoder_filters[-2] + encoder_filters[-3], decoder_filters[-2])\n        self.conv8 = ConvSilu(decoder_filters[-2], decoder_filters[-3])\n        self.conv8_2 = ConvSilu(decoder_filters[-3] + encoder_filters[-4], decoder_filters[-3])\n        self.conv9 = ConvSilu(decoder_filters[-3], decoder_filters[-4])\n\n        if len(encoder_filters) == 4:\n            self.conv9_2 = None\n        else:\n            self.conv9_2 = ConvSilu(decoder_filters[-4] + encoder_filters[-5], decoder_filters[-4])\n        \n        self.conv10 = ConvSilu(decoder_filters[-4], decoder_filters[-5])\n        \n        self.res = nn.Conv2d(decoder_filters[-5], otp_size, 1, stride=1, padding=0)\n\n        self.cls =  nn.Linear(encoder_filters[-1] * 2, 5)\n        self.pix_sz =  nn.Linear(encoder_filters[-1] * 2, 1)\n\n        self._initialize_weights()\n\n        self.encoder = encoder\n\n        # add\n        self.in_chans = in_chans\n        self.img_total_num = img_total_num\n        self.img_size = img_size\n        self.head_in_chans = decoder_filters[-5]\n\n        self.avg3d = nn.AdaptiveAvgPool3d(output_size=(None, None, 1))\n\n        self.seq_pool_type = seq_pool_type\n        \n    def forward(self, x):\n        # add\n        bs = x.shape[0]\n        seq_len = self.img_total_num // self.in_chans\n        x = x.view(bs * seq_len, self.in_chans, self.img_size, self.img_size)\n\n        # batch_size, C, H, W = x.shape\n\n        if self.conv9_2 is None:\n            enc2, enc3, enc4, enc5 = self.encoder(x)\n        else:\n            enc1, enc2, enc3, enc4, enc5 = self.encoder(x)\n            \n        if self.seq_pool_type == 'encoder':\n            enc2 = enc2.view(bs, seq_len, *enc2.shape[1:]).mean(1)\n            enc3 = enc3.view(bs, seq_len, *enc3.shape[1:]).mean(1)\n            enc4 = enc4.view(bs, seq_len, *enc4.shape[1:]).mean(1)\n            enc5 = enc5.view(bs, seq_len, *enc5.shape[1:]).mean(1)\n            \n        dec6 = self.conv6(F.interpolate(enc5, scale_factor=2))\n        dec6 = self.conv6_2(torch.cat([dec6, enc4\n                ], 1))\n\n        dec7 = self.conv7(F.interpolate(dec6, scale_factor=2))\n        dec7 = self.conv7_2(torch.cat([dec7, enc3\n                ], 1))\n        \n        dec8 = self.conv8(F.interpolate(dec7, scale_factor=2))\n        dec8 = self.conv8_2(torch.cat([dec8, enc2\n                ], 1))\n\n        dec9 = self.conv9(F.interpolate(dec8, scale_factor=2))\n\n        if self.conv9_2 is not None:\n            dec9 = self.conv9_2(torch.cat([dec9, \n                    enc1\n                    ], 1))\n        \n        # dec10 = self.conv10(dec9) # F.interpolate(dec9, scale_factor=2))\n        dec10 = self.conv10(F.interpolate(dec9, scale_factor=2))\n\n        \"\"\"\n        x1 = torch.cat([F.adaptive_avg_pool2d(enc5, output_size=1).view(batch_size, -1), \n                        F.adaptive_max_pool2d(enc5, output_size=1).view(batch_size, -1)], 1)\n\n        # x1 = F.dropout(x1, p=0.3, training=self.training)\n        organ_cls = self.cls(x1)\n        pixel_size = self.pix_sz(x1)\n\n        return self.res(dec10), organ_cls, pixel_size\n        \"\"\"\n\n        # add\n        if self.seq_pool_type == 'decoder':\n            dec10 = dec10.view(bs, seq_len, self.head_in_chans, self.img_size, self.img_size)\n            # dec10 = dec10.permute(0, 2, 3, 4, 1)\n            # dec10 = self.avg3d(dec10).squeeze(-1)\n            dec10 = dec10.mean(1)\n\n        return self.res(dec10)\n\n\n    def _initialize_weights(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d) or isinstance(m, nn.Linear):\n                m.weight.data = nn.init.kaiming_normal_(m.weight.data)\n                if m.bias is not None:\n                    m.bias.data.zero_()\n            elif isinstance(m, nn.BatchNorm2d):\n                m.weight.data.fill_(1)\n                m.bias.data.zero_()","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:17.516077Z","iopub.execute_input":"2023-06-11T01:02:17.516490Z","iopub.status.idle":"2023-06-11T01:02:17.552960Z","shell.execute_reply.started":"2023-06-11T01:02:17.516452Z","shell.execute_reply":"2023-06-11T01:02:17.551533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EnsembleModel:\n    def __init__(self):\n        self.models1 = []\n        self.models2 = []\n\n    def __call__(self, x1, x2):\n        outputs = []\n        for model in self.models1:\n            outputs.append(torch.sigmoid(model(x1)).to('cpu').numpy())\n\n        for model in self.models2:\n            outputs.append(torch.sigmoid(model(x2)).to('cpu').numpy())\n\n        avg_preds = np.mean(outputs, axis=0)\n\n        return avg_preds\n\n    def add_model(self, model, mode):\n        if mode == 1:\n            self.models1.append(model)\n        elif mode == 2:\n            self.models2.append(model)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:17.554623Z","iopub.execute_input":"2023-06-11T01:02:17.555014Z","iopub.status.idle":"2023-06-11T01:02:17.568361Z","shell.execute_reply.started":"2023-06-11T01:02:17.554975Z","shell.execute_reply":"2023-06-11T01:02:17.567185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = EnsembleModel()","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:17.570004Z","iopub.execute_input":"2023-06-11T01:02:17.570898Z","iopub.status.idle":"2023-06-11T01:02:17.579655Z","shell.execute_reply.started":"2023-06-11T01:02:17.570866Z","shell.execute_reply":"2023-06-11T01:02:17.578245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inf_weight = 'last'\n","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:17.580965Z","iopub.execute_input":"2023-06-11T01:02:17.581463Z","iopub.status.idle":"2023-06-11T01:02:17.589983Z","shell.execute_reply.started":"2023-06-11T01:02:17.581406Z","shell.execute_reply":"2023-06-11T01:02:17.588960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nexp_name = 'vesuvius_final_exp006'\nmodel_name = 'Unet'\nbackbone = 'se_resnext50_32x4d'\nin_chans = 3\nimg_total_num = 15*2\nmode = 1\nseq_pool_type = 'encoder'\n\nfor fold in [0,1,2,3,4]:\n    _model = CustomUnet(\n            encoder_name=backbone, \n            encoder_weights=None,\n            in_channels=in_chans, # cfg.in_chans\n            classes=1,\n            activation=None,\n            img_total_num = img_total_num,\n            img_size = CFG.size,\n            seq_pool_type = seq_pool_type\n        )\n    # _model = build_model(CFG, weight=None)\n    _model.to(device)\n\n    model_path = f'/kaggle/input/vesuvius-models/{exp_name}/vesuvius-models/{model_name}_fold{fold}_{inf_weight}.pth'\n    state = torch.load(model_path)['model']\n    _model.load_state_dict(state)\n    _model.eval()\n\n    \"\"\"\n    _model = tta.SegmentationTTAWrapper(\n        _model, tta.aliases.d4_transform(), merge_mode='mean')\n    \"\"\"\n    \"\"\"\n    transforms = tta.Compose(\n        [\n            # tta.HorizontalFlip(),\n            # tta.VerticalFlip(),\n            tta.Rotate90(angles=[0, 90, 180, 270]),    \n        ]\n    )\n\n    _model = tta.SegmentationTTAWrapper(\n        _model, transforms, merge_mode='mean')\n    \"\"\"\n    model.add_model(_model, mode)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:17.591365Z","iopub.execute_input":"2023-06-11T01:02:17.592483Z","iopub.status.idle":"2023-06-11T01:02:34.388077Z","shell.execute_reply.started":"2023-06-11T01:02:17.592437Z","shell.execute_reply":"2023-06-11T01:02:34.386948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nexp_name = 'vesuvius-paperspace-exp003'\nmodel_name = 'Unet'\nbackbone = 'tu-eca_nfnet_l1'\nin_chans = 3\nimg_total_num = 15*2\nmode = 1\nseq_pool_type = 'encoder'\n\nfor fold in [0,1,2,3,4]:\n    _model = CustomUnet(\n            encoder_name=backbone, \n            encoder_weights=None,\n            in_channels=in_chans, # cfg.in_chans\n            classes=1,\n            activation=None,\n            img_total_num = img_total_num,\n            img_size = CFG.size,\n            seq_pool_type = seq_pool_type\n        )\n    # _model = build_model(CFG, weight=None)\n    _model.to(device)\n\n    # model_path = f'/kaggle/input/vesuvius-models/{exp_name}/vesuvius-models/{model_name}_fold{fold}_{inf_weight}.pth'\n    model_path = f'/kaggle/input/{exp_name}/vesuvius-models/{model_name}_fold{fold}_{inf_weight}.pth'\n    state = torch.load(model_path)['model']\n    _model.load_state_dict(state)\n    _model.eval()\n\n    \"\"\"\n    _model = tta.SegmentationTTAWrapper(\n        _model, tta.aliases.d4_transform(), merge_mode='mean')\n    \"\"\"\n    \"\"\"\n    transforms = tta.Compose(\n        [\n            # tta.HorizontalFlip(),\n            # tta.VerticalFlip(),\n            tta.Rotate90(angles=[0, 90, 180, 270]),    \n        ]\n    )\n    _model = tta.SegmentationTTAWrapper(\n        _model, transforms, merge_mode='mean')\n    \"\"\"\n    model.add_model(_model, mode)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:34.392926Z","iopub.execute_input":"2023-06-11T01:02:34.395759Z","iopub.status.idle":"2023-06-11T01:02:52.338435Z","shell.execute_reply.started":"2023-06-11T01:02:34.395715Z","shell.execute_reply":"2023-06-11T01:02:52.337337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"exp_name = 'vesuvius_final_exp001'\nmodel_name = 'Unet'\nbackbone = 'tu-tf_efficientnetv2_m'\nin_chans = 3\nimg_total_num = 15*2\nmode = 1\nseq_pool_type = 'encoder'\n\nfor fold in [0,1,2,3,4]:\n    _model = CustomUnet(\n            encoder_name=backbone, \n            encoder_weights=None,\n            in_channels=in_chans, # cfg.in_chans\n            classes=1,\n            activation=None,\n            img_total_num = img_total_num,\n            img_size = CFG.size,\n            seq_pool_type = seq_pool_type\n        )\n    # _model = build_model(CFG, weight=None)\n    _model.to(device)\n\n    model_path = f'/kaggle/input/vesuvius-models/{exp_name}/vesuvius-models/{model_name}_fold{fold}_{inf_weight}.pth'\n    state = torch.load(model_path)['model']\n    _model.load_state_dict(state)\n    _model.eval()\n\n    \"\"\"\n    _model = tta.SegmentationTTAWrapper(\n        _model, tta.aliases.d4_transform(), merge_mode='mean')\n    \"\"\"\n    \"\"\"\n    transforms = tta.Compose(\n        [\n            # tta.HorizontalFlip(),\n            # tta.VerticalFlip(),\n            tta.Rotate90(angles=[0, 90, 180, 270]),    \n        ]\n    )\n    _model = tta.SegmentationTTAWrapper(\n        _model, transforms, merge_mode='mean')\n    \"\"\"\n\n    model.add_model(_model, mode)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:52.340146Z","iopub.execute_input":"2023-06-11T01:02:52.340878Z","iopub.status.idle":"2023-06-11T01:03:08.151722Z","shell.execute_reply.started":"2023-06-11T01:02:52.340833Z","shell.execute_reply":"2023-06-11T01:03:08.150639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nexp_name = 'vesuvius_final_exp002'\nmodel_name = 'Unet'\nbackbone = 'convnext_small_in22ft1k'\nin_chans = 3\nimg_total_num = 15*2\nmode = 1\nseq_pool_type = 'encoder'\n\nfor fold in [0,1,2,3,4]:\n    _model = UnetConvNeXt(\n            pretrained=False,\n            name=backbone, \n            in_chans=in_chans, # cfg.in_chans\n            img_total_num = img_total_num,\n            img_size = CFG.size,\n            seq_pool_type = seq_pool_type\n        )\n    # _model = build_model(CFG, weight=None)\n    _model.to(device)\n\n    model_path = f'/kaggle/input/vesuvius-models/{exp_name}/vesuvius-models/{model_name}_fold{fold}_{inf_weight}.pth'\n    state = torch.load(model_path)['model']\n    _model.load_state_dict(state)\n    _model.eval()\n\n    \"\"\"\n    _model = tta.SegmentationTTAWrapper(\n        _model, tta.aliases.d4_transform(), merge_mode='mean')\n    \"\"\"\n    \"\"\"\n    transforms = tta.Compose(\n        [\n            # tta.HorizontalFlip(),\n            # tta.VerticalFlip(),\n            tta.Rotate90(angles=[0, 90, 180, 270]),    \n        ]\n    )\n    _model = tta.SegmentationTTAWrapper(\n        _model, transforms, merge_mode='mean')\n    \"\"\"\n    model.add_model(_model, mode)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:03:08.153084Z","iopub.execute_input":"2023-06-11T01:03:08.153515Z","iopub.status.idle":"2023-06-11T01:03:21.674490Z","shell.execute_reply.started":"2023-06-11T01:03:08.153471Z","shell.execute_reply":"2023-06-11T01:03:21.673372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"exp_name = 'vesuvius_final_exp005'\nmodel_name = 'Unet'\nbackbone = 'se_resnext101_32x4d'\nin_chans = 3\nimg_total_num = 21\nmode = 2\nseq_pool_type = 'encoder'\n\nfor fold in [0,1,2,3,4]:\n    _model = CustomUnet(\n            encoder_name=backbone, \n            encoder_weights=None,\n            in_channels=in_chans, # cfg.in_chans\n            classes=1,\n            activation=None,\n            img_total_num = img_total_num,\n            img_size = CFG.size,\n            seq_pool_type = seq_pool_type\n        )\n    # _model = build_model(CFG, weight=None)\n    _model.to(device)\n\n    model_path = f'/kaggle/input/vesuvius-models/{exp_name}/vesuvius-models/{model_name}_fold{fold}_{inf_weight}.pth'\n    state = torch.load(model_path)['model']\n    _model.load_state_dict(state)\n    _model.eval()\n\n    \"\"\"\n    _model = tta.SegmentationTTAWrapper(\n        _model, tta.aliases.d4_transform(), merge_mode='mean')\n    \"\"\"\n    \"\"\"\n    transforms = tta.Compose(\n        [\n            # tta.HorizontalFlip(),\n            # tta.VerticalFlip(),\n            tta.Rotate90(angles=[0, 90, 180, 270]),    \n        ]\n    )\n    _model = tta.SegmentationTTAWrapper(\n        _model, transforms, merge_mode='mean')\n    \"\"\"\n    model.add_model(_model, mode)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:03:21.678894Z","iopub.execute_input":"2023-06-11T01:03:21.679210Z","iopub.status.idle":"2023-06-11T01:03:33.766913Z","shell.execute_reply.started":"2023-06-11T01:03:21.679179Z","shell.execute_reply":"2023-06-11T01:03:33.765615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nexp_name = 'vesuvius-paperspace-exp008'\nmodel_name = 'Unet'\nbackbone = 'convnext_base_in22ft1k'\nin_chans = 3\nimg_total_num = 21\nmode = 2\nseq_pool_type = 'encoder'\n\nfor fold in [0,1,2,3,4]:\n    _model = UnetConvNeXt(\n            pretrained=False,\n            name=backbone, \n            in_chans=in_chans, # cfg.in_chans\n            img_total_num = img_total_num,\n            img_size = CFG.size,\n            seq_pool_type = seq_pool_type\n        )\n    # _model = build_model(CFG, weight=None)\n    _model.to(device)\n\n    model_path = f'/kaggle/input/{exp_name}/vesuvius-models/{model_name}_fold{fold}_{inf_weight}.pth'\n    state = torch.load(model_path)['model']\n    _model.load_state_dict(state)\n    _model.eval()\n\n    \"\"\"\n    _model = tta.SegmentationTTAWrapper(\n        _model, tta.aliases.d4_transform(), merge_mode='mean')\n    \"\"\"\n    \"\"\"\n    transforms = tta.Compose(\n        [\n            # tta.HorizontalFlip(),\n            # tta.VerticalFlip(),\n            tta.Rotate90(angles=[0, 90, 180, 270]),    \n        ]\n    )\n    _model = tta.SegmentationTTAWrapper(\n        _model, transforms, merge_mode='mean')\n    \"\"\"\n    model.add_model(_model, mode)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:03:33.768699Z","iopub.execute_input":"2023-06-11T01:03:33.769116Z","iopub.status.idle":"2023-06-11T01:04:02.353033Z","shell.execute_reply.started":"2023-06-11T01:03:33.769070Z","shell.execute_reply":"2023-06-11T01:04:02.351933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:04:02.355480Z","iopub.execute_input":"2023-06-11T01:04:02.356119Z","iopub.status.idle":"2023-06-11T01:04:03.423178Z","shell.execute_reply.started":"2023-06-11T01:04:02.356075Z","shell.execute_reply":"2023-06-11T01:04:03.421814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fragment_ids = sorted(os.listdir(CFG.comp_dataset_path + 'test'))","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:04:03.425570Z","iopub.execute_input":"2023-06-11T01:04:03.425975Z","iopub.status.idle":"2023-06-11T01:04:03.433394Z","shell.execute_reply.started":"2023-06-11T01:04:03.425929Z","shell.execute_reply":"2023-06-11T01:04:03.431122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fragment_ids","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:04:03.435394Z","iopub.execute_input":"2023-06-11T01:04:03.436759Z","iopub.status.idle":"2023-06-11T01:04:03.445476Z","shell.execute_reply.started":"2023-06-11T01:04:03.436719Z","shell.execute_reply":"2023-06-11T01:04:03.444369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## main","metadata":{}},{"cell_type":"code","source":"results = []\nfor fragment_id in fragment_ids:\n    \n    test_loader, xyxys = make_test_dataset(fragment_id)\n    \n    binary_mask = cv2.imread(CFG.comp_dataset_path + f\"test/{fragment_id}/mask.png\", 0)\n    binary_mask = (binary_mask / 255).astype(int)\n    \n    ori_h = binary_mask.shape[0]\n    ori_w = binary_mask.shape[1]\n    # mask = mask / 255\n\n    pad0 = (CFG.tile_size - binary_mask.shape[0] % CFG.tile_size)\n    pad1 = (CFG.tile_size - binary_mask.shape[1] % CFG.tile_size)\n\n    binary_mask = np.pad(binary_mask, [(0, pad0), (0, pad1)], constant_values=0)\n    \n    mask_pred = np.zeros(binary_mask.shape)\n    mask_count = np.zeros(binary_mask.shape)\n\n    for step, (images1, images2) in tqdm(enumerate(test_loader), total=len(test_loader)):\n        images1 = images1.to(device)\n        images2 = images2.to(device)\n        batch_size = images1.size(0)\n        \n        with torch.no_grad():\n            y_preds = model(images1, images2)\n\n        start_idx = step*CFG.batch_size\n        end_idx = start_idx + batch_size\n        for i, (x1, y1, x2, y2) in enumerate(xyxys[start_idx:end_idx]):\n            mask_pred[y1:y2, x1:x2] += y_preds[i].squeeze(0)\n            mask_count[y1:y2, x1:x2] += np.ones((CFG.tile_size, CFG.tile_size))\n\n    print(f'mask_count_min: {mask_count.min()}')\n\n    mask_count[mask_count == 0] = 1\n    \n    plt.imshow(mask_count)\n    plt.show()\n    \n\n    mask_pred /= mask_count\n    \n    mask_pred = mask_pred[:ori_h, :ori_w]\n    binary_mask = binary_mask[:ori_h, :ori_w]\n\n    plt.imshow(mask_pred)\n    plt.show()\n\n    plt.hist(mask_pred[mask_pred>0].flatten(), bins=20)\n    plt.show()\n    \n    mask_pred = (mask_pred >= TH).astype(int)\n    mask_pred *= binary_mask\n    \n    plt.imshow(mask_pred)\n    plt.show()\n    \n    \n    inklabels_rle = rle(mask_pred)\n    \n    results.append((fragment_id, inklabels_rle))\n    \n\n    del mask_pred, mask_count\n    del test_loader\n    \n    gc.collect()\n    torch.cuda.empty_cache()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:04:03.452966Z","iopub.execute_input":"2023-06-11T01:04:03.453254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## submission","metadata":{}},{"cell_type":"code","source":"sub = pd.DataFrame(results, columns=['Id', 'Predicted'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv(CFG.comp_dataset_path + 'sample_submission.csv')\nsample_sub = pd.merge(sample_sub[['Id']], sub, on='Id', how='left')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}