{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, accuracy_score, f1_score, log_loss\nimport pickle\nfrom torch.utils.data import DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nimport warnings\nimport sys\nimport pandas as pd\nimport os\nimport gc\nimport sys\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\nimport cv2\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport argparse\nimport importlib\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD, AdamW\n\nimport datetime\nimport wandb","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:46:28.527073Z","iopub.execute_input":"2023-06-13T18:46:28.528108Z","iopub.status.idle":"2023-06-13T18:46:28.538560Z","shell.execute_reply.started":"2023-06-13T18:46:28.528065Z","shell.execute_reply":"2023-06-13T18:46:28.537432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/einops/einops-0.6.1-py3-none-any.whl\n!pip install /kaggle/input/monai-packages/monai-1.1.0-202212191849-py3-none-any.whl[\"einops\"]","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:46:28.544334Z","iopub.execute_input":"2023-06-13T18:46:28.544862Z","iopub.status.idle":"2023-06-13T18:47:32.141250Z","shell.execute_reply.started":"2023-06-13T18:46:28.544832Z","shell.execute_reply":"2023-06-13T18:47:32.139766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append('/kaggle/input/pretrainedmodels/pretrainedmodels-0.7.4')\nsys.path.append('/kaggle/input/efficientnet-pytorch/EfficientNet-PyTorch-master')\nsys.path.append('/kaggle/input/timm-pytorch-image-models/pytorch-image-models-master')\nsys.path.append('/kaggle/input/segmentation-models-pytorch/segmentation_models.pytorch-master')\nsys.path.append('/kaggle/input/unet3d/pytorch3dunet/pytorch3dunet')\nsys.path.append('/kaggle/input/unet3d/pytorch3dunet')\nsys.path.append('/kaggle/input/unet3d/')\n\nimport segmentation_models_pytorch as smp\nfrom unet3d.model import get_model\nfrom unetr import UNETR","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.144522Z","iopub.execute_input":"2023-06-13T18:47:32.144973Z","iopub.status.idle":"2023-06-13T18:47:32.155026Z","shell.execute_reply.started":"2023-06-13T18:47:32.144921Z","shell.execute_reply":"2023-06-13T18:47:32.153989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom torch.utils.data import DataLoader, Dataset\nimport cv2\nimport torch\nimport os\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.158614Z","iopub.execute_input":"2023-06-13T18:47:32.159168Z","iopub.status.idle":"2023-06-13T18:47:32.165854Z","shell.execute_reply.started":"2023-06-13T18:47:32.159134Z","shell.execute_reply":"2023-06-13T18:47:32.164735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## config","metadata":{}},{"cell_type":"code","source":"import os\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nclass CFG:\n    # ============== comp exp name =============\n    comp_name = 'vesuvius'\n\n    # comp_dir_path = './'\n    comp_dir_path = '/kaggle/input/'\n    comp_folder_name = 'vesuvius-challenge-ink-detection'\n    # comp_dataset_path = f'{comp_dir_path}datasets/{comp_folder_name}/'\n    comp_dataset_path = f'{comp_dir_path}{comp_folder_name}/'\n    \n    exp_name = '3d_unet_subv2'\n\n    # ============== pred target =============\n    target_size = 1\n\n    # ============== model cfg =============\n    model_name = '3d_unet_segformer'\n    backbone = 'None'\n#     backbone = 'se_resnext50_32x4d'\n\n    in_chans = 16\n    # ============== training cfg =============\n    size = 1024\n    tile_size = 1024\n    stride = tile_size // 4\n\n    batch_size = 3 # 32\n    use_amp = True\n\n    scheduler = 'GradualWarmupSchedulerV2'\n    # scheduler = 'CosineAnnealingLR'\n    epochs = 15\n\n    warmup_factor = 10\n    lr = 1e-4 / warmup_factor\n\n    # ============== fold =============\n    valid_id = 2\n\n    objective_cv = 'binary'  # 'binary', 'multiclass', 'regression'\n    metric_direction = 'maximize'  # maximize, 'minimize'\n    # metrics = 'dice_coef'\n\n    # ============== fixed =============\n    pretrained = True\n    inf_weight = 'best'  # 'best'\n\n    min_lr = 1e-6\n    weight_decay = 1e-6\n    max_grad_norm = 1000\n\n    print_freq = 50\n    num_workers = 2\n\n    seed = 42\n\n    # ============== augmentation =============\n    train_aug_list = [\n        # A.RandomResizedCrop(\n        #     size, size, scale=(0.85, 1.0)),\n        A.Resize(size, size),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.RandomBrightnessContrast(p=0.75),\n        A.ShiftScaleRotate(p=0.75),\n        A.OneOf([\n                A.GaussNoise(var_limit=[10, 50]),\n                A.GaussianBlur(),\n                A.MotionBlur(),\n                ], p=0.4),\n        A.GridDistortion(num_steps=5, distort_limit=0.3, p=0.5),\n        A.CoarseDropout(max_holes=1, max_width=int(size * 0.3), max_height=int(size * 0.3), \n                        mask_fill_value=0, p=0.5),\n        # A.Cutout(max_h_size=int(size * 0.6),\n        #          max_w_size=int(size * 0.6), num_holes=1, p=1.0),\n        A.Normalize(\n            mean= [0] * in_chans,\n            std= [1] * in_chans\n        ),\n        ToTensorV2(transpose_mask=True),\n    ]\n\n    valid_aug_list = [\n        A.Resize(size, size),\n        A.Normalize(\n            mean= [0] * in_chans,\n            std= [1] * in_chans\n        ),\n        ToTensorV2(transpose_mask=True),\n    ]\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.167579Z","iopub.execute_input":"2023-06-13T18:47:32.167976Z","iopub.status.idle":"2023-06-13T18:47:32.185820Z","shell.execute_reply.started":"2023-06-13T18:47:32.167939Z","shell.execute_reply":"2023-06-13T18:47:32.184563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_DEBUG = False\nmode = 'train' if IS_DEBUG else 'test'\nTH = 0.5","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.190334Z","iopub.execute_input":"2023-06-13T18:47:32.190646Z","iopub.status.idle":"2023-06-13T18:47:32.197655Z","shell.execute_reply.started":"2023-06-13T18:47:32.190619Z","shell.execute_reply":"2023-06-13T18:47:32.196588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.199355Z","iopub.execute_input":"2023-06-13T18:47:32.199735Z","iopub.status.idle":"2023-06-13T18:47:32.207087Z","shell.execute_reply.started":"2023-06-13T18:47:32.199698Z","shell.execute_reply":"2023-06-13T18:47:32.206034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## helper","metadata":{}},{"cell_type":"code","source":"# ref.: https://www.kaggle.com/stainsby/fast-tested-rle\ndef rle(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.flatten()\n    # pixels = (pixels >= thr).astype(int)\n    \n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.208718Z","iopub.execute_input":"2023-06-13T18:47:32.209518Z","iopub.status.idle":"2023-06-13T18:47:32.217509Z","shell.execute_reply.started":"2023-06-13T18:47:32.209443Z","shell.execute_reply":"2023-06-13T18:47:32.216506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## dataset","metadata":{}},{"cell_type":"code","source":"def read_image(fragment_id):\n    images = []\n\n#     idxs = range(65)\n    mid = 65 // 2\n    start = mid - CFG.in_chans // 2\n    end = mid + CFG.in_chans // 2\n    idxs = range(start, end)\n\n    for i in tqdm(idxs):\n        \n        image = cv2.imread(CFG.comp_dataset_path + f\"{mode}/{fragment_id}/surface_volume/{i:02}.tif\", 0)\n\n        pad0 = (CFG.tile_size - image.shape[0] % CFG.tile_size)\n        pad1 = (CFG.tile_size - image.shape[1] % CFG.tile_size)\n\n        image = np.pad(image, [(0, pad0), (0, pad1)], constant_values=0)\n\n        images.append(image)\n    images = np.stack(images, axis=2)\n    \n    return images","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.218976Z","iopub.execute_input":"2023-06-13T18:47:32.219907Z","iopub.status.idle":"2023-06-13T18:47:32.229533Z","shell.execute_reply.started":"2023-06-13T18:47:32.219868Z","shell.execute_reply":"2023-06-13T18:47:32.228425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transforms(data, cfg):\n    if data == 'train':\n        aug = A.Compose(cfg.train_aug_list)\n    elif data == 'valid':\n        aug = A.Compose(cfg.valid_aug_list)\n\n    # print(aug)\n    return aug\n\nclass CustomDataset(Dataset):\n    def __init__(self, images, cfg, labels=None, transform=None):\n        self.images = np.array(images)\n        self.cfg = cfg\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        # return len(self.xyxys)\n        return len(self.images)\n\n    def __getitem__(self, idx):\n        image = np.load(self.images[idx])\n        data = self.transform(image=image)\n        image = data['image']\n        return image[None, :, :, :]\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.230920Z","iopub.execute_input":"2023-06-13T18:47:32.232079Z","iopub.status.idle":"2023-06-13T18:47:32.243334Z","shell.execute_reply.started":"2023-06-13T18:47:32.232038Z","shell.execute_reply":"2023-06-13T18:47:32.242350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_test_dataset(fragment_id):\n    test_images = read_image(fragment_id)\n    \n    x1_list = list(range(0, test_images.shape[1]-CFG.tile_size+1, CFG.stride))\n    y1_list = list(range(0, test_images.shape[0]-CFG.tile_size+1, CFG.stride))\n    \n    test_images_list = []\n    xyxys = []\n    for y1 in y1_list:\n        for x1 in x1_list:\n            y2 = y1 + CFG.tile_size\n            x2 = x1 + CFG.tile_size\n            if test_images[y1:y2, x1:x2].max() != 0:\n                if not os.path.exists(f\"{x1}_{y1}_{x2}_{y2}.npy\"):\n                    np.save(f\"{x1}_{y1}_{x2}_{y2}.npy\", test_images[y1:y2, x1:x2])\n                test_images_list.append(f\"{x1}_{y1}_{x2}_{y2}.npy\")\n                xyxys.append((x1, y1, x2, y2))\n    del test_images\n    gc.collect()\n    xyxys = np.stack(xyxys)\n            \n    test_dataset = CustomDataset(test_images_list, CFG, transform=get_transforms(data='valid', cfg=CFG))\n    \n    test_loader = DataLoader(test_dataset,\n                          batch_size=CFG.batch_size,\n                          shuffle=False,\n                          num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n    \n    return test_loader, xyxys","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.244604Z","iopub.execute_input":"2023-06-13T18:47:32.244894Z","iopub.status.idle":"2023-06-13T18:47:32.259416Z","shell.execute_reply.started":"2023-06-13T18:47:32.244867Z","shell.execute_reply":"2023-06-13T18:47:32.258268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## model","metadata":{}},{"cell_type":"code","source":"from transformers import SegformerForSemanticSegmentation, SegformerModel, SegformerConfig","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.260989Z","iopub.execute_input":"2023-06-13T18:47:32.261641Z","iopub.status.idle":"2023-06-13T18:47:32.269214Z","shell.execute_reply.started":"2023-06-13T18:47:32.261603Z","shell.execute_reply":"2023-06-13T18:47:32.268128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_3d_segformer_b1_config = SegformerConfig(**{\n  \"architectures\": [\n    \"SegformerForImageClassification\"\n  ],\n  \"attention_probs_dropout_prob\": 0.0,\n  \"classifier_dropout_prob\": 0.1,\n  \"decoder_hidden_size\": 256,\n  \"depths\": [\n    2,\n    2,\n    2,\n    2\n  ],\n  \"downsampling_rates\": [\n    1,\n    4,\n    8,\n    16\n  ],\n  \"drop_path_rate\": 0.1,\n  \"hidden_act\": \"gelu\",\n  \"hidden_dropout_prob\": 0.0,\n  \"hidden_sizes\": [\n    64,\n    128,\n    320,\n    512\n  ],\n  \"image_size\": 224,\n  \"initializer_range\": 0.02,\n  \"layer_norm_eps\": 1e-06,\n  \"mlp_ratios\": [\n    4,\n    4,\n    4,\n    4\n  ],\n  \"model_type\": \"segformer\",\n  \"num_attention_heads\": [\n    1,\n    2,\n    5,\n    8\n  ],\n  \"num_channels\": 32,\n  \"num_encoder_blocks\": 4,\n  \"patch_sizes\": [\n    7,\n    3,\n    3,\n    3\n  ],\n  \"sr_ratios\": [\n    8,\n    4,\n    2,\n    1\n  ],\n  \"strides\": [\n    4,\n    2,\n    2,\n    2\n  ],\n  \"torch_dtype\": \"float32\",\n  \"transformers_version\": \"4.12.0.dev0\",\n  \"num_labels\":1,\n})","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.270680Z","iopub.execute_input":"2023-06-13T18:47:32.271323Z","iopub.status.idle":"2023-06-13T18:47:32.281294Z","shell.execute_reply.started":"2023-06-13T18:47:32.271284Z","shell.execute_reply":"2023-06-13T18:47:32.280531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_3d_segformer_b2_config = SegformerConfig(**{\n  \"architectures\": [\n    \"SegformerForImageClassification\"\n  ],\n  \"attention_probs_dropout_prob\": 0.0,\n  \"classifier_dropout_prob\": 0.1,\n  \"decoder_hidden_size\": 768,\n  \"depths\": [\n    3,\n    4,\n    6,\n    3\n  ],\n  \"downsampling_rates\": [\n    1,\n    4,\n    8,\n    16\n  ],\n  \"drop_path_rate\": 0.1,\n  \"hidden_act\": \"gelu\",\n  \"hidden_dropout_prob\": 0.0,\n  \"hidden_sizes\": [\n    64,\n    128,\n    320,\n    512\n  ],\n  \"image_size\": 224,\n  \"initializer_range\": 0.02,\n  \"layer_norm_eps\": 1e-06,\n  \"mlp_ratios\": [\n    4,\n    4,\n    4,\n    4\n  ],\n  \"model_type\": \"segformer\",\n  \"num_attention_heads\": [\n    1,\n    2,\n    5,\n    8\n  ],\n  \"num_channels\": 32,\n  \"num_encoder_blocks\": 4,\n  \"patch_sizes\": [\n    7,\n    3,\n    3,\n    3\n  ],\n  \"sr_ratios\": [\n    8,\n    4,\n    2,\n    1\n  ],\n  \"strides\": [\n    4,\n    2,\n    2,\n    2\n  ],\n  \"torch_dtype\": \"float32\",\n  \"transformers_version\": \"4.12.0.dev0\",\n  \"num_labels\":1\n})","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.282702Z","iopub.execute_input":"2023-06-13T18:47:32.283346Z","iopub.status.idle":"2023-06-13T18:47:32.293631Z","shell.execute_reply.started":"2023-06-13T18:47:32.283308Z","shell.execute_reply":"2023-06-13T18:47:32.292817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_3d_segformer_b4_config = SegformerConfig(**{\n  \"architectures\": [\n    \"SegformerForImageClassification\"\n  ],\n  \"attention_probs_dropout_prob\": 0.0,\n  \"classifier_dropout_prob\": 0.1,\n  \"decoder_hidden_size\": 768,\n  \"depths\": [\n    3,\n    8,\n    27,\n    3\n  ],\n  \"downsampling_rates\": [\n    1,\n    4,\n    8,\n    16\n  ],\n  \"drop_path_rate\": 0.1,\n  \"hidden_act\": \"gelu\",\n  \"hidden_dropout_prob\": 0.0,\n  \"hidden_sizes\": [\n    64,\n    128,\n    320,\n    512\n  ],\n  \"image_size\": 224,\n  \"initializer_range\": 0.02,\n  \"layer_norm_eps\": 1e-06,\n  \"mlp_ratios\": [\n    4,\n    4,\n    4,\n    4\n  ],\n  \"model_type\": \"segformer\",\n  \"num_attention_heads\": [\n    1,\n    2,\n    5,\n    8\n  ],\n  \"num_channels\": 32,\n  \"num_encoder_blocks\": 4,\n  \"patch_sizes\": [\n    7,\n    3,\n    3,\n    3\n  ],\n  \"sr_ratios\": [\n    8,\n    4,\n    2,\n    1\n  ],\n  \"strides\": [\n    4,\n    2,\n    2,\n    2\n  ],\n  \"torch_dtype\": \"float32\",\n  \"transformers_version\": \"4.12.0.dev0\",\n  \"num_labels\":1\n})","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.299110Z","iopub.execute_input":"2023-06-13T18:47:32.299668Z","iopub.status.idle":"2023-06-13T18:47:32.309271Z","shell.execute_reply.started":"2023-06-13T18:47:32.299627Z","shell.execute_reply":"2023-06-13T18:47:32.308261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_3d_segformer_b5_config = SegformerConfig(**{\n  \"architectures\": [\n    \"SegformerForImageClassification\"\n  ],\n  \"attention_probs_dropout_prob\": 0.0,\n  \"classifier_dropout_prob\": 0.1,\n  \"decoder_hidden_size\": 768,\n  \"depths\": [\n    3,\n    6,\n    40,\n    3\n  ],\n  \"downsampling_rates\": [\n    1,\n    4,\n    8,\n    16\n  ],\n  \"drop_path_rate\": 0.1,\n  \"hidden_act\": \"gelu\",\n  \"hidden_dropout_prob\": 0.0,\n  \"hidden_sizes\": [\n    64,\n    128,\n    320,\n    512\n  ],\n  \"image_size\": 224,\n  \"initializer_range\": 0.02,\n  \"layer_norm_eps\": 1e-06,\n  \"mlp_ratios\": [\n    4,\n    4,\n    4,\n    4\n  ],\n  \"model_type\": \"segformer\",\n  \"num_attention_heads\": [\n    1,\n    2,\n    5,\n    8\n  ],\n  \"num_channels\": 32,\n  \"num_encoder_blocks\": 4,\n  \"patch_sizes\": [\n    7,\n    3,\n    3,\n    3\n  ],\n  \"sr_ratios\": [\n    8,\n    4,\n    2,\n    1\n  ],\n  \"strides\": [\n    4,\n    2,\n    2,\n    2\n  ],\n  \"torch_dtype\": \"float32\",\n  \"transformers_version\": \"4.12.0.dev0\",\n  \"num_labels\":1\n})","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.310936Z","iopub.execute_input":"2023-06-13T18:47:32.311587Z","iopub.status.idle":"2023-06-13T18:47:32.322041Z","shell.execute_reply.started":"2023-06-13T18:47:32.311548Z","shell.execute_reply":"2023-06-13T18:47:32.321115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_3d_config = SegformerConfig(**{\n  \"architectures\": [\n    \"SegformerForImageClassification\"\n  ],\n  \"attention_probs_dropout_prob\": 0.0,\n  \"classifier_dropout_prob\": 0.1,\n  \"decoder_hidden_size\": 768,\n  \"depths\": [\n    3,\n    4,\n    18,\n    3\n  ],\n  \"downsampling_rates\": [\n    1,\n    4,\n    8,\n    16\n  ],\n  \"drop_path_rate\": 0.1,\n  \"hidden_act\": \"gelu\",\n  \"hidden_dropout_prob\": 0.0,\n  \"hidden_sizes\": [\n    64,\n    128,\n    320,\n    512\n  ],\n\n  \"image_size\": 224,\n  \"initializer_range\": 0.02,\n  \n  \"layer_norm_eps\": 1e-06,\n  \"mlp_ratios\": [\n    4,\n    4,\n    4,\n    4\n  ],\n  \"model_type\": \"segformer\",\n  \"num_attention_heads\": [\n    1,\n    2,\n    5,\n    8\n  ],\n  \"num_encoder_blocks\": 4,\n  \"patch_sizes\": [\n    7,\n    3,\n    3,\n    3\n  ],\n  \"sr_ratios\": [\n    8,\n    4,\n    2,\n    1\n  ],\n  \"strides\": [\n    4,\n    2,\n    2,\n    2\n  ],\n  \"torch_dtype\": \"float32\",\n  \"num_labels\":1,\n  \"num_channels\":32})\ncnn_3d_more_filters_config = SegformerConfig(**{\n  \"architectures\": [\n    \"SegformerForImageClassification\"\n  ],\n  \"attention_probs_dropout_prob\": 0.0,\n  \"classifier_dropout_prob\": 0.1,\n  \"decoder_hidden_size\": 768,\n  \"depths\": [\n    3,\n    4,\n    18,\n    3\n  ],\n  \"downsampling_rates\": [\n    1,\n    4,\n    8,\n    16\n  ],\n  \"drop_path_rate\": 0.1,\n  \"hidden_act\": \"gelu\",\n  \"hidden_dropout_prob\": 0.0,\n  \"hidden_sizes\": [\n    64,\n    128,\n    320,\n    512\n  ],\n\n  \"image_size\": 224,\n  \"initializer_range\": 0.02,\n  \n  \"layer_norm_eps\": 1e-06,\n  \"mlp_ratios\": [\n    4,\n    4,\n    4,\n    4\n  ],\n  \"model_type\": \"segformer\",\n  \"num_attention_heads\": [\n    1,\n    2,\n    5,\n    8\n  ],\n  \"num_encoder_blocks\": 4,\n  \"patch_sizes\": [\n    7,\n    3,\n    3,\n    3\n  ],\n  \"sr_ratios\": [\n    8,\n    4,\n    2,\n    1\n  ],\n  \"strides\": [\n    4,\n    2,\n    2,\n    2\n  ],\n  \"torch_dtype\": \"float32\",\n  \"num_labels\":1,\n  \"num_channels\":64})\n\nunet_3d_config = SegformerConfig(**{\n  \"architectures\": [\n    \"SegformerForImageClassification\"\n  ],\n  \"attention_probs_dropout_prob\": 0.0,\n  \"classifier_dropout_prob\": 0.1,\n  \"decoder_hidden_size\": 768,\n  \"depths\": [\n    3,\n    4,\n    18,\n    3\n  ],\n  \"downsampling_rates\": [\n    1,\n    4,\n    8,\n    16\n  ],\n  \"drop_path_rate\": 0.1,\n  \"hidden_act\": \"gelu\",\n  \"hidden_dropout_prob\": 0.0,\n  \"hidden_sizes\": [\n    64,\n    128,\n    320,\n    512\n  ],\n\n  \"image_size\": 224,\n  \"initializer_range\": 0.02,\n  \n  \"layer_norm_eps\": 1e-06,\n  \"mlp_ratios\": [\n    4,\n    4,\n    4,\n    4\n  ],\n  \"model_type\": \"segformer\",\n  \"num_attention_heads\": [\n    1,\n    2,\n    5,\n    8\n  ],\n  \"num_channels\": 3,\n  \"num_encoder_blocks\": 4,\n  \"patch_sizes\": [\n    7,\n    3,\n    3,\n    3\n  ],\n  \"sr_ratios\": [\n    8,\n    4,\n    2,\n    1\n  ],\n  \"strides\": [\n    4,\n    2,\n    2,\n    2\n  ],\n  \"torch_dtype\": \"float32\",\n  \"num_labels\":1,\n  \"num_channels\":16})","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.323664Z","iopub.execute_input":"2023-06-13T18:47:32.324303Z","iopub.status.idle":"2023-06-13T18:47:32.345075Z","shell.execute_reply.started":"2023-06-13T18:47:32.324264Z","shell.execute_reply":"2023-06-13T18:47:32.344005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unet_3d_jumbo_config = SegformerConfig(**{\n  \"architectures\": [\n    \"SegformerForImageClassification\"\n  ],\n  \"attention_probs_dropout_prob\": 0.0,\n  \"classifier_dropout_prob\": 0.1,\n  \"decoder_hidden_size\": 768,\n  \"depths\": [\n    3,\n    6,\n    40,\n    3\n  ],\n  \"downsampling_rates\": [\n    1,\n    4,\n    8,\n    16\n  ],\n  \"drop_path_rate\": 0.1,\n  \"hidden_act\": \"gelu\",\n  \"hidden_dropout_prob\": 0.0,\n  \"hidden_sizes\": [\n    64,\n    128,\n    320,\n    512\n  ],\n  \"image_size\": 224,\n  \"initializer_range\": 0.02,\n  \"layer_norm_eps\": 1e-06,\n  \"mlp_ratios\": [\n    4,\n    4,\n    4,\n    4\n  ],\n  \"model_type\": \"segformer\",\n  \"num_attention_heads\": [\n    1,\n    2,\n    5,\n    8\n  ],\n  \"num_channels\": 32,\n  \"num_encoder_blocks\": 4,\n  \"patch_sizes\": [\n    7,\n    3,\n    3,\n    3\n  ],\n  \"sr_ratios\": [\n    8,\n    4,\n    2,\n    1\n  ],\n  \"strides\": [\n    4,\n    2,\n    2,\n    2\n  ],\n  \"torch_dtype\": \"float32\",\n  \"transformers_version\": \"4.12.0.dev0\",\n  \"num_labels\":1\n})\n\nunetr_multiclass_config = SegformerConfig(**{\n  \"architectures\": [\n    \"SegformerForImageClassification\"\n  ],\n  \"attention_probs_dropout_prob\": 0.0,\n  \"classifier_dropout_prob\": 0.1,\n  \"decoder_hidden_size\": 768,\n  \"depths\": [\n    3,\n    6,\n    40,\n    3\n  ],\n  \"downsampling_rates\": [\n    1,\n    4,\n    8,\n    16\n  ],\n  \"drop_path_rate\": 0.1,\n  \"hidden_act\": \"gelu\",\n  \"hidden_dropout_prob\": 0.0,\n  \"hidden_sizes\": [\n    64,\n    128,\n    320,\n    512\n  ],\n  \"image_size\": 224,\n  \"initializer_range\": 0.02,\n  \"layer_norm_eps\": 1e-06,\n  \"mlp_ratios\": [\n    4,\n    4,\n    4,\n    4\n  ],\n  \"model_type\": \"segformer\",\n  \"num_attention_heads\": [\n    1,\n    2,\n    5,\n    8\n  ],\n  \"num_channels\": 32,\n  \"num_encoder_blocks\": 4,\n  \"patch_sizes\": [\n    7,\n    3,\n    3,\n    3\n  ],\n  \"sr_ratios\": [\n    8,\n    4,\n    2,\n    1\n  ],\n  \"strides\": [\n    4,\n    2,\n    2,\n    2\n  ],\n  \"torch_dtype\": \"float32\",\n  \"transformers_version\": \"4.12.0.dev0\",\n  \"num_labels\":3\n})","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:32.346748Z","iopub.execute_input":"2023-06-13T18:47:32.347429Z","iopub.status.idle":"2023-06-13T18:47:32.363295Z","shell.execute_reply.started":"2023-06-13T18:47:32.347391Z","shell.execute_reply":"2023-06-13T18:47:32.362289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from unetr import UNETR\nclass UNETR_Segformer(nn.Module):\n    def __init__(self, cfg, dropout = .2):\n        super().__init__()\n        self.cfg = cfg\n        self.dropout = nn.Dropout2d(dropout)\n        self.encoder = UNETR(\n            in_channels=1,\n            out_channels=32,\n            img_size=(16, self.cfg.size, self.cfg.size),\n            conv_block=True\n        )\n        self.encoder_2d = SegformerForSemanticSegmentation(self.cfg.segformer_config)\n        self.upscaler1 = nn.ConvTranspose2d(\n            1, 1, kernel_size=(4, 4), stride=2, padding=1)\n        self.upscaler2 = nn.ConvTranspose2d(\n            1, 1, kernel_size=(4, 4), stride=2, padding=1)\n\n    def forward(self, image):\n        output = self.encoder(image).max(axis=2)[0]\n        output = self.dropout(output)\n        output = self.encoder_2d(output).logits\n        output = self.upscaler1(output)\n        output = self.upscaler2(output)\n        return output\nclass UNETR_SegformerMC(nn.Module):\n    def __init__(self, cfg, dropout = .2):\n        super().__init__()\n        self.cfg = cfg\n        self.dropout = nn.Dropout2d(dropout)\n        self.encoder = UNETR(\n            in_channels=1,\n            out_channels=32,\n            img_size=(16, self.cfg.size, self.cfg.size),\n#             conv_block=True\n        )\n        self.encoder_2d = SegformerForSemanticSegmentation(self.cfg.segformer_config)\n        self.upscaler1 = nn.ConvTranspose2d(\n            3, 3, kernel_size=(4, 4), stride=2, padding=1)\n        self.upscaler2 = nn.ConvTranspose2d(\n            3, 3, kernel_size=(4, 4), stride=2, padding=1)\n\n    def forward(self, image):\n        output = self.encoder(image).max(axis=2)[0]\n        output = self.dropout(output)\n        output = self.encoder_2d(output).logits\n        output = self.upscaler1(output)\n        output = self.upscaler2(output)\n        return output[:, 2:, :, :]\n    \nclass cnn3d_segformer(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n        \n        self.conv3d_1 = nn.Conv3d(1, 4, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1))\n        self.conv3d_2 = nn.Conv3d(4, 8, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1))\n        self.conv3d_3 = nn.Conv3d(8, 16, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1))\n        self.conv3d_4 = nn.Conv3d(16, 32, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1))\n\n        self.xy_encoder_2d = SegformerForSemanticSegmentation(self.cfg.segformer_config)\n        self.upscaler1 = nn.ConvTranspose2d(1, 1, kernel_size=(4, 4), stride = 2, padding=1)\n        self.upscaler2 = nn.ConvTranspose2d(1, 1, kernel_size=(4, 4), stride = 2, padding=1)\n        \n    def forward(self, image):\n        output = self.conv3d_1(image)\n        output = self.conv3d_2(output)\n        output = self.conv3d_3(output)\n        output = self.conv3d_4(output).max(axis = 2)[0]\n        output = self.xy_encoder_2d(output).logits\n        output = self.upscaler1(output)\n        output = self.upscaler2(output)\n        return output\n    \nclass cnn3d_segformer_more_filters(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n        \n        self.conv3d_1 = nn.Conv3d(1, 4, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1))\n        self.conv3d_2 = nn.Conv3d(4, 8, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1))\n        self.conv3d_3 = nn.Conv3d(8, 16, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1))\n        self.conv3d_4 = nn.Conv3d(16, 64, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1))\n\n        self.xy_encoder_2d = SegformerForSemanticSegmentation(self.cfg.segformer_config)\n        self.upscaler1 = nn.ConvTranspose2d(1, 1, kernel_size=(4, 4), stride = 2, padding=1)\n        self.upscaler2 = nn.ConvTranspose2d(1, 1, kernel_size=(4, 4), stride = 2, padding=1)\n        \n    def forward(self, image):\n        output = self.conv3d_1(image)\n        output = self.conv3d_2(output)\n        output = self.conv3d_3(output)\n        output = self.conv3d_4(output).max(axis = 2)[0]\n        output = self.xy_encoder_2d(output).logits\n        output = self.upscaler1(output)\n        output = self.upscaler2(output)\n        return output\n    \nclass unet3d_segformer(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n        \n        self.model = get_model({\"name\":\"UNet3D\", \"in_channels\":1, \"out_channels\":16, \"f_maps\":8, \"num_groups\":4, \"is_segmentation\":False})\n        self.encoder_2d = SegformerForSemanticSegmentation(self.cfg.segformer_config)\n        self.upscaler1 = nn.ConvTranspose2d(1, 1, kernel_size=(4, 4), stride = 2, padding=1)\n        self.upscaler2 = nn.ConvTranspose2d(1, 1, kernel_size=(4, 4), stride = 2, padding=1)\n    def forward(self, image):\n        output = self.model(image).max(axis = 2)[0]\n        output = self.encoder_2d(output).logits\n        output = self.upscaler1(output)\n        output = self.upscaler2(output)\n        return output\n    \nclass unet3d_segformer_jumbo(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n        \n        self.model = get_model({\"name\":\"UNet3D\", \"in_channels\":1, \"out_channels\":32, \"f_maps\":8, \"num_groups\":4, \"is_segmentation\":False})\n        self.encoder_2d = SegformerForSemanticSegmentation(self.cfg.segformer_config)\n        self.upscaler1 = nn.ConvTranspose2d(1, 1, kernel_size=(4, 4), stride = 2, padding=1)\n        self.upscaler2 = nn.ConvTranspose2d(1, 1, kernel_size=(4, 4), stride = 2, padding=1)\n    def forward(self, image):\n        output = self.model(image).max(axis = 2)[0]\n        output = self.encoder_2d(output).logits\n        output = self.upscaler1(output)\n        output = self.upscaler2(output)\n        return output\n    \ndef build_model(cfg, model_arch = None):\n    print('model_name', cfg.model_name)\n    if model_arch == \"cnn3d\":\n        model = cnn3d_segformer(cfg)\n    if model_arch == \"cnn3d_more_filters\":\n        model = cnn3d_segformer_more_filters(cfg)\n    if model_arch == \"unet3d\":\n        model = unet3d_segformer(cfg)\n    if model_arch == \"unet3d_jumbo\":\n        model = unet3d_segformer_jumbo(cfg)\n    if model_arch == \"unetr\":\n        model = UNETR_Segformer(cfg)\n    if model_arch == \"unetr_mc\":\n        model = UNETR_SegformerMC(cfg)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:51:40.199301Z","iopub.execute_input":"2023-06-13T18:51:40.200220Z","iopub.status.idle":"2023-06-13T18:51:40.245421Z","shell.execute_reply.started":"2023-06-13T18:51:40.200174Z","shell.execute_reply":"2023-06-13T18:51:40.244245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EnsembleModel:\n    def __init__(self, use_tta=False):\n        self.models = []\n        self.use_tta = use_tta\n    def tta_infer(self, model:nn.Module, x):\n        #x.shape=(batch,c,h,w)\n        shape=x.shape\n        x=[x,*[torch.rot90(x,k=i,dims=(-2,-1)) for i in range(1,4)]]\n        x=[model(single_x) for single_x in x]\n        x=torch.cat(x,dim=0)\n        x=x.reshape(4,shape[0],*shape[3:])\n        x=[torch.rot90(x[i],k=-i,dims=(-2,-1)) for i in range(4)]\n        x=torch.stack(x,dim=0)\n        return x.mean(0)\n                \n    def __call__(self, x):\n        if self.use_tta:\n            outputs = [self.tta_infer(model, x).to('cpu').numpy()\n                   for model in self.models]\n        else:\n            outputs = [model(x).mean(axis = 1).to('cpu').numpy()\n                       for model in self.models]\n        avg_preds = np.mean(outputs, axis=0)\n        return avg_preds\n\n    def add_model(self, model):\n        self.models.append(model)\n\ndef build_ensemble_model(model_path, model_arch):\n    model = EnsembleModel(use_tta = True)\n    _model = build_model(CFG, model_arch)\n    _model.to(device)\n    state = torch.load(model_path)\n    try:\n        _model.load_state_dict(state)\n    except:\n        _model = nn.DataParallel(_model)\n        _model.load_state_dict(state)\n    _model.eval()\n\n    model.add_model(_model)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:51:40.686815Z","iopub.execute_input":"2023-06-13T18:51:40.687161Z","iopub.status.idle":"2023-06-13T18:51:40.703840Z","shell.execute_reply.started":"2023-06-13T18:51:40.687128Z","shell.execute_reply":"2023-06-13T18:51:40.702772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if mode == 'test':\n    fragment_ids = sorted(os.listdir(CFG.comp_dataset_path + mode))\nelse:\n    fragment_ids = [3]","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:51:41.044163Z","iopub.execute_input":"2023-06-13T18:51:41.044515Z","iopub.status.idle":"2023-06-13T18:51:41.052813Z","shell.execute_reply.started":"2023-06-13T18:51:41.044483Z","shell.execute_reply":"2023-06-13T18:51:41.051798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_tuples = [\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 3,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dcnn_segformer_1024_3dcnn_segformer_best.pth\", \"segformer_config\": cnn_3d_config, \"score\": .75},\n    {\"model_arch\": \"unet3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 1,\n     \"weight_path\": \"/kaggle/input/3d-unet/3d_unet_segformer_1024_3d_unet_segformer_final_all_train.pth\", \"segformer_config\": unet_3d_config, \"score\":.78},\n#     {\"model_arch\": \"unet3d\", \"tile_size\": 512, \"size\": 512, \"batch_size\": 4,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3d_unet_segformer_512_3d_unet_segformer_final.pth\", \"segformer_config\": unet_3d_config, \"score\":.77},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 3,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dcnn_segformer_1024_full_train_3dcnn_segformer_final.pth\", \"segformer_config\": cnn_3d_config, \"score\":.76},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 3,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dcnn_segformer_1024_swa_slow_3dcnn_segformer_final_swa.pth\", \"segformer_config\": cnn_3d_config, \"score\".74},\n#     {\"model_arch\": \"unet3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 1,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dunet_segformer_1024_swa_slow_3dunet_segformer_final_swa.pth\", \"segformer_config\": unet_3d_config, \"score\":.75},\n    {\"model_arch\": \"unet3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 1,\n     \"weight_path\": \"/kaggle/input/3d-unet/3dunet_segformer_1024_swa_slow_all_train_3dunet_segformer_final.pth\", \"segformer_config\": unet_3d_config, \"score\":.78},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 3,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dcnn_segformer_all_train_swa_3dcnn_segformer_10_final.pth\", \"segformer_config\": cnn_3d_config, \"score\":.76},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 3,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dcnn_segformer_all_train_swa_3dcnn_segformer_15_final.pth\", \"segformer_config\": cnn_3d_config, \"score\":.76},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 3,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dcnn_segformer_all_train_swa_3dcnn_segformer_20_final.pth\", \"segformer_config\": cnn_3d_config, \"score\":.77},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 3,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dcnn_segformer_all_train_swa_3dcnn_segformer_25_final.pth\", \"segformer_config\": cnn_3d_config},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 3,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dcnn_segformer_all_train_swa_3dcnn_segformer_final_swa.pth\", \"segformer_config\": cnn_3d_config, \"score\":.78},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 5,\n#      \"weight_path\": \"/kaggle/input/3d-unet/b1_3dcnn_segformer_b1_final_swa.pth\", \"segformer_config\": cnn_3d_segformer_b1_config, \"score\":.71},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 4,\n#      \"weight_path\": \"/kaggle/input/3d-unet/b2_3dcnn_segformer_b2_final_swa.pth\", \"segformer_config\": cnn_3d_segformer_b2_config, \"score\":.68},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 2,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dcnn_segformer_b4_3dcnn_segformer_b4_final_swa.pth\", \"segformer_config\": cnn_3d_segformer_b4_config, \"score\":.74},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 1,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dcnn_bigsegformer_3dcnn_bigsegformer_final.pth\", \"segformer_config\": cnn_3d_segformer_b5_config, \"score\":.76},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 2,\n#      \"weight_path\": \"/kaggle/input/3d-unet/b5_long_train_all_frags_3dcnn_segformer_b5_final_swa.pth\", \"segformer_config\": cnn_3d_segformer_b5_config, \"score\": .77},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 1,\n#      \"weight_path\": \"/kaggle/input/3d-unet/b5_long_train_all_frags_3dcnn_segformer_b5_10_final.pth\", \"segformer_config\": cnn_3d_segformer_b5_config, \"score\": .74},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 1,\n#      \"weight_path\": \"/kaggle/input/3d-unet/b5_long_train_all_frags_3dcnn_segformer_b5_30_final.pth\", \"segformer_config\": cnn_3d_segformer_b5_config, \"score\":.76},\n#     {\"model_arch\": \"cnn3d_more_filters\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 1,\n#      \"weight_path\": \"/kaggle/input/3d-unet/b3_more_fmaps_3dcnn_segformerb364_final_swa.pth\", \"segformer_config\": cnn_3d_more_filters_config, \"score\": .74},\n    {\"model_arch\": \"cnn3d_more_filters\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 2,\n     \"weight_path\": \"/kaggle/input/3d-unet/b3_more_fmaps_all_train_3dcnn_segformerb364_final_swa.pth\", \"segformer_config\": cnn_3d_more_filters_config, \"score\":.78},\n#     {\"model_arch\": \"unet3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 1,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3d_unet_3dunet_b3_final_swa.pth\", \"segformer_config\": unet_3d_config, \"score\":.73},\n#     {\"model_arch\": \"unet3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 1,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3d_unet_all_train_3dunet_b3_final_swa.pth\", \"segformer_config\": unet_3d_config, \"score\":.76},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 512, \"size\": 512, \"batch_size\": 4,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dcnn_512_b2_all_train_3dcnn_b2_final_swa.pth\", \"segformer_config\": cnn_3d_segformer_b2_config},\n    # ran at wrong resolution. Scored .73\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 2,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dcnn_768_b4_adam_3dcnn_b4_final_swa.pth\", \"segformer_config\": cnn_3d_segformer_b4_config, \"score\":.73},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 768, \"size\": 768, \"batch_size\": 2,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dcnn_768_b4_adam_3dcnn_b4_final_swa.pth\", \"segformer_config\": cnn_3d_segformer_b4_config, \"score\":.74},\n#     {\"model_arch\": \"cnn3d\", \"tile_size\": 768, \"size\": 768, \"batch_size\": 2,\n#      \"weight_path\": \"/kaggle/input/3d-unet/3dcnn_768_b4_adam_3dcnn_b4_final_swa_all_train.pth\", \"segformer_config\": cnn_3d_segformer_b4_config, \"score\":.75},\n    {\"model_arch\": \"unet3d_jumbo\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 1,\n     \"weight_path\": \"/kaggle/input/3d-unet/Jumbo_Unet_Jumbo_Unet_69_final_swa_all_train.pth\", \"segformer_config\": unet_3d_jumbo_config, \"score\":.79},\n#     {\"model_arch\": \"unet3d_jumbo\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 1,\n#      \"weight_path\": \"/kaggle/input/3d-unet/Jumbo_Unet_Jumbo_Unet_69_new_label_final_swa_all_train.pth\", \"segformer_config\": unet_3d_jumbo_config, \"score\":.77},\n#     {\"model_arch\": \"unet3d_jumbo\", \"tile_size\": 1024, \"size\": 1024, \"batch_size\": 1,\n#      \"weight_path\": \"/kaggle/input/3d-unet/Jumbo_Unet_Jumbo_Unet_5_final_swa_all_train.pth\", \"segformer_config\": unet_3d_jumbo_config, \"score\": .69},\n#     {\"model_arch\": \"unetr\", \"tile_size\": 512, \"size\": 512, \"batch_size\": 8,\n#      \"weight_path\": \"/kaggle/input/3d-unet/jumbo_unetr_unetr_1245_final_swa_all_train.pth\", \"segformer_config\": unet_3d_jumbo_config},\n#     {\"model_arch\": \"unetr_mc\", \"tile_size\": 512, \"size\": 512, \"batch_size\": 8,\n#      \"weight_path\": \"/kaggle/input/3d-unet/unetr_multiclass_512_b5_unet_final_4Ryan.pth\", \"segformer_config\": unetr_multiclass_config, \"score\":.77},\n    {\"model_arch\": \"unetr\", \"tile_size\": 512, \"size\": 512, \"batch_size\": 8,\n     \"weight_path\": \"/kaggle/input/3d-unet/jumbo_unetr_unetr_888_final_swa_all_train_long.pth\", \"segformer_config\": unet_3d_jumbo_config, \"score\":.82},\n    {\"model_arch\": \"unetr_mc\", \"tile_size\": 512, \"size\": 512, \"batch_size\": 8,\n     \"weight_path\": \"/kaggle/input/3d-unet/unetr_multiclass_NOVALIDATION_512_b5_unet_final_swa_all_train.pth\", \"segformer_config\": unetr_multiclass_config},\n\n]\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:51:41.419541Z","iopub.execute_input":"2023-06-13T18:51:41.419850Z","iopub.status.idle":"2023-06-13T18:51:41.432832Z","shell.execute_reply.started":"2023-06-13T18:51:41.419820Z","shell.execute_reply":"2023-06-13T18:51:41.431670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## main","metadata":{}},{"cell_type":"code","source":"def post_process(probability, threshold, min_size = 20000):\n    \"\"\"\n    Post processing of each predicted mask, components with lesser number of pixels\n    than `min_size` are ignored\n    \"\"\"\n    # don't remember where I saw it\n    mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]\n    num_component, component = cv2.connectedComponents(mask.astype(np.uint8))\n    predictions = np.zeros_like(probability, np.float32)\n    num = 0\n    for c in range(1, num_component):\n        p = (component == c)\n        if p.sum() > min_size:\n            predictions[p] = 1\n    return predictions","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:51:42.269950Z","iopub.execute_input":"2023-06-13T18:51:42.270335Z","iopub.status.idle":"2023-06-13T18:51:42.278614Z","shell.execute_reply.started":"2023-06-13T18:51:42.270298Z","shell.execute_reply":"2023-06-13T18:51:42.277433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport time","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:51:42.679050Z","iopub.execute_input":"2023-06-13T18:51:42.679952Z","iopub.status.idle":"2023-06-13T18:51:42.684597Z","shell.execute_reply.started":"2023-06-13T18:51:42.679913Z","shell.execute_reply":"2023-06-13T18:51:42.683346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = []\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    for fragment_id in fragment_ids:\n        mask_preds = None\n        last_res = None\n        for model_config in model_tuples:\n            mask_pred = None\n            mask_count = None\n            if last_res != model_config[\"size\"]:\n                for file in glob.glob(\"*.npy\"):\n                    os.remove(file)\n                last_res = model_config[\"size\"]\n            CFG.tile_size = model_config[\"tile_size\"]\n            CFG.size = model_config[\"size\"]\n            CFG.batch_size = model_config[\"batch_size\"]\n            CFG.stride = CFG.tile_size // 4\n            CFG.valid_aug_list = [\n                A.Resize(CFG.size, CFG.size),\n                A.Normalize(\n                    mean= [0] * CFG.in_chans,\n                    std= [1] * CFG.in_chans\n                ),\n                ToTensorV2(transpose_mask=True),\n            ]\n            CFG.segformer_config = model_config[\"segformer_config\"]\n            model = build_ensemble_model(model_config[\"weight_path\"], model_config[\"model_arch\"])\n            test_loader, xyxys = make_test_dataset(fragment_id)\n\n            binary_mask = cv2.imread(CFG.comp_dataset_path + f\"{mode}/{fragment_id}/mask.png\", 0)\n            binary_mask = (binary_mask / 255).astype(int)\n\n            ori_h = binary_mask.shape[0]\n            ori_w = binary_mask.shape[1]\n            # mask = mask / 255\n\n            pad0 = (CFG.tile_size - binary_mask.shape[0] % CFG.tile_size)\n            pad1 = (CFG.tile_size - binary_mask.shape[1] % CFG.tile_size)\n\n            binary_mask = np.pad(binary_mask, [(0, pad0), (0, pad1)], constant_values=0)\n            if mask_pred is None:\n                mask_pred = np.zeros(binary_mask.shape)\n                mask_count = np.zeros(binary_mask.shape)\n\n            for step, (images) in tqdm(enumerate(test_loader), total=len(test_loader)):\n                images = images.to(device)\n                batch_size = images.size(0)\n                with autocast():            \n                    with torch.no_grad():\n                        y_preds = model(images)\n\n                start_idx = step*CFG.batch_size\n                end_idx = start_idx + batch_size\n                for i, (x1, y1, x2, y2) in enumerate(xyxys[start_idx:end_idx]):\n                    mask_pred[y1:y2, x1:x2] += y_preds[i]\n                    mask_count[y1:y2, x1:x2] += np.ones((CFG.tile_size, CFG.tile_size))\n            del test_loader\n            del model\n            gc.collect()\n            torch.cuda.empty_cache()\n            mask_pred = mask_pred[:ori_h, :ori_w]\n            mask_count = mask_count[:ori_h, :ori_w]\n            binary_mask = binary_mask[:ori_h, :ori_w]\n\n            print(f'mask_count_min: {mask_count.min()}')\n            mask_pred = mask_pred/mask_count\n            mask_pred = torch.sigmoid(torch.tensor(mask_pred)).numpy()\n            if mask_preds is None:\n                mask_preds = mask_pred/len(model_tuples)\n            else:\n                mask_preds += mask_pred/len(model_tuples)\n\n        mask_pred = (mask_preds >= TH).astype(int)\n        mask_pred *= binary_mask\n        mask_pred = post_process(mask_pred.astype(float), TH, 10000).astype(int)\n        plt.imshow(mask_pred)\n        inklabels_rle = rle(mask_pred)\n        results.append((fragment_id, inklabels_rle))\n        del mask_pred, mask_count\n        gc.collect()\n        torch.cuda.empty_cache()\n        for file in glob.glob(\"*.npy\"):\n            os.remove(file)\nelse:\n    pass\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:51:43.075483Z","iopub.execute_input":"2023-06-13T18:51:43.076309Z","iopub.status.idle":"2023-06-13T19:31:51.441564Z","shell.execute_reply.started":"2023-06-13T18:51:43.076260Z","shell.execute_reply":"2023-06-13T19:31:51.440365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## submission","metadata":{}},{"cell_type":"code","source":"sub = pd.DataFrame(results, columns=['Id', 'Predicted'])","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:36.683281Z","iopub.status.idle":"2023-06-13T18:47:36.684466Z","shell.execute_reply.started":"2023-06-13T18:47:36.684177Z","shell.execute_reply":"2023-06-13T18:47:36.684205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:36.686213Z","iopub.status.idle":"2023-06-13T18:47:36.686709Z","shell.execute_reply.started":"2023-06-13T18:47:36.686460Z","shell.execute_reply":"2023-06-13T18:47:36.686486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv(CFG.comp_dataset_path + 'sample_submission.csv')\nsample_sub = pd.merge(sample_sub[['Id']], sub, on='Id', how='left')","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:36.688567Z","iopub.status.idle":"2023-06-13T18:47:36.689177Z","shell.execute_reply.started":"2023-06-13T18:47:36.688937Z","shell.execute_reply":"2023-06-13T18:47:36.688962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:36.690811Z","iopub.status.idle":"2023-06-13T18:47:36.691312Z","shell.execute_reply.started":"2023-06-13T18:47:36.691052Z","shell.execute_reply":"2023-06-13T18:47:36.691077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T18:47:36.693114Z","iopub.status.idle":"2023-06-13T18:47:36.694089Z","shell.execute_reply.started":"2023-06-13T18:47:36.693852Z","shell.execute_reply":"2023-06-13T18:47:36.693878Z"},"trusted":true},"execution_count":null,"outputs":[]}]}