{"cells":[{"metadata":{},"cell_type":"markdown","source":"Credits\n\nhttps://www.kaggle.com/ammarali32/resnet200d-inference-single-model-lb-96-5\n\nhttps://www.kaggle.com/ammarali32/seresnet152d-inference-single-model-lb-96-2\n\nhttps://www.kaggle.com/underwearfitting/resnet200d-public-benchmark-2xtta-lb0-965\n\nhttps://www.kaggle.com/ttahara/ranzcr-multi-head-model-inference/data?scriptVersionId=55373372&select=fold0\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path.append('../input/pytorch-images-seresnet')\n\nimport os\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\nimport gc\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nfrom matplotlib import pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nimport albumentations\nfrom albumentations import *\nfrom albumentations.pytorch import ToTensorV2\n\n\nimport timm\n\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"BATCH_SIZE = 128\nTEST_PATH = '../input/ranzcr-clip-catheter-line-classification/test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_transforms(image_size=640):\n        return Compose([\n            Resize(image_size, image_size),\n            Normalize(),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResNet200D(nn.Module):\n    def __init__(self, model_name='resnet200d_320'):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, 11)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output\n\nclass SeResNet152D(nn.Module):\n    def __init__(self, model_name='seresnet152d_320'):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, 11)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output\n    \nclass RANZCRResNet200D(nn.Module):\n    def __init__(self, model_name='resnet200d', out_dim=11, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, out_dim)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference(models, test_loader, device):\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        for model in models:\n            with torch.no_grad():\n                y_preds1 = model(images)\n                y_preds2 = model(images.flip(-1))\n            y_preds = (y_preds1.sigmoid().to('cpu').numpy() + y_preds2.sigmoid().to('cpu').numpy()) / 2\n            avg_preds.append(y_preds)\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models200D = []\nmodel = ResNet200D()\nmodel.load_state_dict(torch.load(\"../input/resnet200d-public/resnet200d_320_CV9632.pth\")['model'])\nmodel.eval()\nmodel.to(device)\nmodels200D.append(model)\n\nmodels200D_2 = []\nmodel = RANZCRResNet200D()\nmodel.load_state_dict(torch.load(\"../input/resnet200d-baseline-benchmark-public/resnet200d_fold0_cv953.pth\", map_location='cuda:0'))\nmodel.eval()\nmodel.to(device)\nmodels200D_2.append(model)\n\nmodel = RANZCRResNet200D()\nmodel.load_state_dict(torch.load(\"../input/resnet200d-baseline-benchmark-public/resnet200d_fold1_cv955.pth\", map_location='cuda:0'))\nmodel.eval()\nmodel.to(device)\nmodels200D_2.append(model)\n\nmodel = RANZCRResNet200D()\nmodel.load_state_dict(torch.load(\"../input/resnet200d-baseline-benchmark-public/resnet200d_fold2_cv955.pth\", map_location='cuda:0'))\nmodel.eval()\nmodel.to(device)\nmodels200D_2.append(model)\n\nmodel = RANZCRResNet200D()\nmodel.load_state_dict(torch.load(\"../input/resnet200d-baseline-benchmark-public/resnet200d_fold3_cv957.pth\", map_location='cuda:0'))\nmodel.eval()\nmodel.to(device)\nmodels200D_2.append(model)\n\nmodel = RANZCRResNet200D()\nmodel.load_state_dict(torch.load(\"../input/resnet200d-baseline-benchmark-public/resnet200d_fold4_cv954.pth\", map_location='cuda:0'))\nmodel.eval()\nmodel.to(device)\nmodels200D_2.append(model)\n\nmodels152D = []\nmodel = SeResNet152D()\nmodel.load_state_dict(torch.load('../input/seresnet152d-cv9615/seresnet152d_320_CV96.15.pth')['model'])\nmodel.eval()\nmodel.to(device)\nmodels152D.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset_512 = TestDataset(test, transform=get_transforms(image_size=512))\ntest_loader_512 = DataLoader(test_dataset_512, batch_size=BATCH_SIZE, shuffle=False, num_workers=4 , pin_memory=True)\n\ntest_dataset_640 = TestDataset(test, transform=get_transforms(image_size=640))\ntest_loader_640 = DataLoader(test_dataset_640, batch_size=BATCH_SIZE, shuffle=False, num_workers=4 , pin_memory=True)\n\npredictions200d = inference(models200D, test_loader_640, device)\npredictions200d_2 = inference(models200D_2, test_loader_512, device)\npredictions152d = inference(models152D, test_loader_640, device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del models200D\ndel models200D_2\ndel models152D\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions200d.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"https://www.kaggle.com/ttahara/ranzcr-multi-head-model-inference/data?scriptVersionId=55373372"},{"metadata":{"trusted":true},"cell_type":"code","source":"from joblib import Parallel, delayed\nfrom pathlib import Path\nimport typing as tp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT = ROOT / \"input\"\nOUTPUT = ROOT / \"output\"\nDATA = INPUT / \"ranzcr-clip-catheter-line-classification\"\nTRAIN = DATA / \"train\"\nTEST = DATA / \"test\"\n\n\nTRAINED_MODEL = INPUT / \"ranzcr-clip-weights-for-multi-head-model-v2\"\nTMP = ROOT / \"tmp\"\nTMP.mkdir(exist_ok=True)\n\nRANDAM_SEED = 1086\nN_CLASSES = 11\nFOLDS = [0, 1, 2, 3, 4]\nN_FOLD = len(FOLDS)\nIMAGE_SIZE = (512, 512)\n\nCONVERT_TO_RANK = False\nFAST_COMMIT = False\n\nCLASSES = [\n    'ETT - Abnormal',\n    'ETT - Borderline',\n    'ETT - Normal',\n    'NGT - Abnormal',\n    'NGT - Borderline',\n    'NGT - Incompletely Imaged',\n    'NGT - Normal',\n    'CVC - Abnormal',\n    'CVC - Borderline',\n    'CVC - Normal',\n    'Swan Ganz Catheter Present'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"smpl_sub =  pd.read_csv(DATA / \"sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_images(img_id, input_dir, output_dir, resize_to=(512, 512), ext=\"png\"):\n    img_path = input_dir / f\"{img_id}.jpg\"\n    save_path = output_dir / f\"{img_id}.{ext}\"\n    \n    img = cv2.imread(str(img_path), cv2.IMREAD_GRAYSCALE)\n    img = cv2.resize(img, resize_to)\n    cv2.imwrite(str(save_path), img, )\n\nTEST_RESIZED = TMP / \"test_{0}x{1}\".format(*IMAGE_SIZE)\nTEST_RESIZED.mkdir(exist_ok=True)\nTEST_RESIZED\n\n_ = Parallel(n_jobs=2, verbose=5)([\n    delayed(resize_images)(img_id, TEST, TEST_RESIZED, IMAGE_SIZE, \"png\")\n    for img_id in smpl_sub.StudyInstanceUID.values\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_activation(activ_name: str=\"relu\"):\n    \"\"\"\"\"\"\n    act_dict = {\n        \"relu\": nn.ReLU(inplace=True),\n        \"tanh\": nn.Tanh(),\n        \"sigmoid\": nn.Sigmoid(),\n        \"identity\": nn.Identity()}\n    if activ_name in act_dict:\n        return act_dict[activ_name]\n    else:\n        raise NotImplementedError\n        \n\nclass Conv2dBNActiv(nn.Module):\n    \"\"\"Conv2d -> (BN ->) -> Activation\"\"\"\n    \n    def __init__(\n        self, in_channels: int, out_channels: int,\n        kernel_size: int, stride: int=1, padding: int=0,\n        bias: bool=False, use_bn: bool=True, activ: str=\"relu\"\n    ):\n        \"\"\"\"\"\"\n        super(Conv2dBNActiv, self).__init__()\n        layers = []\n        layers.append(nn.Conv2d(\n            in_channels, out_channels,\n            kernel_size, stride, padding, bias=bias))\n        if use_bn:\n            layers.append(nn.BatchNorm2d(out_channels))\n            \n        layers.append(get_activation(activ))\n        self.layers = nn.Sequential(*layers)\n        \n    def forward(self, x):\n        \"\"\"Forward\"\"\"\n        return self.layers(x)\n        \n\nclass SSEBlock(nn.Module):\n    \"\"\"channel `S`queeze and `s`patial `E`xcitation Block.\"\"\"\n\n    def __init__(self, in_channels: int):\n        \"\"\"Initialize.\"\"\"\n        super(SSEBlock, self).__init__()\n        self.channel_squeeze = nn.Conv2d(\n            in_channels=in_channels, out_channels=1,\n            kernel_size=1, stride=1, padding=0, bias=False)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        \"\"\"Forward.\"\"\"\n        # # x: (bs, ch, h, w) => h: (bs, 1, h, w)\n        h = self.sigmoid(self.channel_squeeze(x))\n        # # x, h => return: (bs, ch, h, w)\n        return x * h\n    \n    \nclass SpatialAttentionBlock(nn.Module):\n    \"\"\"Spatial Attention for (C, H, W) feature maps\"\"\"\n    \n    def __init__(\n        self, in_channels: int,\n        out_channels_list: tp.List[int],\n    ):\n        \"\"\"Initialize\"\"\"\n        super(SpatialAttentionBlock, self).__init__()\n        self.n_layers = len(out_channels_list)\n        channels_list = [in_channels] + out_channels_list\n        assert self.n_layers > 0\n        assert channels_list[-1] == 1\n        \n        for i in range(self.n_layers - 1):\n            in_chs, out_chs = channels_list[i: i + 2]\n            layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ=\"relu\")\n            setattr(self, f\"conv{i + 1}\", layer)\n            \n        in_chs, out_chs = channels_list[-2:]\n        layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ=\"sigmoid\")\n        setattr(self, f\"conv{self.n_layers}\", layer)\n    \n    def forward(self, x):\n        \"\"\"Forward\"\"\"\n        h = x\n        for i in range(self.n_layers):\n            h = getattr(self, f\"conv{i + 1}\")(h)\n            \n        h = h * x\n        return h","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiHeadResNet200D(nn.Module):\n    \n    def __init__(\n        self, out_dims_head: tp.List[int]=[3, 4, 3, 1], pretrained=False\n    ):\n        \"\"\"\"\"\"\n        self.base_name = \"resnet200d_320\"\n        self.n_heads = len(out_dims_head)\n        super(MultiHeadResNet200D, self).__init__()\n        \n        # # load base model\n        base_model = timm.create_model(\n            self.base_name, num_classes=sum(out_dims_head), pretrained=False)\n        in_features = base_model.num_features\n        \n        if pretrained:\n            pretrained_model_path = '../input/startingpointschestx/resnet200d_320_chestx.pth'\n            state_dict = dict()\n            for k, v in torch.load(pretrained_model_path, map_location='cpu')[\"model\"].items():\n                if k[:6] == \"model.\":\n                    k = k.replace(\"model.\", \"\")\n                state_dict[k] = v\n            base_model.load_state_dict(state_dict)\n        \n        # # remove global pooling and head classifier\n        base_model.reset_classifier(0, '')\n        \n        # # Shared CNN Bacbone\n        self.backbone = base_model\n        \n        # # Multi Heads.\n        for i, out_dim in enumerate(out_dims_head):\n            layer_name = f\"head_{i}\"\n            layer = nn.Sequential(\n                SpatialAttentionBlock(in_features, [64, 32, 16, 1]),\n                nn.AdaptiveAvgPool2d(output_size=1),\n                nn.Flatten(start_dim=1),\n                nn.Linear(in_features, in_features),\n                nn.ReLU(inplace=True),\n                nn.Dropout(0.5),\n                nn.Linear(in_features, out_dim))\n            setattr(self, layer_name, layer)\n\n    def forward(self, x):\n        \"\"\"\"\"\"\n        h = self.backbone(x)\n        hs = [\n            getattr(self, f\"head_{i}\")(h) for i in range(self.n_heads)]\n        y = torch.cat(hs, axis=1)\n        return y\n    \n\n# ## forward test\n# m = MultiHeadResNet200D([3, 4, 3, 1], False)\n# m = m.eval()\n\n# x = torch.rand(1, 3, 256, 256)\n# with torch.no_grad():\n#     y = m(x)\n# print(\"[forward test]\")\n# print(\"input:\\t{}\\noutput:\\t{}\".format(x.shape, y.shape))\n\n# del m; del x; del y\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LabeledImageDataset(Dataset):\n    \"\"\"\n    Dataset class for (image, label) pairs\n\n    reads images and applys transforms to them.\n\n    Attributes\n    ----------\n    file_list : List[Tuple[tp.Union[str, Path], tp.Union[int, float, np.ndarray]]]\n        list of (image file, label) pair\n    transform_list : List[Dict]\n        list of dict representing image transform \n    \"\"\"\n\n    def __init__(\n        self,\n        file_list: tp.List[\n            tp.Tuple[tp.Union[str, Path], tp.Union[int, float, np.ndarray]]],\n        transform_list: tp.List[tp.Dict],\n    ):\n        \"\"\"Initialize\"\"\"\n        self.file_list = file_list\n        self.transform = ImageTransformForCls(transform_list)\n\n    def __len__(self):\n        \"\"\"Return Num of Images.\"\"\"\n        return len(self.file_list)\n\n    def __getitem__(self, index):\n        \"\"\"Return transformed image and mask for given index.\"\"\"\n        img_path, label = self.file_list[index]\n        img = self._read_image_as_array(img_path)\n        \n        img, label = self.transform((img, label))\n        return img, label\n\n    def _read_image_as_array(self, path: str):\n        \"\"\"Read image file and convert into numpy.ndarray\"\"\"\n        img_arr = cv2.imread(str(path))\n        img_arr = cv2.cvtColor(img_arr, cv2.COLOR_BGR2RGB)\n        return img_arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataloaders_for_inference(\n    file_list: tp.List[tp.List], batch_size=32,\n):\n    \"\"\"Create DataLoader\"\"\"\n    dataset = LabeledImageDataset(\n        file_list,\n        transform_list=[\n          [\"Normalize\", {\n              \"always_apply\": True, \"max_pixel_value\": 255.0,\n              \"mean\": [\"0.4887381077884414\"], \"std\": [\"0.23064819430546407\"]}],\n          [\"ToTensorV2\", {\"always_apply\": True}],\n        ])\n    loader = DataLoader(\n        dataset,\n        batch_size=batch_size, shuffle=False,\n        num_workers=0, pin_memory=True,\n        drop_last=False)\n\n    return loader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageTransformBase:\n    \"\"\"\n    Base Image Transform class.\n\n    Args:\n        data_augmentations: List of tuple(method: str, params :dict), each elems pass to albumentations\n    \"\"\"\n\n    def __init__(self, data_augmentations: tp.List[tp.Tuple[str, tp.Dict]]):\n        \"\"\"Initialize.\"\"\"\n        augmentations_list = [\n            self._get_augmentation(aug_name)(**params)\n            for aug_name, params in data_augmentations]\n        self.data_aug = albumentations.Compose(augmentations_list)\n\n    def __call__(self, pair: tp.Tuple[np.ndarray]) -> tp.Tuple[np.ndarray]:\n        \"\"\"You have to implement this by task\"\"\"\n        raise NotImplementedError\n\n    def _get_augmentation(self, aug_name: str) -> tp.Tuple[ImageOnlyTransform, DualTransform]:\n        \"\"\"Get augmentations from albumentations\"\"\"\n        if hasattr(albumentations, aug_name):\n            return getattr(albumentations, aug_name)\n        else:\n            return eval(aug_name)\n\n\nclass ImageTransformForCls(ImageTransformBase):\n    \"\"\"Data Augmentor for Classification Task.\"\"\"\n\n    def __init__(self, data_augmentations: tp.List[tp.Tuple[str, tp.Dict]]):\n        \"\"\"Initialize.\"\"\"\n        super(ImageTransformForCls, self).__init__(data_augmentations)\n\n    def __call__(self, in_arrs: tp.Tuple[np.ndarray]) -> tp.Tuple[np.ndarray]:\n        \"\"\"Apply Transform.\"\"\"\n        img, label = in_arrs\n        augmented = self.data_aug(image=img)\n        img = augmented[\"image\"]\n\n        return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_setting_file(path: str):\n    \"\"\"Load YAML setting file.\"\"\"\n    with open(path) as f:\n        settings = yaml.safe_load(f)\n    return settings\n\n\ndef set_random_seed(seed: int = 42, deterministic: bool = False):\n    \"\"\"Set seeds\"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = deterministic  # type: ignore\n    \n\ndef run_inference_loop(stgs, model, loader, device):\n    model.to(device)\n    model.eval()\n    pred_list = []\n    with torch.no_grad():\n        for x, t in tqdm(loader):\n            y = model(x.to(device))\n            pred_list.append(y.sigmoid().detach().cpu().numpy())\n            # pred_list.append(y.detach().cpu().numpy())\n        \n    pred_arr = np.concatenate(pred_list)\n    del pred_list\n    return pred_arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not torch.cuda.is_available():\n    device = torch.device(\"cpu\")\nelse:\n    device = torch.device(\"cuda\")\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import yaml","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_dir = TRAINED_MODEL\ntest_dir = TEST_RESIZED\n\ntest_file_list = [\n    (test_dir / f\"{img_id}.png\", [-1] * 11)\n    for img_id in smpl_sub[\"StudyInstanceUID\"].values]\ntest_loader = get_dataloaders_for_inference(test_file_list, batch_size=32)\n        \ntest_preds_arr = np.zeros((N_FOLD, len(smpl_sub), N_CLASSES))    \nfor fold_id in FOLDS:\n    print(f\"[fold {fold_id}]\")\n    stgs = load_setting_file(model_dir / f\"fold{fold_id}\" / \"settings.yml\")\n    # # prepare \n    stgs[\"model\"][\"params\"][\"pretrained\"] = False\n    model = MultiHeadResNet200D(**stgs[\"model\"][\"params\"])\n    model_path = model_dir / f\"best_model_fold{fold_id}.pth\"\n    model.load_state_dict(torch.load(model_path, map_location=device))\n\n    # # inference test\n    test_pred = run_inference_loop(stgs, model, test_loader, device)\n    test_preds_arr[fold_id] = test_pred\n    \n    del model\n    del stgs\n    torch.cuda.empty_cache()\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_MultiHeadResNet200D1 = test_preds_arr.mean(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_MultiHeadResNet200D1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_dir = INPUT / \"ranzcr-multi-head-model-training-f0/\"\ntest_dir = TEST_RESIZED\n\ntest_file_list = [\n    (test_dir / f\"{img_id}.png\", [-1] * 11)\n    for img_id in smpl_sub[\"StudyInstanceUID\"].values]\ntest_loader = get_dataloaders_for_inference(test_file_list, batch_size=32)\n        \ntest_preds_arr = np.zeros((1, len(smpl_sub), N_CLASSES))    \nfor fold_id in [0]:\n    print(f\"[fold {fold_id}]\")\n    stgs = load_setting_file(model_dir / f\"fold{fold_id}\" / \"settings.yml\")\n    # # prepare \n    stgs[\"model\"][\"params\"][\"pretrained\"] = False\n    model = MultiHeadResNet200D(**stgs[\"model\"][\"params\"])\n    model_path = model_dir / f\"best_model_fold{fold_id}.pth\"\n    model.load_state_dict(torch.load(model_path, map_location=device))\n\n    # # inference test\n    test_pred = run_inference_loop(stgs, model, test_loader, device)\n    test_preds_arr[fold_id] = test_pred\n    \n    del model\n    del stgs\n    torch.cuda.empty_cache()\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_MultiHeadResNet200D2 = test_preds_arr.mean(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_dir = INPUT / \"ranzcr-multi-head-model-training-f1/\"\ntest_dir = TEST_RESIZED\n\ntest_file_list = [\n    (test_dir / f\"{img_id}.png\", [-1] * 11)\n    for img_id in smpl_sub[\"StudyInstanceUID\"].values]\ntest_loader = get_dataloaders_for_inference(test_file_list, batch_size=32)\n        \ntest_preds_arr = np.zeros((1, len(smpl_sub), N_CLASSES))    \nfor fold_id in [1]:\n    print(f\"[fold {fold_id}]\")\n    stgs = load_setting_file(model_dir / f\"fold{fold_id}\" / \"settings.yml\")\n    # # prepare \n    stgs[\"model\"][\"params\"][\"pretrained\"] = False\n    model = MultiHeadResNet200D(**stgs[\"model\"][\"params\"])\n    model_path = model_dir / f\"best_model_fold{fold_id}.pth\"\n    model.load_state_dict(torch.load(model_path, map_location=device))\n\n    # # inference test\n    test_pred = run_inference_loop(stgs, model, test_loader, device)\n    test_preds_arr[0] = test_pred\n    \n    del model\n    del stgs\n    torch.cuda.empty_cache()\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_MultiHeadResNet200D3 = test_preds_arr.mean(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_dir = INPUT / \"ranzcr-multi-head-model-training-f2/\"\ntest_dir = TEST_RESIZED\n\ntest_file_list = [\n    (test_dir / f\"{img_id}.png\", [-1] * 11)\n    for img_id in smpl_sub[\"StudyInstanceUID\"].values]\ntest_loader = get_dataloaders_for_inference(test_file_list, batch_size=32)\n        \ntest_preds_arr = np.zeros((1, len(smpl_sub), N_CLASSES))    \nfor fold_id in [2]:\n    print(f\"[fold {fold_id}]\")\n    stgs = load_setting_file(model_dir / f\"fold{fold_id}\" / \"settings.yml\")\n    # # prepare \n    stgs[\"model\"][\"params\"][\"pretrained\"] = False\n    model = MultiHeadResNet200D(**stgs[\"model\"][\"params\"])\n    model_path = model_dir / f\"best_model_fold{fold_id}.pth\"\n    model.load_state_dict(torch.load(model_path, map_location=device))\n\n    # # inference test\n    test_pred = run_inference_loop(stgs, model, test_loader, device)\n    test_preds_arr[0] = test_pred\n    \n    del model\n    del stgs\n    torch.cuda.empty_cache()\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_MultiHeadResNet200D4 = test_preds_arr.mean(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = (predictions200d + predictions200d_2 + 0.5 * predictions152d + 0.5 * predictions_MultiHeadResNet200D1 + 0.25* predictions_MultiHeadResNet200D2 + 0.25 * predictions_MultiHeadResNet200D3 + 0.25 * predictions_MultiHeadResNet200D4) / 3.75","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#predictions = (predictions200d + predictions200d_2 + predictions152d + predictions_MultiHeadResNet200D) / 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_cols = test.iloc[:, 1:12].columns.tolist()\ntest[target_cols] = predictions\ntest[['StudyInstanceUID'] + target_cols].to_csv('submission.csv', index=False)\ntest.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}