{"cells":[{"metadata":{},"cell_type":"markdown","source":"4"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Credits: I just ensembled the 2 following notebooks published by ammarali32. All credit goes to him for notebook and published model. please upvote following 2 notebooks.\n\nhttps://www.kaggle.com/ammarali32/resnet200d-inference-single-model-lb-96-5\nhttps://www.kaggle.com/ammarali32/seresnet152d-inference-single-model-lb-96-2\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path.append('../input/pytorch-images-seresnet')\n\nimport os\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nfrom matplotlib import pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nimport albumentations\nfrom albumentations import *\nfrom albumentations.pytorch import ToTensorV2\n\n\nimport timm\n\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"IMAGE_SIZE = 640\nBATCH_SIZE = 128\nTEST_PATH = '../input/ranzcr-clip-catheter-line-classification/test'\nMODEL_PATH_resnet200d = '../input/resnet200d-public/resnet200d_320_CV9632.pth'\nMODEL_PATH_seresnet152d = '../input/seresnet152d-cv9615/seresnet152d_320_CV96.15.pth'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\nsub_test = test.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_transforms():\n        return Compose([\n            Resize(IMAGE_SIZE, IMAGE_SIZE),\n            Normalize(\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResNet200D(nn.Module):\n    def __init__(self, model_name='resnet200d_320'):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, 11)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output\n\nclass SeResNet152D(nn.Module):\n    def __init__(self, model_name='seresnet152d_320'):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, 11)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference(models, test_loader, device):\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        for model in models:\n            with torch.no_grad():\n                y_preds1 = model(images)\n                y_preds2 = model(images.flip(-1))\n            y_preds = (y_preds1.sigmoid().to('cpu').numpy() + y_preds2.sigmoid().to('cpu').numpy()) / 2\n            avg_preds.append(y_preds)\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models200D = []\nmodel = ResNet200D()\nmodel.load_state_dict(torch.load(MODEL_PATH_resnet200d)['model'])\nmodel.eval()\nmodel.to(device)\nmodels200D.append(model)\n\nmodels152D = []\nmodel = SeResNet152D()\nmodel.load_state_dict(torch.load(MODEL_PATH_seresnet152d)['model'])\nmodel.eval()\nmodel.to(device)\nmodels152D.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = TestDataset(test, transform=get_transforms())\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, \n                         num_workers=4 , pin_memory=True)\npred200d = inference(models200D, test_loader, device)\npred152d = inference(models152D, test_loader, device)\n# predictions = (2 * pred200d + pred152d) / 3.0\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(pred200d).to_csv('pred200d.csv')\npd.DataFrame(pred152d).to_csv('pred152d.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## pred_2xTTA965"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nbatch_size = 1\nimage_size = 512\ntta = True\nsubmit = True\nenet_type = ['resnet200d'] * 5\nmodel_path = ['../input/resnet200d-baseline-benchmark-public/resnet200d_fold0_cv953.pth',\n              '../input/resnet200d-baseline-benchmark-public/resnet200d_fold1_cv955.pth',\n              '../input/resnet200d-baseline-benchmark-public/resnet200d_fold2_cv955.pth',\n              '../input/resnet200d-baseline-benchmark-public/resnet200d_fold3_cv957.pth',\n              '../input/resnet200d-baseline-benchmark-public/resnet200d_fold4_cv954.pth']\n# you can save GPU quota using fast sub attached in the last markdown file\nfast_sub = False\nfast_sub_path = '../input/xxxxxx/your_submission.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport numpy as np\nDEBUG = False\nimport time\nimport cv2\nimport PIL.Image\nfrom sklearn.metrics import accuracy_score\nfrom tqdm.notebook import tqdm\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nimport albumentations\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom pylab import rcParams\nimport timm\nfrom albumentations import *\nfrom albumentations.pytorch import ToTensorV2\ndevice = torch.device('cuda') if not DEBUG else torch.device('cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZCRResNet200D(nn.Module):\n    def __init__(self, model_name='resnet200d', out_dim=11, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, out_dim)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_test = albumentations.Compose([\n    Resize(image_size, image_size),\n    Normalize(\n         mean=[0.485, 0.456, 0.406],\n         std=[0.229, 0.224, 0.225],\n     ),\n    ToTensorV2()\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZCRDataset(Dataset):\n    def __init__(self, df, mode, transform=None):\n        \n        self.df = df.reset_index(drop=True)\n        self.mode = mode\n        self.transform = transform\n        self.labels = df[target_cols].values\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        row = self.df.loc[index]\n        img = cv2.imread(row.file_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        if self.transform is not None:\n            res = self.transform(image=img)\n            img = res['image']\n        label = torch.tensor(self.labels[index]).float()\n        if self.mode == 'test':\n            return img\n        else:\n            return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\ntest['file_path'] = test.StudyInstanceUID.apply(lambda x: os.path.join('../input/ranzcr-clip-catheter-line-classification/test', f'{x}.jpg'))\ntarget_cols = test.iloc[:, 1:12].columns.tolist()\n\ntest_dataset = RANZCRDataset(test, 'test', transform=transforms_test)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False,  num_workers=24)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_func(test_loader):\n    model.eval()\n    bar = tqdm(test_loader)\n    LOGITS = []\n    PREDS = []\n    \n    with torch.no_grad():\n        for batch_idx, images in enumerate(bar):\n            x = images.to(device)\n            logits = model(x)\n            LOGITS.append(logits.cpu())\n            PREDS += [logits.sigmoid().detach().cpu()]\n        PREDS = torch.cat(PREDS).cpu().numpy()\n        LOGITS = torch.cat(LOGITS).cpu().numpy()\n    return PREDS\n\ndef tta_inference_func(test_loader):\n    model.eval()\n    bar = tqdm(test_loader)\n    PREDS = []\n    LOGITS = []\n\n    with torch.no_grad():\n        for batch_idx, images in enumerate(bar):\n            x = images.to(device)\n            x = torch.stack([x,x.flip(-1)],0) # hflip\n            x = x.view(-1, 3, image_size, image_size)\n            logits = model(x)\n            logits = logits.view(batch_size, 2, -1).mean(1)\n            PREDS += [logits.sigmoid().detach().cpu()]\n            LOGITS.append(logits.cpu())\n        PREDS = torch.cat(PREDS).cpu().numpy()\n        \n    return PREDS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if submit:\n    test_preds = []\n    for i in range(len(enet_type)):\n        if enet_type[i] == 'resnet200d':\n            print('resnet200d loaded')\n            model = RANZCRResNet200D(enet_type[i], out_dim=len(target_cols))\n            model = model.to(device)\n        model.load_state_dict(torch.load(model_path[i], map_location='cuda:0'))\n        if tta:\n            test_preds += [tta_inference_func(test_loader)]\n        else:\n            test_preds += [inference_func(test_loader)]\n\n    submission = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\n    \n    submission[target_cols] = np.mean(test_preds, axis=0)\n    submission.to_csv('submission.csv', index=False)\n    \n    pred_2xTTA965 = np.mean(test_preds, axis=0)\nelse:\n    pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv').to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(pred_2xTTA965).to_csv('pred_2xTTA965.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Multi-Head Model [inference] - 0.964\nhttps://www.kaggle.com/ttahara/ranzcr-multi-head-model-inference?scriptVersionId=55373372"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport sys\nimport time\nimport copy\nimport random\nimport shutil\nimport typing as tp\nfrom pathlib import Path\nfrom argparse import ArgumentParser\n\nimport yaml\nimport numpy as np\nimport pandas as pd\nfrom scipy.sparse import coo_matrix\nfrom sklearn.metrics import roc_auc_score\n\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed\n\nimport cv2\nimport albumentations\n\nfrom albumentations.core.transforms_interface import ImageOnlyTransform, DualTransform\nfrom albumentations.pytorch import ToTensorV2\n\nimport torch\nfrom torch import nn\nfrom torch.utils import data\nfrom torchvision import models as torchvision_models\n\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nimport timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT = ROOT / \"input\"\nOUTPUT = ROOT / \"output\"\nDATA = INPUT / \"ranzcr-clip-catheter-line-classification\"\nTRAIN = DATA / \"train\"\nTEST = DATA / \"test\"\n\n\nTRAINED_MODEL = INPUT / \"ranzcr-clip-weights-for-multi-head-model-v2\"\nTMP = ROOT / \"tmp\"\nTMP.mkdir(exist_ok=True)\n\nRANDAM_SEED = 1086\nN_CLASSES = 11\nFOLDS = [0, 1, 2, 3, 4]\nN_FOLD = len(FOLDS)\nIMAGE_SIZE = (512, 512)\n\nCONVERT_TO_RANK = False\nFAST_COMMIT = False\n\nCLASSES = [\n    'ETT - Abnormal',\n    'ETT - Borderline',\n    'ETT - Normal',\n    'NGT - Abnormal',\n    'NGT - Borderline',\n    'NGT - Incompletely Imaged',\n    'NGT - Normal',\n    'CVC - Abnormal',\n    'CVC - Borderline',\n    'CVC - Normal',\n    'Swan Ganz Catheter Present'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for p in DATA.iterdir():\n    print(p.name)\n\ntrain = pd.read_csv(DATA / \"train.csv\")\nsmpl_sub =  pd.read_csv(DATA / \"sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if FAST_COMMIT and len(smpl_sub) == 3582:\n    smpl_sub = smpl_sub.iloc[:64 * 3].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def multi_label_stratified_group_k_fold(label_arr: np.array, gid_arr: np.array, n_fold: int, seed: int=42):\n    \"\"\"\n    create multi-label stratified group kfold indexs.\n\n    reference: https://www.kaggle.com/jakubwasikowski/stratified-group-k-fold-cross-validation\n    input:\n        label_arr: numpy.ndarray, shape = (n_train, n_class)\n            multi-label for each sample's index using multi-hot vectors\n        gid_arr: numpy.array, shape = (n_train,)\n            group id for each sample's index\n        n_fold: int. number of fold.\n        seed: random seed.\n    output:\n        yield indexs array list for each fold's train and validation.\n    \"\"\"\n    np.random.seed(seed)\n    random.seed(seed)\n    start_time = time.time()\n    n_train, n_class = label_arr.shape\n    gid_unique = sorted(set(gid_arr))\n    n_group = len(gid_unique)\n\n    # # aid_arr: (n_train,), indicates alternative id for group id.\n    # # generally, group ids are not 0-index and continuous or not integer.\n    gid2aid = dict(zip(gid_unique, range(n_group)))\n#     aid2gid = dict(zip(range(n_group), gid_unique))\n    aid_arr = np.vectorize(lambda x: gid2aid[x])(gid_arr)\n\n    # # count labels by class\n    cnts_by_class = label_arr.sum(axis=0)  # (n_class, )\n\n    # # count labels by group id.\n    col, row = np.array(sorted(enumerate(aid_arr), key=lambda x: x[1])).T\n    cnts_by_group = coo_matrix(\n        (np.ones(len(label_arr)), (row, col))\n    ).dot(coo_matrix(label_arr)).toarray().astype(int)\n    del col\n    del row\n    cnts_by_fold = np.zeros((n_fold, n_class), int)\n\n    groups_by_fold = [[] for fid in range(n_fold)]\n    group_and_cnts = list(enumerate(cnts_by_group))  # pair of aid and cnt by group\n    np.random.shuffle(group_and_cnts)\n    print(\"finished preparation\", time.time() - start_time)\n    for aid, cnt_by_g in sorted(group_and_cnts, key=lambda x: -np.std(x[1])):\n        best_fold = None\n        min_eval = None\n        for fid in range(n_fold):\n            # # eval assignment.\n            cnts_by_fold[fid] += cnt_by_g\n            fold_eval = (cnts_by_fold / cnts_by_class).std(axis=0).mean()\n            cnts_by_fold[fid] -= cnt_by_g\n\n            if min_eval is None or fold_eval < min_eval:\n                min_eval = fold_eval\n                best_fold = fid\n\n        cnts_by_fold[best_fold] += cnt_by_g\n        groups_by_fold[best_fold].append(aid)\n    print(\"finished assignment.\", time.time() - start_time)\n\n    gc.collect()\n    idx_arr = np.arange(n_train)\n    for fid in range(n_fold):\n        val_groups = groups_by_fold[fid]\n\n        val_indexs_bool = np.isin(aid_arr, val_groups)\n        train_indexs = idx_arr[~val_indexs_bool]\n        val_indexs = idx_arr[val_indexs_bool]\n\n        print(\"[fold {}]\".format(fid), end=\" \")\n        print(\"n_group: (train, val) = ({}, {})\".format(n_group - len(val_groups), len(val_groups)), end=\" \")\n        print(\"n_sample: (train, val) = ({}, {})\".format(len(train_indexs), len(val_indexs)))\n\n        yield train_indexs, val_indexs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_arr = train[CLASSES].values\ngroup_id = train.PatientID.values\n\ntrain_val_indexs = list(\n    multi_label_stratified_group_k_fold(label_arr, group_id, N_FOLD, RANDAM_SEED))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"fold\"] = -1\nfor fold_id, (trn_idx, val_idx) in enumerate(train_val_indexs):\n    train.loc[val_idx, \"fold\"] = fold_id\n    \ntrain.groupby(\"fold\")[CLASSES].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_images(img_id, input_dir, output_dir, resize_to=(512, 512), ext=\"png\"):\n    img_path = input_dir / f\"{img_id}.jpg\"\n    save_path = output_dir / f\"{img_id}.{ext}\"\n    \n    img = cv2.imread(str(img_path), cv2.IMREAD_GRAYSCALE)\n    img = cv2.resize(img, resize_to)\n    cv2.imwrite(str(save_path), img, )\n\nTEST_RESIZED = TMP / \"test_{0}x{1}\".format(*IMAGE_SIZE)\nTEST_RESIZED.mkdir(exist_ok=True)\nTEST_RESIZED\n\n_ = Parallel(n_jobs=2, verbose=5)([\n    delayed(resize_images)(img_id, TEST, TEST_RESIZED, IMAGE_SIZE, \"png\")\n    for img_id in smpl_sub.StudyInstanceUID.values\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_activation(activ_name: str=\"relu\"):\n    \"\"\"\"\"\"\n    act_dict = {\n        \"relu\": nn.ReLU(inplace=True),\n        \"tanh\": nn.Tanh(),\n        \"sigmoid\": nn.Sigmoid(),\n        \"identity\": nn.Identity()}\n    if activ_name in act_dict:\n        return act_dict[activ_name]\n    else:\n        raise NotImplementedError\n        \n\nclass Conv2dBNActiv(nn.Module):\n    \"\"\"Conv2d -> (BN ->) -> Activation\"\"\"\n    \n    def __init__(\n        self, in_channels: int, out_channels: int,\n        kernel_size: int, stride: int=1, padding: int=0,\n        bias: bool=False, use_bn: bool=True, activ: str=\"relu\"\n    ):\n        \"\"\"\"\"\"\n        super(Conv2dBNActiv, self).__init__()\n        layers = []\n        layers.append(nn.Conv2d(\n            in_channels, out_channels,\n            kernel_size, stride, padding, bias=bias))\n        if use_bn:\n            layers.append(nn.BatchNorm2d(out_channels))\n            \n        layers.append(get_activation(activ))\n        self.layers = nn.Sequential(*layers)\n        \n    def forward(self, x):\n        \"\"\"Forward\"\"\"\n        return self.layers(x)\n        \n\nclass SSEBlock(nn.Module):\n    \"\"\"channel `S`queeze and `s`patial `E`xcitation Block.\"\"\"\n\n    def __init__(self, in_channels: int):\n        \"\"\"Initialize.\"\"\"\n        super(SSEBlock, self).__init__()\n        self.channel_squeeze = nn.Conv2d(\n            in_channels=in_channels, out_channels=1,\n            kernel_size=1, stride=1, padding=0, bias=False)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        \"\"\"Forward.\"\"\"\n        # # x: (bs, ch, h, w) => h: (bs, 1, h, w)\n        h = self.sigmoid(self.channel_squeeze(x))\n        # # x, h => return: (bs, ch, h, w)\n        return x * h\n    \n    \nclass SpatialAttentionBlock(nn.Module):\n    \"\"\"Spatial Attention for (C, H, W) feature maps\"\"\"\n    \n    def __init__(\n        self, in_channels: int,\n        out_channels_list: tp.List[int],\n    ):\n        \"\"\"Initialize\"\"\"\n        super(SpatialAttentionBlock, self).__init__()\n        self.n_layers = len(out_channels_list)\n        channels_list = [in_channels] + out_channels_list\n        assert self.n_layers > 0\n        assert channels_list[-1] == 1\n        \n        for i in range(self.n_layers - 1):\n            in_chs, out_chs = channels_list[i: i + 2]\n            layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ=\"relu\")\n            setattr(self, f\"conv{i + 1}\", layer)\n            \n        in_chs, out_chs = channels_list[-2:]\n        layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ=\"sigmoid\")\n        setattr(self, f\"conv{self.n_layers}\", layer)\n    \n    def forward(self, x):\n        \"\"\"Forward\"\"\"\n        h = x\n        for i in range(self.n_layers):\n            h = getattr(self, f\"conv{i + 1}\")(h)\n            \n        h = h * x\n        return h","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiHeadResNet200D(nn.Module):\n    \n    def __init__(\n        self, out_dims_head: tp.List[int]=[3, 4, 3, 1], pretrained=False\n    ):\n        \"\"\"\"\"\"\n        self.base_name = \"resnet200d_320\"\n        self.n_heads = len(out_dims_head)\n        super(MultiHeadResNet200D, self).__init__()\n        \n        # # load base model\n        base_model = timm.create_model(\n            self.base_name, num_classes=sum(out_dims_head), pretrained=False)\n        in_features = base_model.num_features\n        \n        if pretrained:\n            pretrained_model_path = '../input/startingpointschestx/resnet200d_320_chestx.pth'\n            state_dict = dict()\n            for k, v in torch.load(pretrained_model_path, map_location='cpu')[\"model\"].items():\n                if k[:6] == \"model.\":\n                    k = k.replace(\"model.\", \"\")\n                state_dict[k] = v\n            base_model.load_state_dict(state_dict)\n        \n        # # remove global pooling and head classifier\n        base_model.reset_classifier(0, '')\n        \n        # # Shared CNN Bacbone\n        self.backbone = base_model\n        \n        # # Multi Heads.\n        for i, out_dim in enumerate(out_dims_head):\n            layer_name = f\"head_{i}\"\n            layer = nn.Sequential(\n                SpatialAttentionBlock(in_features, [64, 32, 16, 1]),\n                nn.AdaptiveAvgPool2d(output_size=1),\n                nn.Flatten(start_dim=1),\n                nn.Linear(in_features, in_features),\n                nn.ReLU(inplace=True),\n                nn.Dropout(0.5),\n                nn.Linear(in_features, out_dim))\n            setattr(self, layer_name, layer)\n\n    def forward(self, x):\n        \"\"\"\"\"\"\n        h = self.backbone(x)\n        hs = [\n            getattr(self, f\"head_{i}\")(h) for i in range(self.n_heads)]\n        y = torch.cat(hs, axis=1)\n        return y\n    \n\n## forward test\nm = MultiHeadResNet200D([3, 4, 3, 1], False)\nm = m.eval()\n\nx = torch.rand(1, 3, 256, 256)\nwith torch.no_grad():\n    y = m(x)\nprint(\"[forward test]\")\nprint(\"input:\\t{}\\noutput:\\t{}\".format(x.shape, y.shape))\n\ndel m; del x; del y\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LabeledImageDataset(data.Dataset):\n    \"\"\"\n    Dataset class for (image, label) pairs\n\n    reads images and applys transforms to them.\n\n    Attributes\n    ----------\n    file_list : List[Tuple[tp.Union[str, Path], tp.Union[int, float, np.ndarray]]]\n        list of (image file, label) pair\n    transform_list : List[Dict]\n        list of dict representing image transform \n    \"\"\"\n\n    def __init__(\n        self,\n        file_list: tp.List[\n            tp.Tuple[tp.Union[str, Path], tp.Union[int, float, np.ndarray]]],\n        transform_list: tp.List[tp.Dict],\n    ):\n        \"\"\"Initialize\"\"\"\n        self.file_list = file_list\n        self.transform = ImageTransformForCls(transform_list)\n\n    def __len__(self):\n        \"\"\"Return Num of Images.\"\"\"\n        return len(self.file_list)\n\n    def __getitem__(self, index):\n        \"\"\"Return transformed image and mask for given index.\"\"\"\n        img_path, label = self.file_list[index]\n        img = self._read_image_as_array(img_path)\n        \n        img, label = self.transform((img, label))\n        return img, label\n\n    def _read_image_as_array(self, path: str):\n        \"\"\"Read image file and convert into numpy.ndarray\"\"\"\n        img_arr = cv2.imread(str(path))\n        img_arr = cv2.cvtColor(img_arr, cv2.COLOR_BGR2RGB)\n        return img_arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataloaders_for_inference(\n    file_list: tp.List[tp.List], batch_size=64,\n):\n    \"\"\"Create DataLoader\"\"\"\n    dataset = LabeledImageDataset(\n        file_list,\n        transform_list=[\n          [\"Normalize\", {\n              \"always_apply\": True, \"max_pixel_value\": 255.0,\n              \"mean\": [\"0.4887381077884414\"], \"std\": [\"0.23064819430546407\"]}],\n          [\"ToTensorV2\", {\"always_apply\": True}],\n        ])\n    loader = data.DataLoader(\n        dataset,\n        batch_size=batch_size, shuffle=False,\n        num_workers=2, pin_memory=True,\n        drop_last=False)\n\n    return loader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageTransformBase:\n    \"\"\"\n    Base Image Transform class.\n\n    Args:\n        data_augmentations: List of tuple(method: str, params :dict), each elems pass to albumentations\n    \"\"\"\n\n    def __init__(self, data_augmentations: tp.List[tp.Tuple[str, tp.Dict]]):\n        \"\"\"Initialize.\"\"\"\n        augmentations_list = [\n            self._get_augmentation(aug_name)(**params)\n            for aug_name, params in data_augmentations]\n        self.data_aug = albumentations.Compose(augmentations_list)\n\n    def __call__(self, pair: tp.Tuple[np.ndarray]) -> tp.Tuple[np.ndarray]:\n        \"\"\"You have to implement this by task\"\"\"\n        raise NotImplementedError\n\n    def _get_augmentation(self, aug_name: str) -> tp.Tuple[ImageOnlyTransform, DualTransform]:\n        \"\"\"Get augmentations from albumentations\"\"\"\n        if hasattr(albumentations, aug_name):\n            return getattr(albumentations, aug_name)\n        else:\n            return eval(aug_name)\n\n\nclass ImageTransformForCls(ImageTransformBase):\n    \"\"\"Data Augmentor for Classification Task.\"\"\"\n\n    def __init__(self, data_augmentations: tp.List[tp.Tuple[str, tp.Dict]]):\n        \"\"\"Initialize.\"\"\"\n        super(ImageTransformForCls, self).__init__(data_augmentations)\n\n    def __call__(self, in_arrs: tp.Tuple[np.ndarray]) -> tp.Tuple[np.ndarray]:\n        \"\"\"Apply Transform.\"\"\"\n        img, label = in_arrs\n        augmented = self.data_aug(image=img)\n        img = augmented[\"image\"]\n\n        return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_setting_file(path: str):\n    \"\"\"Load YAML setting file.\"\"\"\n    with open(path) as f:\n        settings = yaml.safe_load(f)\n    return settings\n\n\ndef set_random_seed(seed: int = 42, deterministic: bool = False):\n    \"\"\"Set seeds\"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = deterministic  # type: ignore\n    \n\ndef run_inference_loop(stgs, model, loader, device):\n    model.to(device)\n    model.eval()\n    pred_list = []\n    with torch.no_grad():\n        for x, t in tqdm(loader):\n            y = model(x.to(device))\n            pred_list.append(y.sigmoid().detach().cpu().numpy())\n            # pred_list.append(y.detach().cpu().numpy())\n        \n    pred_arr = np.concatenate(pred_list)\n    del pred_list\n    return pred_arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not torch.cuda.is_available():\n    device = torch.device(\"cpu\")\nelse:\n    device = torch.device(\"cuda\")\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_dir = TRAINED_MODEL\ntest_dir = TEST_RESIZED\n\ntest_file_list = [\n    (test_dir / f\"{img_id}.png\", [-1] * 11)\n    for img_id in smpl_sub[\"StudyInstanceUID\"].values]\ntest_loader = get_dataloaders_for_inference(test_file_list, batch_size=64)\n        \ntest_preds_arr = np.zeros((N_FOLD, len(smpl_sub), N_CLASSES))    \nfor fold_id in FOLDS:\n    print(f\"[fold {fold_id}]\")\n    stgs = load_setting_file(model_dir / f\"fold{fold_id}\" / \"settings.yml\")\n    # # prepare \n    stgs[\"model\"][\"params\"][\"pretrained\"] = False\n    model = MultiHeadResNet200D(**stgs[\"model\"][\"params\"])\n    model_path = model_dir / f\"best_model_fold{fold_id}.pth\"\n    model.load_state_dict(torch.load(model_path, map_location=device))\n\n    # # inference test\n    test_pred = run_inference_loop(stgs, model, test_loader, device)\n    test_preds_arr[fold_id] = test_pred\n    \n    del model\n    torch.cuda.empty_cache()\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if CONVERT_TO_RANK:\n    # # shape: (fold, n_example, class)\n    test_preds_arr = test_preds_arr.argsort(axis=1).argsort(axis=1)\n\nMulti_pred = test_preds_arr.mean(axis=0)\nMulti_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(Multi_pred).to_csv('Multi_pred.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make fin_sub"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = (0.4*pred200d + 0.2*pred152d + 0.4*pred_2xTTA965) * 0.8 + Multi_pred*0.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# init test\ntest = sub_test.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntarget_cols = test.iloc[:, 1:12].columns.tolist()\ntest[target_cols] = predictions\ntest[['StudyInstanceUID'] + target_cols].to_csv('submission.csv', index=False)\ntest.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}