{"cells":[{"metadata":{},"cell_type":"markdown","source":"Original Notebooks:\n- https://www.kaggle.com/ammarali32/resnet200d-inference-single-model-lb-96-5\n- https://www.kaggle.com/ammarali32/seresnet152d-inference-single-model-lb-96-2\n- https://www.kaggle.com/ammarali32/efficientnetb5-inference-single-model"},{"metadata":{},"cell_type":"markdown","source":"# ResNet200D SeResNet152D EfficientNetB5"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path.append('../input/pytorch-images-seresnet')\n\nfrom albumentations import *\nfrom albumentations.pytorch import ToTensorV2\nfrom collections import defaultdict,  Counter\nfrom contextlib import contextmanager\nfrom functools import partial\nfrom matplotlib import pyplot as plt\nfrom pathlib import Path\nfrom PIL import Image\nfrom torch.cuda.amp import autocast,  GradScaler\nfrom torch.nn.parameter import Parameter\nfrom torch.optim import Adam,  SGD\nfrom torch.utils.data import DataLoader,  Dataset\nfrom tqdm.auto import tqdm\nimport cv2\nimport gc\nimport math\nimport numpy as np\nimport pandas as pd\nimport random\nimport scipy as sp\nimport shutil\nimport time\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as models\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"IMAGE_SIZE = 640\nBATCH_SIZE = 64\nTEST_PATH = '../input/ranzcr-clip-catheter-line-classification/test'\nMODEL_PATH_resnet200d = '../input/resnet200d-public/resnet200d_320_CV9632.pth'\nMODEL_PATH_seresnet152d = '../input/seresnet152d-cv9615/seresnet152d_320_CV96.15.pth'\nMODEL_PATH_efficientnet_b5 = '../input/efficientnetb5cv9621/tf_efficientnet_b5_ns_CV96.21.pth'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_transforms():\n    return Compose([\n        Resize(IMAGE_SIZE, IMAGE_SIZE),\n        Normalize(),\n        ToTensorV2(),\n    ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResNet200D(nn.Module):\n    def __init__(self, model_name='resnet200d_320'):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, 11)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output\n\nclass SeResNet152D(nn.Module):\n    def __init__(self, model_name='seresnet152d_320'):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, 11)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output\n    \nclass EfficientNetB5(nn.Module):\n    def __init__(self, model_name='tf_efficientnet_b5_ns'):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        n_features = self.model.classifier.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.classifier = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.classifier = nn.Linear(n_features, 11)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.classifier(pooled_features)\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference(models, test_loader, device):\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        for model in models:\n            with torch.no_grad():\n                y_preds1 = model(images)\n                y_preds2 = model(images.flip(-1))\n            y_preds = (y_preds1.sigmoid().to('cpu').numpy() + y_preds2.sigmoid().to('cpu').numpy()) / 2\n            avg_preds.append(y_preds)\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models200D = []\nmodel = ResNet200D()\nmodel.load_state_dict(torch.load(MODEL_PATH_resnet200d)['model'])\nmodel.eval()\nmodel.to(device)\nmodels200D.append(model)\n\nmodels152D = []\nmodel = SeResNet152D()\nmodel.load_state_dict(torch.load(MODEL_PATH_seresnet152d)['model'])\nmodel.eval()\nmodel.to(device)\nmodels152D.append(model)\n\nmodelsB5 = []\nmodel = EfficientNetB5()\nmodel.load_state_dict(torch.load(MODEL_PATH_efficientnet_b5)['model'])\nmodel.eval()\nmodel.to(device)\nmodelsB5.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = TestDataset(test, transform=get_transforms())\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, \n                         num_workers=4 , pin_memory=True)\n\npredictions200d = inference(models200D, test_loader, device)\ndel models200D\ngc.collect()\n\npredictions152d = inference(models152D, test_loader, device)\ndel models152D\ngc.collect()\n\npredictionsB5 = inference(modelsB5, test_loader, device)\n\n# predictions = 0.5 * (predictions200d ** 0.5) + 0.2 * (predictions152d ** 0.5) + 0.3 * (predictionsB5 ** 0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# target_cols = test.iloc[:, 1:12].columns.tolist()\n# test[target_cols] = predictions\n# test[['StudyInstanceUID'] + target_cols].to_csv('submission.csv', index=False)\n# test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Clean Up Variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"sys.path.remove('../input/pytorch-images-seresnet')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"variable_list = %who_ls\nfor _ in variable_list:\n    if _ not in (\"predictions200d\", \"predictions152d\", \"predictionsB5\"):\n        del globals()[_]\n\n%who_ls","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<br>\n<br>\n<br>\n<br>\n\n# Yet another ResNet200D\n\nOriginal Notebook: https://www.kaggle.com/underwearfitting/resnet200d-public-benchmark-2xtta-lb0-965"},{"metadata":{},"cell_type":"markdown","source":"### Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 1\nimage_size = 512\ntta = True\nenet_type = ['resnet200d'] * 5\nmodel_path = ['../input/resnet200d-baseline-benchmark-public/resnet200d_fold0_cv953.pth',\n              '../input/resnet200d-baseline-benchmark-public/resnet200d_fold1_cv955.pth',\n              '../input/resnet200d-baseline-benchmark-public/resnet200d_fold2_cv955.pth',\n              '../input/resnet200d-baseline-benchmark-public/resnet200d_fold3_cv957.pth',\n              '../input/resnet200d-baseline-benchmark-public/resnet200d_fold4_cv954.pth']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport numpy as np\n%matplotlib inline\nDEBUG = False\nfrom albumentations import *\nfrom albumentations.pytorch import ToTensorV2\nfrom pylab import rcParams\nfrom sklearn.metrics import accuracy_score\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm.notebook import tqdm\nimport albumentations\nimport cv2\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport PIL.Image\nimport seaborn as sns\nimport time\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torchvision\nimport torchvision.transforms as transforms\ndevice = torch.device('cuda') if not DEBUG else torch.device('cpu')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZCRResNet200D(nn.Module):\n    def __init__(self, model_name='resnet200d', out_dim=11, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, out_dim)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_test = albumentations.Compose([\n    Resize(image_size, image_size),\n    Normalize(\n         mean=[0.485, 0.456, 0.406],\n         std=[0.229, 0.224, 0.225],\n     ),\n    ToTensorV2()\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZCRDataset(Dataset):\n    def __init__(self, df, mode, transform=None):\n        \n        self.df = df.reset_index(drop=True)\n        self.mode = mode\n        self.transform = transform\n        self.labels = df[target_cols].values\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        row = self.df.loc[index]\n        img = cv2.imread(row.file_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        if self.transform is not None:\n            res = self.transform(image=img)\n            img = res['image']\n        label = torch.tensor(self.labels[index]).float()\n        if self.mode == 'test':\n            return img\n        else:\n            return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\ntest['file_path'] = test.StudyInstanceUID.apply(\n    lambda x: os.path.join('../input/ranzcr-clip-catheter-line-classification/test', f'{x}.jpg')\n)\ntarget_cols = test.iloc[:, 1:12].columns.tolist()\ntest_dataset = RANZCRDataset(test, 'test', transform=transforms_test)\ntest_loader = torch.utils.data.DataLoader(\n    test_dataset, batch_size=batch_size, shuffle=False, num_workers=24\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_func(test_loader):\n    model.eval()\n    bar = tqdm(test_loader)\n    LOGITS = []\n    PREDS = []\n    \n    with torch.no_grad():\n        for batch_idx, images in enumerate(bar):\n            x = images.to(device)\n            logits = model(x)\n            LOGITS.append(logits.cpu())\n            PREDS += [logits.sigmoid().detach().cpu()]\n        PREDS = torch.cat(PREDS).cpu().numpy()\n        LOGITS = torch.cat(LOGITS).cpu().numpy()\n    return PREDS\n\ndef tta_inference_func(test_loader):\n    model.eval()\n    bar = tqdm(test_loader)\n    PREDS = []\n    LOGITS = []\n\n    with torch.no_grad():\n        for batch_idx, images in enumerate(bar):\n            x = images.to(device)\n            x = torch.stack([x,x.flip(-1)],0) # hflip\n            x = x.view(-1, 3, image_size, image_size)\n            logits = model(x)\n            logits = logits.view(batch_size, 2, -1).mean(1)\n            PREDS += [logits.sigmoid().detach().cpu()]\n            LOGITS.append(logits.cpu())\n        PREDS = torch.cat(PREDS).cpu().numpy()\n        \n    return PREDS","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = []\nfor i in range(len(enet_type)):\n    if enet_type[i] == 'resnet200d':\n        print('resnet200d loaded')\n        model = RANZCRResNet200D(enet_type[i], out_dim=len(target_cols))\n        model = model.to(device)\n    model.load_state_dict(torch.load(model_path[i], map_location='cuda:0'))\n    if tta:\n        test_preds += [tta_inference_func(test_loader)]\n    else:\n        test_preds += [inference_func(test_loader)]\n\n# submission = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\n# submission[target_cols] = np.mean(test_preds, axis=0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Clean Up Variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"sys.path.remove('../input/pytorch-image-models/pytorch-image-models-master')\nsys.path.remove('../input/timm-pytorch-image-models/pytorch-image-models-master')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"variable_list = %who_ls\nfor _ in variable_list:\n    if _ not in (\"test_preds\", \"predictions200d\", \"predictions152d\", \"predictionsB5\"):\n        del globals()[_]\n\n%who_ls","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<br>\n<br>\n<br>\n<br>\n\n# EfficientNetB5 Tez framework\n\nOriginal Notebook: https://www.kaggle.com/abhishek/ranzcr-tez-inference-efficientnet5-512-tta"},{"metadata":{"trusted":true},"cell_type":"code","source":"tez_path = '../input/tez-lib/'\neffnet_path = '../input/efficientnet-pytorch/'\nimport sys\nsys.path.append(tez_path)\nsys.path.append(effnet_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport albumentations\nimport pandas as pd\nimport numpy as np\n\nimport tez\nfrom tez.datasets import ImageDataset\n\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\n\nfrom efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_PATH = \"../input/ranzcr-clip-catheter-line-classification/\"\nIMAGE_PATH = \"../input/ranzcr-clip-catheter-line-classification/test/\"\nMODEL_PATH = \"../input/ranzcr-effnet5/\"\nIMAGE_SIZE = 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(os.path.join(INPUT_PATH, \"sample_submission.csv\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RanzcrModel(tez.Model):\n    def __init__(self):\n        super().__init__()\n\n        self.effnet = EfficientNet.from_name(\"efficientnet-b5\")\n\n        self.effnet._conv_stem.in_channels = 1\n        weight = self.effnet._conv_stem.weight.mean(1, keepdim=True)\n        self.effnet._conv_stem.weight = torch.nn.Parameter(weight)\n\n        self.dropout = nn.Dropout(0.1)\n        self.out = nn.Linear(2048, 11)\n\n    def forward(self, image, targets=None):\n        batch_size, _, _, _ = image.shape\n\n        x = self.effnet.extract_features(image)\n        x = F.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)\n        outputs = self.out(self.dropout(x))\n        return outputs, None, {}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_aug = albumentations.Compose(\n    [\n        albumentations.Resize(IMAGE_SIZE, IMAGE_SIZE, p=1.0),\n        albumentations.HorizontalFlip(p=0.5),\n        albumentations.Normalize(\n            mean=[0.485],\n            std=[0.229],\n            max_pixel_value=255.0,\n            p=1.0,\n        ),\n    ],\n    p=1.0,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image_paths = [\n    os.path.join(IMAGE_PATH, x + \".jpg\") \n    for x in df.StudyInstanceUID.values\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = RanzcrModel()\nmodel.load(os.path.join(MODEL_PATH, \"effnet5_fold_0.bin\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_preds = None\nfor j in range(2):\n    test_dataset = ImageDataset(\n        image_paths=test_image_paths,\n        targets=[0]*len(test_image_paths),\n        augmentations=test_aug,\n        grayscale=True,\n    )\n    preds = model.predict(test_dataset, batch_size=32, n_jobs=-1)\n    temp_preds = None\n    for p in preds:\n        if temp_preds is None:\n            temp_preds = p\n        else:\n            temp_preds = np.vstack((temp_preds, p))\n    if final_preds is None:\n        final_preds = temp_preds\n    else:\n        final_preds += temp_preds\nfinal_preds /= 2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Clean Up Variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"sys.path.remove(tez_path)\nsys.path.remove(effnet_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"variable_list = %who_ls\nfor _ in variable_list:\n    if _ not in (\"test_preds\", \"predictions200d\", \"predictions152d\", \"predictionsB5\", \"final_preds\"):\n        del globals()[_]\n\n%who_ls","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<br>\n<br>\n<br>\n<br>\n\n# ResNet200D Multi-head\n\nOriginal Notebook: https://www.kaggle.com/ttahara/ranzcr-multi-head-model-inference"},{"metadata":{},"cell_type":"markdown","source":"### Import"},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations.core.transforms_interface import ImageOnlyTransform, DualTransform\nfrom albumentations.pytorch import ToTensorV2\nfrom argparse import ArgumentParser\nfrom joblib import Parallel, delayed\nfrom pathlib import Path\nfrom scipy.sparse import coo_matrix\nfrom sklearn.metrics import roc_auc_score\nfrom torch import nn\nfrom torch.utils import data\nfrom torchvision import models as torchvision_models\nfrom tqdm import tqdm\nimport albumentations\nimport copy\nimport cv2\nimport gc\nimport numpy as np\nimport os\nimport pandas as pd\nimport random\nimport shutil\nimport sys\nimport time\nimport torch\nimport typing as tp\nimport yaml\n\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nimport timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT = ROOT / \"input\"\nOUTPUT = ROOT / \"output\"\nDATA = INPUT / \"ranzcr-clip-catheter-line-classification\"\nTRAIN = DATA / \"train\"\nTEST = DATA / \"test\"\nTRAINED_MODEL = INPUT / \"ranzcr-clip-weights-for-multi-head-model-v2\"\nTMP = ROOT / \"tmp\"\nTMP.mkdir(exist_ok=True)\n\nRANDAM_SEED = 1086\nN_CLASSES = 11\nFOLDS = [0, 1, 2, 3, 4]\nN_FOLD = len(FOLDS)\nIMAGE_SIZE = (512, 512)\n\nFAST_COMMIT = False\n\nCLASSES = [\n    'ETT - Abnormal',\n    'ETT - Borderline',\n    'ETT - Normal',\n    'NGT - Abnormal',\n    'NGT - Borderline',\n    'NGT - Incompletely Imaged',\n    'NGT - Normal',\n    'CVC - Abnormal',\n    'CVC - Borderline',\n    'CVC - Normal',\n    'Swan Ganz Catheter Present'\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Read data"},{"metadata":{"trusted":true},"cell_type":"code","source":"for p in DATA.iterdir():\n    print(p.name)\n\ntrain = pd.read_csv(DATA / \"train.csv\")\nsmpl_sub =  pd.read_csv(DATA / \"sample_submission.csv\")\nsmpl_sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if FAST_COMMIT and len(smpl_sub) == 3582:\n    smpl_sub = smpl_sub.iloc[:64 * 3].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Split fold"},{"metadata":{"trusted":true},"cell_type":"code","source":"def multi_label_stratified_group_k_fold(label_arr: np.array, gid_arr: np.array, n_fold: int, seed: int=42):\n    \"\"\"\n    create multi-label stratified group kfold indexs.\n\n    reference: https://www.kaggle.com/jakubwasikowski/stratified-group-k-fold-cross-validation\n    input:\n        label_arr: numpy.ndarray, shape = (n_train, n_class)\n            multi-label for each sample's index using multi-hot vectors\n        gid_arr: numpy.array, shape = (n_train,)\n            group id for each sample's index\n        n_fold: int. number of fold.\n        seed: random seed.\n    output:\n        yield indexs array list for each fold's train and validation.\n    \"\"\"\n    np.random.seed(seed)\n    random.seed(seed)\n    start_time = time.time()\n    n_train, n_class = label_arr.shape\n    gid_unique = sorted(set(gid_arr))\n    n_group = len(gid_unique)\n\n    # aid_arr: (n_train,), indicates alternative id for group id\n    # generally, group ids are not 0-index and continuous or not integer\n    gid2aid = dict(zip(gid_unique, range(n_group)))\n    # aid2gid = dict(zip(range(n_group), gid_unique))\n    aid_arr = np.vectorize(lambda x: gid2aid[x])(gid_arr)\n\n    # count labels by class\n    cnts_by_class = label_arr.sum(axis=0)  # (n_class, )\n\n    # count labels by group id\n    col, row = np.array(sorted(enumerate(aid_arr), key=lambda x: x[1])).T\n    cnts_by_group = coo_matrix(\n        (np.ones(len(label_arr)), (row, col))\n    ).dot(coo_matrix(label_arr)).toarray().astype(int)\n    del col\n    del row\n    cnts_by_fold = np.zeros((n_fold, n_class), int)\n\n    groups_by_fold = [[] for fid in range(n_fold)]\n    group_and_cnts = list(enumerate(cnts_by_group)) # pair of aid and cnt by group\n    np.random.shuffle(group_and_cnts)\n    print(\"finished preparation\", time.time() - start_time)\n    for aid, cnt_by_g in sorted(group_and_cnts, key=lambda x: -np.std(x[1])):\n        best_fold = None\n        min_eval = None\n        for fid in range(n_fold):\n            # eval assignment\n            cnts_by_fold[fid] += cnt_by_g\n            fold_eval = (cnts_by_fold / cnts_by_class).std(axis=0).mean()\n            cnts_by_fold[fid] -= cnt_by_g\n\n            if min_eval is None or fold_eval < min_eval:\n                min_eval = fold_eval\n                best_fold = fid\n\n        cnts_by_fold[best_fold] += cnt_by_g\n        groups_by_fold[best_fold].append(aid)\n    print(\"finished assignment.\", time.time() - start_time)\n\n    gc.collect()\n    idx_arr = np.arange(n_train)\n    for fid in range(n_fold):\n        val_groups = groups_by_fold[fid]\n        val_indexs_bool = np.isin(aid_arr, val_groups)\n        train_indexs = idx_arr[~val_indexs_bool]\n        val_indexs = idx_arr[val_indexs_bool]\n\n        print(\"[fold {}]\".format(fid), end=\" \")\n        print(\"n_group: (train, val) = ({}, {})\".format(n_group - len(val_groups), len(val_groups)), end=\" \")\n        print(\"n_sample: (train, val) = ({}, {})\".format(len(train_indexs), len(val_indexs)))\n\n        yield train_indexs, val_indexs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_arr = train[CLASSES].values\ngroup_id = train.PatientID.values\n\ntrain_val_indexs = list(\n    multi_label_stratified_group_k_fold(\n        label_arr, group_id, N_FOLD, RANDAM_SEED\n    )\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"fold\"] = -1\nfor fold_id, (trn_idx, val_idx) in enumerate(train_val_indexs):\n    train.loc[val_idx, \"fold\"] = fold_id\n    \ntrain.groupby(\"fold\")[CLASSES].sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Pre-process test images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_images(img_id, input_dir, output_dir, resize_to=(512, 512), ext=\"png\"):\n    img_path = input_dir / f\"{img_id}.jpg\"\n    save_path = output_dir / f\"{img_id}.{ext}\"\n    \n    img = cv2.imread(str(img_path), cv2.IMREAD_GRAYSCALE)\n    img = cv2.resize(img, resize_to)\n    cv2.imwrite(str(save_path), img, )\n\nTEST_RESIZED = TMP / \"test_{0}x{1}\".format(*IMAGE_SIZE)\nTEST_RESIZED.mkdir(exist_ok=True)\nTEST_RESIZED\n\n_ = Parallel(n_jobs=2, verbose=5)([\n    delayed(resize_images)(img_id, TEST, TEST_RESIZED, IMAGE_SIZE, \"png\")\n    for img_id in smpl_sub.StudyInstanceUID.values\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inference"},{"metadata":{},"cell_type":"markdown","source":"### Custom model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_activation(activ_name: str=\"relu\"):\n    act_dict = {\n        \"relu\": nn.ReLU(inplace=True),\n        \"tanh\": nn.Tanh(),\n        \"sigmoid\": nn.Sigmoid(),\n        \"identity\": nn.Identity()}\n    if activ_name in act_dict:\n        return act_dict[activ_name]\n    else:\n        raise NotImplementedError\n        \n\nclass Conv2dBNActiv(nn.Module):\n    \"\"\"Conv2d -> (BN ->) -> Activation\"\"\"\n    \n    def __init__(\n        self, in_channels: int, out_channels: int,\n        kernel_size: int, stride: int=1, padding: int=0,\n        bias: bool=False, use_bn: bool=True, activ: str=\"relu\"\n    ):\n        super(Conv2dBNActiv, self).__init__()\n        layers = []\n        layers.append(nn.Conv2d(\n            in_channels, out_channels,\n            kernel_size, stride, padding, bias=bias))\n        if use_bn:\n            layers.append(nn.BatchNorm2d(out_channels))\n            \n        layers.append(get_activation(activ))\n        self.layers = nn.Sequential(*layers)\n        \n    def forward(self, x):\n        \"\"\"Forward\"\"\"\n        return self.layers(x)\n    \n    \nclass SpatialAttentionBlock(nn.Module):\n    \"\"\"Spatial Attention for (C, H, W) feature maps\"\"\"\n    \n    def __init__(\n        self, in_channels: int,\n        out_channels_list: tp.List[int],\n    ):\n        \"\"\"Initialize\"\"\"\n        super(SpatialAttentionBlock, self).__init__()\n        self.n_layers = len(out_channels_list)\n        channels_list = [in_channels] + out_channels_list\n        assert self.n_layers > 0\n        assert channels_list[-1] == 1\n        \n        for i in range(self.n_layers - 1):\n            in_chs, out_chs = channels_list[i: i + 2]\n            layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ=\"relu\")\n            setattr(self, f\"conv{i + 1}\", layer)\n            \n        in_chs, out_chs = channels_list[-2:]\n        layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ=\"sigmoid\")\n        setattr(self, f\"conv{self.n_layers}\", layer)\n    \n    def forward(self, x):\n        \"\"\"Forward\"\"\"\n        h = x\n        for i in range(self.n_layers):\n            h = getattr(self, f\"conv{i + 1}\")(h)\n            \n        h = h * x\n        return h","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiHeadResNet200D(nn.Module):\n    \n    def __init__(\n        self, out_dims_head: tp.List[int]=[3, 4, 3, 1], pretrained=False\n    ):\n        self.base_name = \"resnet200d_320\"\n        self.n_heads = len(out_dims_head)\n        super(MultiHeadResNet200D, self).__init__()\n        \n        # load base model\n        base_model = timm.create_model(\n            self.base_name, num_classes=sum(out_dims_head), pretrained=False)\n        in_features = base_model.num_features\n        \n        if pretrained:\n            pretrained_model_path = '../input/startingpointschestx/resnet200d_320_chestx.pth'\n            state_dict = dict()\n            for k, v in torch.load(pretrained_model_path, map_location='cpu')[\"model\"].items():\n                if k[:6] == \"model.\":\n                    k = k.replace(\"model.\", \"\")\n                state_dict[k] = v\n            base_model.load_state_dict(state_dict)\n        \n        # remove global pooling and head classifier\n        base_model.reset_classifier(0, '')\n        \n        # Shared CNN Bacbone\n        self.backbone = base_model\n        \n        # Multi Heads.\n        for i, out_dim in enumerate(out_dims_head):\n            layer_name = f\"head_{i}\"\n            layer = nn.Sequential(\n                SpatialAttentionBlock(in_features, [64, 32, 16, 1]),\n                nn.AdaptiveAvgPool2d(output_size=1),\n                nn.Flatten(start_dim=1),\n                nn.Linear(in_features, in_features),\n                nn.ReLU(inplace=True),\n                nn.Dropout(0.5),\n                nn.Linear(in_features, out_dim))\n            setattr(self, layer_name, layer)\n\n    def forward(self, x):\n        h = self.backbone(x)\n        hs = [\n            getattr(self, f\"head_{i}\")(h) for i in range(self.n_heads)]\n        y = torch.cat(hs, axis=1)\n        return y\n    \n\n# forward test\nm = MultiHeadResNet200D([3, 4, 3, 1], False)\nm = m.eval()\n\nx = torch.rand(1, 3, 256, 256)\nwith torch.no_grad():\n    y = m(x)\nprint(\"[forward test]\")\nprint(\"input:\\t{}\\noutput:\\t{}\".format(x.shape, y.shape))\n\ndel m; del x; del y\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class LabeledImageDataset(data.Dataset):\n    \"\"\"\n    Dataset class for (image, label) pairs\n\n    reads images and applys transforms to them.\n\n    Attributes\n    ----------\n    file_list : List[Tuple[tp.Union[str, Path], tp.Union[int, float, np.ndarray]]]\n        list of (image file, label) pair\n    transform_list : List[Dict]\n        list of dict representing image transform \n    \"\"\"\n\n    def __init__(\n        self,\n        file_list: tp.List[\n            tp.Tuple[tp.Union[str, Path], tp.Union[int, float, np.ndarray]]\n        ],\n        transform_list: tp.List[tp.Dict],\n    ):\n        \"\"\"Initialize\"\"\"\n        self.file_list = file_list\n        self.transform = ImageTransformForCls(transform_list)\n\n    def __len__(self):\n        \"\"\"Return Num of Images.\"\"\"\n        return len(self.file_list)\n\n    def __getitem__(self, index):\n        \"\"\"Return transformed image and mask for given index.\"\"\"\n        img_path, label = self.file_list[index]\n        img = self._read_image_as_array(img_path)\n        \n        img, label = self.transform((img, label))\n        return img, label\n\n    def _read_image_as_array(self, path: str):\n        \"\"\"Read image file and convert into numpy.ndarray\"\"\"\n        img_arr = cv2.imread(str(path))\n        img_arr = cv2.cvtColor(img_arr, cv2.COLOR_BGR2RGB)\n        return img_arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataloaders_for_inference(\n    file_list: tp.List[tp.List], batch_size=64\n):\n    \"\"\"Create DataLoader\"\"\"\n    dataset = LabeledImageDataset(\n        file_list,\n        transform_list=[\n            [\n                \"Normalize\",\n                {\n                    \"always_apply\": True,\n                    \"max_pixel_value\": 255.0,\n                    \"mean\": [\"0.4887381077884414\"],\n                    \"std\": [\"0.23064819430546407\"]\n                }\n            ],\n            [\"ToTensorV2\", {\"always_apply\": True}],\n        ]\n    )\n\n    loader = data.DataLoader(\n        dataset,\n        batch_size=batch_size,\n        shuffle=False,\n        num_workers=2,\n        pin_memory=True,\n        drop_last=False\n    )\n\n    return loader","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Image transform"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageTransformBase:\n    \"\"\"\n    Base Image Transform class.\n\n    Args:\n        data_augmentations:\n        List of tuple(method: str, params :dict). Each elems pass to albumentations\n    \"\"\"\n\n    def __init__(self, data_augmentations: tp.List[tp.Tuple[str, tp.Dict]]):\n        \"\"\"Initialize.\"\"\"\n        augmentations_list = [\n            self._get_augmentation(aug_name)(**params)\n            for aug_name, params in data_augmentations]\n        self.data_aug = albumentations.Compose(augmentations_list)\n\n    def __call__(self, pair: tp.Tuple[np.ndarray]) -> tp.Tuple[np.ndarray]:\n        \"\"\"You have to implement this by task\"\"\"\n        raise NotImplementedError\n\n    def _get_augmentation(self, aug_name: str) -> tp.Tuple[ImageOnlyTransform, DualTransform]:\n        \"\"\"Get augmentations from albumentations\"\"\"\n        if hasattr(albumentations, aug_name):\n            return getattr(albumentations, aug_name)\n        else:\n            return eval(aug_name)\n\n\nclass ImageTransformForCls(ImageTransformBase):\n    \"\"\"Data Augmentor for Classification Task.\"\"\"\n\n    def __init__(self, data_augmentations: tp.List[tp.Tuple[str, tp.Dict]]):\n        \"\"\"Initialize.\"\"\"\n        super(ImageTransformForCls, self).__init__(data_augmentations)\n\n    def __call__(self, in_arrs: tp.Tuple[np.ndarray]) -> tp.Tuple[np.ndarray]:\n        \"\"\"Apply Transform.\"\"\"\n        img, label = in_arrs\n        augmented = self.data_aug(image=img)\n        img = augmented[\"image\"]\n\n        return img, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_setting_file(path: str):\n    \"\"\"Load YAML setting file.\"\"\"\n    with open(path) as f:\n        settings = yaml.safe_load(f)\n    return settings\n\n\ndef set_random_seed(seed: int = 42, deterministic: bool = False):\n    \"\"\"Set seeds\"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed) # type: ignore\n    torch.backends.cudnn.deterministic = deterministic # type: ignore\n    \n\n# def run_inference_loop(stgs, model, loader, device):\n#     model.to(device)\n#     model.eval()\n#     pred_list = []\n#     with torch.no_grad():\n#         for x, t in tqdm(loader):\n#             y = model(x.to(device))\n#             pred_list.append(y.sigmoid().detach().cpu().numpy())\n#             # pred_list.append(y.detach().cpu().numpy())\n        \n#     pred_arr = np.concatenate(pred_list)\n#     del pred_list\n#     return pred_arr\n\ndef run_inference_loop(stgs, model, loader, device):\n    model.to(device)\n    model.eval()\n    pred_list = []\n    with torch.no_grad():\n        for x, t in tqdm(loader):\n            y_1 = model(x.to(device))\n            y_2 = model(x.to(device).flip(-1))\n            y_preds = (\n                y_1.sigmoid().detach().cpu().numpy() +\n                y_2.sigmoid().detach().cpu().numpy()\n            ) / 2\n            pred_list.append(y_preds)\n\n    pred_arr = np.concatenate(pred_list)\n    del pred_list\n    return pred_arr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inference test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"if not torch.cuda.is_available():\n    device = torch.device(\"cpu\")\nelse:\n    device = torch.device(\"cuda\")\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_dir = TRAINED_MODEL\ntest_dir = TEST_RESIZED\n\ntest_file_list = [\n    (test_dir / f\"{img_id}.png\", [-1] * 11)\n    for img_id in smpl_sub[\"StudyInstanceUID\"].values\n]\n\ntest_loader = get_dataloaders_for_inference(test_file_list, batch_size=64)\n\ntest_preds_arr = np.zeros((N_FOLD, len(smpl_sub), N_CLASSES))\n\nfor fold_id in FOLDS:\n    print(f\"[fold {fold_id}]\")\n    stgs = load_setting_file(model_dir / f\"fold{fold_id}\" / \"settings.yml\")\n    # prepare \n    stgs[\"model\"][\"params\"][\"pretrained\"] = False\n    model = MultiHeadResNet200D(**stgs[\"model\"][\"params\"])\n    model_path = model_dir / f\"best_model_fold{fold_id}.pth\"\n    model.load_state_dict(torch.load(model_path, map_location=device))\n\n    # inference test\n    test_pred = run_inference_loop(stgs, model, test_loader, device)\n    test_preds_arr[fold_id] = test_pred\n\n    del model\n    torch.cuda.empty_cache()\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<br>\n<br>\n<br>\n<br>\n\n# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = smpl_sub.copy()\n\n# Convert to rank\nsub[CLASSES] = (\n    0.08 * final_preds.argsort(axis=0).argsort(axis=0)\n    + 0.03 * test_preds[0].argsort(axis=0).argsort(axis=0)\n    + 0.03 * test_preds[1].argsort(axis=0).argsort(axis=0)\n    + 0.03 * test_preds[2].argsort(axis=0).argsort(axis=0)\n    + 0.03 * test_preds[3].argsort(axis=0).argsort(axis=0)\n    + 0.03 * test_preds[4].argsort(axis=0).argsort(axis=0)\n    + 0.08 * (test_preds_arr[0].argsort(axis=0).argsort(axis=0))\n    + 0.08 * (test_preds_arr[1].argsort(axis=0).argsort(axis=0))\n    + 0.08 * (test_preds_arr[2].argsort(axis=0).argsort(axis=0))\n    + 0.08 * (test_preds_arr[3].argsort(axis=0).argsort(axis=0))\n    + 0.08 * (test_preds_arr[4].argsort(axis=0).argsort(axis=0))\n    + 0.45 * (predictions200d.argsort(axis=0).argsort(axis=0))\n    + 0.20 * (predictions152d.argsort(axis=0).argsort(axis=0))\n    + 0.18 * (predictionsB5.argsort(axis=0).argsort(axis=0))\n)\n# sub[CLASSES] = (\n#     0.40 * (test_preds_arr.argsort(axis=1).argsort(axis=1).mean(axis=0))\n#     + 0.50 * (predictions200d.argsort(axis=0).argsort(axis=0))\n#     + 0.17 * (predictions152d.argsort(axis=0).argsort(axis=0))\n#     + 0.15 * (predictionsB5.argsort(axis=0).argsort(axis=0))\n# )\n\n# Not convert to rank \n# sub[CLASSES] = (\n#     0.50 * (test_preds_arr.mean(axis=0) ** 0.5)\n#     + 0.30 * (predictions200d ** 0.5)\n#     + 0.12 * (predictions152d ** 0.5)\n#     + 0.18 * (predictionsB5 ** 0.5)\n# )\n\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}