{"cells":[{"metadata":{},"cell_type":"markdown","source":"# About\n\nI'm trying the suggested concept in this topic:  \nhttps://www.kaggle.com/c/ranzcr-clip-catheter-line-classification/discussion/205208\n\nI share all the experimental results in this topic:  \nhttps://www.kaggle.com/c/ranzcr-clip-catheter-line-classification/discussion/207230\n\nThis is an **inference notebook**.  \n* model:\n    * base: RegNetY_032\n    * multi-head:  independent **Spatial-Attention Module** and MLP by Target Group(ETT(3), NGT(4), CVC(3), and Swan(1))\n* image size: 3x**640x640**\n* CVStrategy: Multi-Label Stratified Group KFold(K=**5**)\n* using mixed precision training\n* add more augmentations to version 15\n\nIf you want to know more details of experimental settings, see training notebook: \n[fold0](https://www.kaggle.com/ttahara/ranzcr-multi-head-model-training?scriptVersionId=53586035), \n[fold1](https://www.kaggle.com/ttahara/ranzcr-multi-head-model-training?scriptVersionId=53589585), \n[fold2](https://www.kaggle.com/ttahara/ranzcr-multi-head-model-training?scriptVersionId=53607319), \n[fold3](https://www.kaggle.com/ttahara/ranzcr-multi-head-model-training?scriptVersionId=53607347), \n[fold4](https://www.kaggle.com/ttahara/ranzcr-multi-head-model-training?scriptVersionId=53621592)"},{"metadata":{},"cell_type":"markdown","source":"# Prepare"},{"metadata":{},"cell_type":"markdown","source":"## import"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport sys\nimport time\nimport copy\nimport random\nimport shutil\nimport typing as tp\nfrom pathlib import Path\nfrom argparse import ArgumentParser\n\nimport yaml\nimport numpy as np\nimport pandas as pd\nfrom scipy.sparse import coo_matrix\nfrom sklearn.metrics import roc_auc_score\n\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed\n\nimport cv2\nimport albumentations\n\nfrom albumentations.core.transforms_interface import ImageOnlyTransform, DualTransform\nfrom albumentations.pytorch import ToTensorV2\n\nimport torch\nfrom torch import nn\nfrom torch.utils import data\nfrom torchvision import models as torchvision_models\n\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nimport timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT = ROOT / \"input\"\nOUTPUT = ROOT / \"output\"\nDATA = INPUT / \"ranzcr-clip-catheter-line-classification\"\nTRAIN = DATA / \"train\"\nTEST = DATA / \"test\"\n\n\nTRAINED_MODEL = INPUT / \"multihead-version18-best-model\"\nTMP = ROOT / \"tmp\"\nTMP.mkdir(exist_ok=True)\n\nRANDAM_SEED = 1086\nN_CLASSES = 11\nFOLDS = [0, 1, 2, 3, 4]\nN_FOLD = len(FOLDS)\nIMAGE_SIZE = (640, 640)\n\nCONVERT_TO_RANK = True\nFAST_COMMIT = False\n\nCLASSES = [\n    'ETT - Abnormal',\n    'ETT - Borderline',\n    'ETT - Normal',\n    'NGT - Abnormal',\n    'NGT - Borderline',\n    'NGT - Incompletely Imaged',\n    'NGT - Normal',\n    'CVC - Abnormal',\n    'CVC - Borderline',\n    'CVC - Normal',\n    'Swan Ganz Catheter Present'\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## read data"},{"metadata":{"trusted":true},"cell_type":"code","source":"for p in DATA.iterdir():\n    print(p.name)\n\ntrain = pd.read_csv(DATA / \"train.csv\")\nsmpl_sub =  pd.read_csv(DATA / \"sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"smpl_sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if FAST_COMMIT and len(smpl_sub) == 3582:\n    smpl_sub = smpl_sub.iloc[:64 * 3].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## split fold"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def multi_label_stratified_group_k_fold(label_arr: np.array, gid_arr: np.array, n_fold: int, seed: int=42):\n    \"\"\"\n    create multi-label stratified group kfold indexs.\n\n    reference: https://www.kaggle.com/jakubwasikowski/stratified-group-k-fold-cross-validation\n    input:\n        label_arr: numpy.ndarray, shape = (n_train, n_class)\n            multi-label for each sample's index using multi-hot vectors\n        gid_arr: numpy.array, shape = (n_train,)\n            group id for each sample's index\n        n_fold: int. number of fold.\n        seed: random seed.\n    output:\n        yield indexs array list for each fold's train and validation.\n    \"\"\"\n    np.random.seed(seed)\n    random.seed(seed)\n    start_time = time.time()\n    n_train, n_class = label_arr.shape\n    gid_unique = sorted(set(gid_arr))\n    n_group = len(gid_unique)\n\n    # # aid_arr: (n_train,), indicates alternative id for group id.\n    # # generally, group ids are not 0-index and continuous or not integer.\n    gid2aid = dict(zip(gid_unique, range(n_group)))\n#     aid2gid = dict(zip(range(n_group), gid_unique))\n    aid_arr = np.vectorize(lambda x: gid2aid[x])(gid_arr)\n\n    # # count labels by class\n    cnts_by_class = label_arr.sum(axis=0)  # (n_class, )\n\n    # # count labels by group id.\n    col, row = np.array(sorted(enumerate(aid_arr), key=lambda x: x[1])).T\n    cnts_by_group = coo_matrix(\n        (np.ones(len(label_arr)), (row, col))\n    ).dot(coo_matrix(label_arr)).toarray().astype(int)\n    del col\n    del row\n    cnts_by_fold = np.zeros((n_fold, n_class), int)\n\n    groups_by_fold = [[] for fid in range(n_fold)]\n    group_and_cnts = list(enumerate(cnts_by_group))  # pair of aid and cnt by group\n    np.random.shuffle(group_and_cnts)\n    print(\"finished preparation\", time.time() - start_time)\n    for aid, cnt_by_g in sorted(group_and_cnts, key=lambda x: -np.std(x[1])):\n        best_fold = None\n        min_eval = None\n        for fid in range(n_fold):\n            # # eval assignment.\n            cnts_by_fold[fid] += cnt_by_g\n            fold_eval = (cnts_by_fold / cnts_by_class).std(axis=0).mean()\n            cnts_by_fold[fid] -= cnt_by_g\n\n            if min_eval is None or fold_eval < min_eval:\n                min_eval = fold_eval\n                best_fold = fid\n\n        cnts_by_fold[best_fold] += cnt_by_g\n        groups_by_fold[best_fold].append(aid)\n    print(\"finished assignment.\", time.time() - start_time)\n\n    gc.collect()\n    idx_arr = np.arange(n_train)\n    for fid in range(n_fold):\n        val_groups = groups_by_fold[fid]\n\n        val_indexs_bool = np.isin(aid_arr, val_groups)\n        train_indexs = idx_arr[~val_indexs_bool]\n        val_indexs = idx_arr[val_indexs_bool]\n\n        print(\"[fold {}]\".format(fid), end=\" \")\n        print(\"n_group: (train, val) = ({}, {})\".format(n_group - len(val_groups), len(val_groups)), end=\" \")\n        print(\"n_sample: (train, val) = ({}, {})\".format(len(train_indexs), len(val_indexs)))\n\n        yield train_indexs, val_indexs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_arr = train[CLASSES].values\ngroup_id = train.PatientID.values\n\ntrain_val_indexs = list(\n    multi_label_stratified_group_k_fold(label_arr, group_id, N_FOLD, RANDAM_SEED))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"train[\"fold\"] = -1\nfor fold_id, (trn_idx, val_idx) in enumerate(train_val_indexs):\n    train.loc[val_idx, \"fold\"] = fold_id\n    \ntrain.groupby(\"fold\")[CLASSES].sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## preprocess test images"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def resize_images(img_id, input_dir, output_dir, resize_to=(512, 512), ext=\"png\"):\n    img_path = input_dir / f\"{img_id}.jpg\"\n    save_path = output_dir / f\"{img_id}.{ext}\"\n    \n    img = cv2.imread(str(img_path), cv2.IMREAD_GRAYSCALE)\n    img = cv2.resize(img, resize_to)\n    cv2.imwrite(str(save_path), img, )\n\nTEST_RESIZED = TMP / \"test_{0}x{1}\".format(*IMAGE_SIZE)\nTEST_RESIZED.mkdir(exist_ok=True)\nTEST_RESIZED\n\n_ = Parallel(n_jobs=2, verbose=5)([\n    delayed(resize_images)(img_id, TEST, TEST_RESIZED, IMAGE_SIZE, \"png\")\n    for img_id in smpl_sub.StudyInstanceUID.values\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{},"cell_type":"markdown","source":"## definition"},{"metadata":{},"cell_type":"markdown","source":"### custom model"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def get_activation(activ_name: str=\"relu\"):\n    \"\"\"\"\"\"\n    act_dict = {\n        \"relu\": nn.ReLU(inplace=True),\n        \"tanh\": nn.Tanh(),\n        \"sigmoid\": nn.Sigmoid(),\n        \"identity\": nn.Identity()}\n    if activ_name in act_dict:\n        return act_dict[activ_name]\n    else:\n        raise NotImplementedError\n        \n\nclass Conv2dBNActiv(nn.Module):\n    \"\"\"Conv2d -> (BN ->) -> Activation\"\"\"\n    \n    def __init__(\n        self, in_channels: int, out_channels: int,\n        kernel_size: int, stride: int=1, padding: int=0,\n        bias: bool=False, use_bn: bool=True, activ: str=\"relu\"\n    ):\n        \"\"\"\"\"\"\n        super(Conv2dBNActiv, self).__init__()\n        layers = []\n        layers.append(nn.Conv2d(\n            in_channels, out_channels,\n            kernel_size, stride, padding, bias=bias))\n        if use_bn:\n            layers.append(nn.BatchNorm2d(out_channels))\n            \n        layers.append(get_activation(activ))\n        self.layers = nn.Sequential(*layers)\n        \n    def forward(self, x):\n        \"\"\"Forward\"\"\"\n        return self.layers(x)\n        \n\nclass SSEBlock(nn.Module):\n    \"\"\"channel `S`queeze and `s`patial `E`xcitation Block.\"\"\"\n\n    def __init__(self, in_channels: int):\n        \"\"\"Initialize.\"\"\"\n        super(SSEBlock, self).__init__()\n        self.channel_squeeze = nn.Conv2d(\n            in_channels=in_channels, out_channels=1,\n            kernel_size=1, stride=1, padding=0, bias=False)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        \"\"\"Forward.\"\"\"\n        # # x: (bs, ch, h, w) => h: (bs, 1, h, w)\n        h = self.sigmoid(self.channel_squeeze(x))\n        # # x, h => return: (bs, ch, h, w)\n        return x * h\n    \n    \nclass SpatialAttentionBlock(nn.Module):\n    \"\"\"Spatial Attention for (C, H, W) feature maps\"\"\"\n    \n    def __init__(\n        self, in_channels: int,\n        out_channels_list: tp.List[int],\n    ):\n        \"\"\"Initialize\"\"\"\n        super(SpatialAttentionBlock, self).__init__()\n        self.n_layers = len(out_channels_list)\n        channels_list = [in_channels] + out_channels_list\n        assert self.n_layers > 0\n        assert channels_list[-1] == 1\n        \n        for i in range(self.n_layers - 1):\n            in_chs, out_chs = channels_list[i: i + 2]\n            layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ=\"relu\")\n            setattr(self, f\"conv{i + 1}\", layer)\n            \n        in_chs, out_chs = channels_list[-2:]\n        layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ=\"sigmoid\")\n        setattr(self, f\"conv{self.n_layers}\", layer)\n    \n    def forward(self, x):\n        \"\"\"Forward\"\"\"\n        h = x\n        for i in range(self.n_layers):\n            h = getattr(self, f\"conv{i + 1}\")(h)\n            \n        h = h * x\n        return h","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SingleHeadModel(nn.Module):\n    \n    def __init__(\n        self, base_name: str='resnext50_32x4d', out_dim: int=11, pretrained=False\n    ):\n        \"\"\"\"\"\"\n        self.base_name = base_name\n        super(SingleHeadModel, self).__init__()\n        \n        # # load base model\n        base_model = timm.create_model(base_name, pretrained=pretrained)\n        in_features = base_model.num_features\n        \n        # # remove global pooling and head classifier\n        # base_model.reset_classifier(0, '')\n        base_model.reset_classifier(0)\n        \n        # # Shared CNN Bacbone\n        self.backbone = base_model\n        \n        # # Single Heads.\n        self.head_fc = nn.Sequential(\n            nn.Linear(in_features, in_features),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.5),\n            nn.Linear(in_features, out_dim))\n\n    def forward(self, x):\n        \"\"\"\"\"\"\n        h = self.backbone(x)\n        h = self.head_fc(h)\n        return h\n        \n\nclass MultiHeadModel(nn.Module):\n    \n    def __init__(\n        self, base_name: str='resnext50_32x4d',\n        out_dims_head: tp.List[int]=[3, 4, 3, 1], pretrained=False):\n        \"\"\"\"\"\"\n        self.base_name = base_name\n        self.n_heads = len(out_dims_head)\n        super(MultiHeadModel, self).__init__()\n        \n        # # load base model\n        base_model = timm.create_model(base_name, pretrained=pretrained)\n        in_features = base_model.num_features\n        \n        # # remove global pooling and head classifier\n        base_model.reset_classifier(0, '')\n        \n        # # Shared CNN Bacbone\n        self.backbone = base_model\n        \n        # # Multi Heads.\n        for i, out_dim in enumerate(out_dims_head):\n            layer_name = f\"head_{i}\"\n            layer = nn.Sequential(\n                SpatialAttentionBlock(in_features, [64, 32, 16, 1]),\n                nn.AdaptiveAvgPool2d(output_size=1),\n                nn.Flatten(start_dim=1),\n                nn.Linear(in_features, in_features),\n                nn.ReLU(inplace=True),\n                nn.Dropout(0.5),\n                nn.Linear(in_features, out_dim))\n            setattr(self, layer_name, layer)\n\n    def forward(self, x):\n        \"\"\"\"\"\"\n        h = self.backbone(x)\n        hs = [\n            getattr(self, f\"head_{i}\")(h) for i in range(self.n_heads)]\n        y = torch.cat(hs, axis=1)\n        return y\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### dataset"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class LabeledImageDataset(data.Dataset):\n    \"\"\"\n    Dataset class for (image, label) pairs\n\n    reads images and applys transforms to them.\n\n    Attributes\n    ----------\n    file_list : List[Tuple[tp.Union[str, Path], tp.Union[int, float, np.ndarray]]]\n        list of (image file, label) pair\n    transform_list : List[Dict]\n        list of dict representing image transform \n    \"\"\"\n\n    def __init__(\n        self,\n        file_list: tp.List[\n            tp.Tuple[tp.Union[str, Path], tp.Union[int, float, np.ndarray]]],\n        transform_list: tp.List[tp.Dict],\n    ):\n        \"\"\"Initialize\"\"\"\n        self.file_list = file_list\n        self.transform = ImageTransformForCls(transform_list)\n\n    def __len__(self):\n        \"\"\"Return Num of Images.\"\"\"\n        return len(self.file_list)\n\n    def __getitem__(self, index):\n        \"\"\"Return transformed image and mask for given index.\"\"\"\n        img_path, label = self.file_list[index]\n        img = self._read_image_as_array(img_path)\n        \n        img, label = self.transform((img, label))\n        return img, label\n\n    def _read_image_as_array(self, path: str):\n        \"\"\"Read image file and convert into numpy.ndarray\"\"\"\n        img_arr = cv2.imread(str(path))\n        img_arr = cv2.cvtColor(img_arr, cv2.COLOR_BGR2RGB)\n        return img_arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def get_dataloaders_for_inference(\n    file_list: tp.List[tp.List], batch_size=64,\n):\n    \"\"\"Create DataLoader\"\"\"\n    dataset = LabeledImageDataset(\n        file_list,\n        transform_list=[\n          [\"Normalize\", {\n              \"always_apply\": True, \"max_pixel_value\": 255.0,\n              \"mean\": [\"0.4887381077884414\"], \"std\": [\"0.23064819430546407\"]}],\n          [\"ToTensorV2\", {\"always_apply\": True}],\n        ])\n    loader = data.DataLoader(\n        dataset,\n        batch_size=batch_size, shuffle=False,\n        num_workers=2, pin_memory=True,\n        drop_last=False)\n\n    return loader","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### image transform"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class ImageTransformBase:\n    \"\"\"\n    Base Image Transform class.\n\n    Args:\n        data_augmentations: List of tuple(method: str, params :dict), each elems pass to albumentations\n    \"\"\"\n\n    def __init__(self, data_augmentations: tp.List[tp.Tuple[str, tp.Dict]]):\n        \"\"\"Initialize.\"\"\"\n        augmentations_list = [\n            self._get_augmentation(aug_name)(**params)\n            for aug_name, params in data_augmentations]\n        self.data_aug = albumentations.Compose(augmentations_list)\n\n    def __call__(self, pair: tp.Tuple[np.ndarray]) -> tp.Tuple[np.ndarray]:\n        \"\"\"You have to implement this by task\"\"\"\n        raise NotImplementedError\n\n    def _get_augmentation(self, aug_name: str) -> tp.Tuple[ImageOnlyTransform, DualTransform]:\n        \"\"\"Get augmentations from albumentations\"\"\"\n        if hasattr(albumentations, aug_name):\n            return getattr(albumentations, aug_name)\n        else:\n            return eval(aug_name)\n\n\nclass ImageTransformForCls(ImageTransformBase):\n    \"\"\"Data Augmentor for Classification Task.\"\"\"\n\n    def __init__(self, data_augmentations: tp.List[tp.Tuple[str, tp.Dict]]):\n        \"\"\"Initialize.\"\"\"\n        super(ImageTransformForCls, self).__init__(data_augmentations)\n\n    def __call__(self, in_arrs: tp.Tuple[np.ndarray]) -> tp.Tuple[np.ndarray]:\n        \"\"\"Apply Transform.\"\"\"\n        img, label = in_arrs\n        augmented = self.data_aug(image=img)\n        img = augmented[\"image\"]\n\n        return img, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### utils"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def load_setting_file(path: str):\n    \"\"\"Load YAML setting file.\"\"\"\n    with open(path) as f:\n        settings = yaml.safe_load(f)\n    return settings\n\n\ndef set_random_seed(seed: int = 42, deterministic: bool = False):\n    \"\"\"Set seeds\"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = deterministic  # type: ignore\n    \n\ndef run_inference_loop(stgs, model, loader, device):\n    model.to(device)\n    model.eval()\n    pred_list = []\n    with torch.no_grad():\n        for x, t in tqdm(loader):\n            y = model(x.to(device))\n            pred_list.append(y.sigmoid().detach().cpu().numpy())\n            # pred_list.append(y.detach().cpu().numpy())\n        \n    pred_arr = np.concatenate(pred_list)\n    del pred_list\n    return pred_arr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## inference test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"if not torch.cuda.is_available():\n    device = torch.device(\"cpu\")\nelse:\n    device = torch.device(\"cuda\")\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_dir = TRAINED_MODEL\ntest_dir = TEST_RESIZED\n\ntest_file_list = [\n    (test_dir / f\"{img_id}.png\", [-1] * 11)\n    for img_id in smpl_sub[\"StudyInstanceUID\"].values]\ntest_loader = get_dataloaders_for_inference(test_file_list, batch_size=64)\n        \ntest_preds_arr = np.zeros((N_FOLD, len(smpl_sub), N_CLASSES))    \nfor fold_id in FOLDS:\n    print(f\"[fold {fold_id}]\")\n#     stgs = load_setting_file(model_dir / f\"fold{fold_id}\" / \"settings.yml\")\n    setting_dir = f'../input/multihead-version18-fold{fold_id}/settings.yml'\n\n    stgs = load_setting_file(setting_dir)\n    # # prepare \n    stgs[\"model\"][\"params\"][\"pretrained\"] = False\n    model = MultiHeadModel(**stgs[\"model\"][\"params\"])\n    model_path = model_dir / f\"best_model_fold{fold_id}.pth\"\n    model.load_state_dict(torch.load(model_path, map_location=device))\n\n    # # inference test\n    test_pred = run_inference_loop(stgs, model, test_loader, device)\n    test_preds_arr[fold_id] = test_pred\n    \n    del model\n    torch.cuda.empty_cache()\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"if CONVERT_TO_RANK:\n    # # shape: (fold, n_example, class)\n    test_preds_arr = test_preds_arr.argsort(axis=1).argsort(axis=1)\n\nsub = smpl_sub.copy()\nsub[CLASSES] = test_preds_arr.mean(axis=0)\n    \nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}