{"cells":[{"metadata":{"execution":{"iopub.execute_input":"2020-12-14T19:52:41.647322Z","iopub.status.busy":"2020-12-14T19:52:41.646482Z","iopub.status.idle":"2020-12-14T19:52:41.650102Z","shell.execute_reply":"2020-12-14T19:52:41.649535Z"},"papermill":{"duration":0.034829,"end_time":"2020-12-14T19:52:41.650216","exception":false,"start_time":"2020-12-14T19:52:41.615387","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import os\n\nMODEL_DIR = '../input/resnet200d-n-stage/'\nOUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\nTEST_PATH = '../input/ranzcr-clip-catheter-line-classification/test'\n\nclass CFG:\n    debug=False\n    num_workers=4\n    size=700\n    batch_size=64\n    seed=42\n    n_class=11\n    target_cols=['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',\n                 'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal', \n                 'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',\n                 'Swan Ganz Catheter Present']\n    \n    model_archs=[ \n        # \"resnet200d\"\n        'MultiHeadResNet200D'\n    ]\n    model=[\"resnet200d_1701_fold3_epoch0.pth\", ## CV0.9767\n          ]\n    multi_gpu = True\n    \nassert len(CFG.model_archs) == len(CFG.model)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-14T19:52:41.750878Z","iopub.status.busy":"2020-12-14T19:52:41.750245Z","iopub.status.idle":"2020-12-14T19:52:45.494184Z","shell.execute_reply":"2020-12-14T19:52:45.492665Z"},"papermill":{"duration":3.779959,"end_time":"2020-12-14T19:52:45.49431","exception":false,"start_time":"2020-12-14T19:52:41.714351","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n\nimport os\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nfrom matplotlib import pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose\n    )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport timm\n\nfrom torch.cuda.amp import autocast, GradScaler\nfrom collections import OrderedDict\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-14T19:52:45.59513Z","iopub.status.busy":"2020-12-14T19:52:45.594471Z","iopub.status.idle":"2020-12-14T19:52:45.60117Z","shell.execute_reply":"2020-12-14T19:52:45.60042Z"},"papermill":{"duration":0.040687,"end_time":"2020-12-14T19:52:45.601288","exception":false,"start_time":"2020-12-14T19:52:45.560601","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.023234,"end_time":"2020-12-14T19:52:45.910123","exception":false,"start_time":"2020-12-14T19:52:45.886889","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-14T19:52:45.967747Z","iopub.status.busy":"2020-12-14T19:52:45.96573Z","iopub.status.idle":"2020-12-14T19:52:45.968559Z","shell.execute_reply":"2020-12-14T19:52:45.969115Z"},"papermill":{"duration":0.036559,"end_time":"2020-12-14T19:52:45.969232","exception":false,"start_time":"2020-12-14T19:52:45.932673","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-14T19:52:46.071509Z","iopub.status.busy":"2020-12-14T19:52:46.070642Z","iopub.status.idle":"2020-12-14T19:52:46.074136Z","shell.execute_reply":"2020-12-14T19:52:46.073613Z"},"papermill":{"duration":0.035507,"end_time":"2020-12-14T19:52:46.074245","exception":false,"start_time":"2020-12-14T19:52:46.038738","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def get_transforms(*, data):\n    if data == 'valid':\n        return Compose([\n            Resize(CFG.size, CFG.size),\n            Normalize(\n                mean=[0.4824, 0.4824, 0.4824],\n                std=[0.22, 0.22, 0.22],\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.022168,"end_time":"2020-12-14T19:52:46.118843","exception":false,"start_time":"2020-12-14T19:52:46.096675","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# MODEL"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-14T19:52:46.17627Z","iopub.status.busy":"2020-12-14T19:52:46.170979Z","iopub.status.idle":"2020-12-14T19:52:46.18312Z","shell.execute_reply":"2020-12-14T19:52:46.183763Z"},"papermill":{"duration":0.042914,"end_time":"2020-12-14T19:52:46.183878","exception":false,"start_time":"2020-12-14T19:52:46.140964","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class RanzcrClassifier(nn.Module):\n    def __init__(self, model_arch, pretrained=False):\n        super().__init__()\n        self.model_arch = model_arch\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        \n\n        if 'efficientnet' in self.model_arch:\n            n_features = self.model.classifier.in_features\n            self.model.classifier = nn.Linear(n_features, CFG.n_class)\n        elif \"resnet\" in self.model_arch:\n            n_features = self.model.fc.in_features\n            self.model.global_pool = nn.Identity()\n            self.model.fc = nn.Identity()\n            self.pooling = nn.AdaptiveAvgPool2d(1)\n            self.fc = nn.Linear(n_features, CFG.n_class)\n\n    def forward(self, x):\n        if 'efficientnet' in self.model_arch:\n            return self.model(x)\n        elif \"resnet\" in self.model_arch:\n            bs = x.size(0)\n            features = self.model(x)\n            pooled_features = self.pooling(features).view(bs, -1)\n            output = self.fc(pooled_features)\n            return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_activation(activ_name: str=\"relu\"):\n    \"\"\"\"\"\"\n    act_dict = {\n        \"relu\": nn.ReLU(inplace=True),\n        \"tanh\": nn.Tanh(),\n        \"sigmoid\": nn.Sigmoid(),\n        \"identity\": nn.Identity()}\n    if activ_name in act_dict:\n        return act_dict[activ_name]\n    else:\n        raise NotImplementedError\n        \n\nclass Conv2dBNActiv(nn.Module):\n    \"\"\"Conv2d -> (BN ->) -> Activation\"\"\"\n    \n    def __init__(\n        self, in_channels, out_channels,\n        kernel_size, stride, padding,\n        bias=False, use_bn=True, activ=\"relu\"\n    ):\n        \"\"\"\"\"\"\n        super(Conv2dBNActiv, self).__init__()\n        layers = []\n        layers.append(nn.Conv2d(\n            in_channels, out_channels,\n            kernel_size, stride, padding, bias=bias))\n        if use_bn:\n            layers.append(nn.BatchNorm2d(out_channels))\n            \n        layers.append(get_activation(activ))\n        self.layers = nn.Sequential(*layers)\n        \n    def forward(self, x):\n        \"\"\"Forward\"\"\"\n        return self.layers(x)\n        \n    \nclass SpatialAttentionBlock(nn.Module):\n    \"\"\"Spatial Attention for (C, H, W) feature maps\"\"\"\n    \n    def __init__(\n        self, in_channels,\n        out_channels_list,\n    ):\n        \"\"\"Initialize\"\"\"\n        super(SpatialAttentionBlock, self).__init__()\n        self.n_layers = len(out_channels_list)\n        channels_list = [in_channels] + out_channels_list\n        assert self.n_layers > 0\n        assert channels_list[-1] == 1\n        \n        for i in range(self.n_layers - 1):\n            in_chs, out_chs = channels_list[i: i + 2]\n            layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ=\"relu\")\n            setattr(self, f\"conv{i + 1}\", layer)\n            \n        in_chs, out_chs = channels_list[-2:]\n        layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ=\"sigmoid\")\n        setattr(self, f\"conv{self.n_layers}\", layer)\n    \n    def forward(self, x):\n        \"\"\"Forward\"\"\"\n        h = x\n        for i in range(self.n_layers):\n            h = getattr(self, f\"conv{i + 1}\")(h)\n            \n        h = h * x\n        return h\n\n\n\nclass MultiHeadResNet200D(nn.Module):\n    def __init__(self, out_dims_head=[3, 4, 3, 1],  pretrained=False):\n        self.base_name = \"resnet200d_320\"\n        self.n_heads = len(out_dims_head)\n        super(MultiHeadResNet200D, self).__init__()\n        \n        # # load base model\n        base_model = timm.create_model(self.base_name, num_classes=sum(out_dims_head), pretrained=False)\n        in_features = base_model.num_features\n        \n        if pretrained:\n            pretrained_model_path = CFG.student\n            state_dict = dict()\n            for k, v in torch.load(pretrained_model_path, map_location='cpu')[\"model\"].items():\n                if k[:6] == \"model.\":\n                    k = k.replace(\"model.\", \"\")\n                state_dict[k] = v\n            base_model.load_state_dict(state_dict)\n        \n        # # remove global pooling and head classifier\n        base_model.reset_classifier(0, '')\n        \n        # # Shared CNN Bacbone\n        self.backbone = base_model\n        \n        # # Multi Heads.\n        for i, out_dim in enumerate(out_dims_head):\n            layer_name = f\"head_{i}\"\n            layer = nn.Sequential(\n                SpatialAttentionBlock(in_features, [64, 32, 16, 1]),\n                nn.AdaptiveAvgPool2d(output_size=1),\n                nn.Flatten(start_dim=1),\n                nn.Linear(in_features, in_features),\n                nn.ReLU(inplace=True),\n                nn.Dropout(0.5),\n                nn.Linear(in_features, out_dim))\n            setattr(self, layer_name, layer)\n\n    def forward(self, x):\n        h = self.backbone(x)\n        hs = [getattr(self, f\"head_{i}\")(h) for i in range(self.n_heads)]\n        y = torch.cat(hs, axis=1)\n        return y","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-14T19:52:46.445205Z","iopub.status.busy":"2020-12-14T19:52:46.443326Z","iopub.status.idle":"2020-12-14T19:52:46.445944Z","shell.execute_reply":"2020-12-14T19:52:46.446449Z"},"papermill":{"duration":0.064625,"end_time":"2020-12-14T19:52:46.446561","exception":false,"start_time":"2020-12-14T19:52:46.381936","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def inference(model, test_loader, device):\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        with torch.no_grad():\n            y_preds = model(images)\n        avg_preds.append(y_preds.sigmoid().to('cpu').numpy())\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.022557,"end_time":"2020-12-14T19:52:46.492442","exception":false,"start_time":"2020-12-14T19:52:46.469885","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"states = [torch.load(MODEL_DIR+a_model) for a_model in CFG.model]\ntest_dataset = TestDataset(test, transform=get_transforms(data='valid'))\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.num_workers, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_list = []\nfor idx,state in enumerate(states):\n    print(idx, CFG.model_archs[idx])\n    if CFG.model_archs[idx] == \"MultiHeadResNet200D\":\n        model = MultiHeadResNet200D([3, 4, 3, 1], pretrained=False)\n    elif CFG.model_archs[idx] == \"resnet200d\":\n        model = RanzcrClassifier(\"resnet200d\", pretrained=False)\n    elif CFG.model_archs[idx] == \"resnet50d\":\n        model = RanzcrClassifier(\"resnet50d\", pretrained=False)\n    model.to(device)\n        \n    if CFG.multi_gpu:\n        state_dict = state['model']\n        new_state_dict = OrderedDict()\n        for k, v in state_dict.items():\n            k=k[7:]\n            new_state_dict[k]=v\n        model.load_state_dict(new_state_dict)\n    else:\n        model.load_state_dict(state['model'])\n    \n    model.eval()\n    predictions = inference(model, test_loader, device)\n    predictions_list.append(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_avg = np.mean(predictions_list, axis=0)\ntest[CFG.target_cols] = pred_avg\ntest[['StudyInstanceUID'] + CFG.target_cols].to_csv(OUTPUT_DIR+'submission.csv', index=False)\ntest.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}