{"cells":[{"metadata":{},"cell_type":"markdown","source":"<div style=\"width: 100%\">\n    <img style=\"width: 100%\" src=\"https://storage.googleapis.com/kaggle-competitions/kaggle/23870/logos/header.png\"/>\n</div>"},{"metadata":{},"cell_type":"markdown","source":"Huge Shout out to these kernels.\n\n1. Sin's https://www.kaggle.com/underwearfitting/resnet200d-public-benchmark-2xtta-lb0-965/data?scriptVersionId=51087772\n2. Ammarali32's https://www.kaggle.com/ammarali32/resnet200d-inference-single-model-lb-96-5/data\n\nPlease like their kernels. In this exercise, we are trying to ensemble the two strategies together"},{"metadata":{},"cell_type":"markdown","source":"<pre>\nLogs\nVersion 10 -  Resnet200d Public Benchmark + Inference Model (.967)\nVersion 6 -  Resnet200d Public Benchmark + Multi-Head Model (.966)\n</pre>"},{"metadata":{},"cell_type":"markdown","source":"Configuration : Sin's Model"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nbatch_size = 1\nimage_size = 512\ntta = True\nsubmit = True\nenet_type = ['resnet200d'] * 5\nmodel_path = ['../input/resnet200d-baseline-benchmark-public/resnet200d_fold0_cv953.pth',\n              '../input/resnet200d-baseline-benchmark-public/resnet200d_fold1_cv955.pth',\n              '../input/resnet200d-baseline-benchmark-public/resnet200d_fold2_cv955.pth',\n              '../input/resnet200d-baseline-benchmark-public/resnet200d_fold3_cv957.pth',\n              '../input/resnet200d-baseline-benchmark-public/resnet200d_fold4_cv954.pth']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport numpy as np\nDEBUG = False\nimport time\nimport cv2\nimport PIL.Image\nfrom sklearn.metrics import accuracy_score\nfrom tqdm.notebook import tqdm\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nimport albumentations\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom pylab import rcParams\nimport timm\nfrom albumentations import *\nfrom albumentations.pytorch import ToTensorV2\ndevice = torch.device('cuda') if not DEBUG else torch.device('cpu')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZCRResNet200D(nn.Module):\n    def __init__(self, model_name='resnet200d', out_dim=11, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, out_dim)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_test = albumentations.Compose([\n    Resize(image_size, image_size),\n    Normalize(\n         mean=[0.485, 0.456, 0.406],\n         std=[0.229, 0.224, 0.225],\n     ),\n    ToTensorV2()\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZCRDataset(Dataset):\n    def __init__(self, df, mode, transform=None):\n        \n        self.df = df.reset_index(drop=True)\n        self.mode = mode\n        self.transform = transform\n        self.labels = df[target_cols].values\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        row = self.df.loc[index]\n        img = cv2.imread(row.file_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        if self.transform is not None:\n            res = self.transform(image=img)\n            img = res['image']\n        label = torch.tensor(self.labels[index]).float()\n        if self.mode == 'test':\n            return img\n        else:\n            return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\ntest['file_path'] = test.StudyInstanceUID.apply(lambda x: os.path.join('../input/ranzcr-clip-catheter-line-classification/test', f'{x}.jpg'))\ntarget_cols = test.iloc[:, 1:12].columns.tolist()\n\ntest_dataset = RANZCRDataset(test, 'test', transform=transforms_test)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False,  num_workers=24)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_func(test_loader):\n    model.eval()\n    bar = tqdm(test_loader)\n    LOGITS = []\n    PREDS = []\n    \n    with torch.no_grad():\n        for batch_idx, images in enumerate(bar):\n            x = images.to(device)\n            logits = model(x)\n            LOGITS.append(logits.cpu())\n            PREDS += [logits.sigmoid().detach().cpu()]\n        PREDS = torch.cat(PREDS).cpu().numpy()\n        LOGITS = torch.cat(LOGITS).cpu().numpy()\n    return PREDS\n\ndef tta_inference_func(test_loader):\n    model.eval()\n    bar = tqdm(test_loader)\n    PREDS = []\n    LOGITS = []\n\n    with torch.no_grad():\n        for batch_idx, images in enumerate(bar):\n            x = images.to(device)\n            x = torch.stack([x,x.flip(-1)],0) # hflip\n            x = x.view(-1, 3, image_size, image_size)\n            logits = model(x)\n            logits = logits.view(batch_size, 2, -1).mean(1)\n            PREDS += [logits.sigmoid().detach().cpu()]\n            LOGITS.append(logits.cpu())\n        PREDS = torch.cat(PREDS).cpu().numpy()\n        \n    return PREDS","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Prepare the first Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"if submit:\n    test_preds_1 = []\n    for i in range(len(enet_type)):\n        if enet_type[i] == 'resnet200d':\n            print('resnet200d loaded')\n            model = RANZCRResNet200D(enet_type[i], out_dim=len(target_cols))\n            model = model.to(device)\n        model.load_state_dict(torch.load(model_path[i], map_location='cuda:0'))\n        if tta:\n            test_preds_1 += [tta_inference_func(test_loader)]\n        else:\n            test_preds_1 += [inference_func(test_loader)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\nsubmission[target_cols] = np.mean(test_preds_1, axis=0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Ammarali32's Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path.append('../input/pytorch-images-seresnet')\nimport timm\n\nimport os\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nfrom matplotlib import pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nimport albumentations\nfrom albumentations import *\nfrom albumentations.pytorch import ToTensorV2\n\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = 640\nBATCH_SIZE = 128\nTEST_PATH = '../input/ranzcr-clip-catheter-line-classification/test'\nMODEL_PATH = '../input/resnet200d-public/resnet200d_320_CV9632.pth'\ntest = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_transforms():\n        return Compose([\n            Resize(IMAGE_SIZE, IMAGE_SIZE),\n            Normalize(\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResNet200D(nn.Module):\n    def __init__(self, model_name='resnet200d_320'):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, 11)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference(models, test_loader, device):\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        for model in models:\n            with torch.no_grad():\n                y_preds1 = model(images)\n                y_preds2 = model(images.flip(-1))\n            y_preds = (y_preds1.sigmoid().to('cpu').numpy() + y_preds2.sigmoid().to('cpu').numpy()) / 2\n            avg_preds.append(y_preds)\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = ResNet200D()\nmodel.load_state_dict(torch.load(MODEL_PATH)['model'])\nmodel.eval()\nmodels = [model.to(device)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = TestDataset(test, transform=get_transforms())\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, \n                         num_workers=4 , pin_memory=True)\npredictions = inference(models, test_loader, device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[target_cols] = predictions\ntest[target_cols].head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission[target_cols].head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DISCUSSION : https://www.kaggle.com/c/ranzcr-clip-catheter-line-classification/discussion/211194\nFinal_Submission = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\n\nFinal_Submission[target_cols] = (test[target_cols]**0.5 + submission[target_cols]**0.5)/2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Final_Submission[target_cols].head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Final_Submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}