{"cells":[{"metadata":{},"cell_type":"markdown","source":"Model trained in https://www.kaggle.com/hmendonca/efficientnetb4-ignite-amp-clr-aptos19"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"!ls ../input/*","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = '../input/*/efficientNet_*.pth'\nmodel_path = 'efficientNet_best.pth'\n!md5sum {target}\n!cp {target} {model_path}\n!md5sum {model_path}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nclass Swish(nn.Module):\n    def forward(self, x):\n        return x * torch.sigmoid(x)\n\nclass Flatten(nn.Module):\n    def forward(self, x):\n        return x.reshape(x.shape[0], -1)\n\nclass SqueezeExcitation(nn.Module):\n    \n    def __init__(self, inplanes, se_planes):\n        super(SqueezeExcitation, self).__init__()\n        self.reduce_expand = nn.Sequential(\n            nn.Conv2d(inplanes, se_planes, \n                      kernel_size=1, stride=1, padding=0, bias=True),\n            Swish(),\n            nn.Conv2d(se_planes, inplanes, \n                      kernel_size=1, stride=1, padding=0, bias=True),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        x_se = torch.mean(x, dim=(-2, -1), keepdim=True)\n        x_se = self.reduce_expand(x_se)\n        return x_se * x\n    \nfrom torch.nn import functional as F\n\nclass MBConv(nn.Module):\n    def __init__(self, inplanes, planes, kernel_size, stride, \n                 expand_rate=1.0, se_rate=0.25, \n                 drop_connect_rate=0.2):\n        super(MBConv, self).__init__()\n\n        expand_planes = int(inplanes * expand_rate)\n        se_planes = max(1, int(inplanes * se_rate))\n\n        self.expansion_conv = None        \n        if expand_rate > 1.0:\n            self.expansion_conv = nn.Sequential(\n                nn.Conv2d(inplanes, expand_planes, \n                          kernel_size=1, stride=1, padding=0, bias=False),\n                nn.BatchNorm2d(expand_planes, momentum=0.01, eps=1e-3),\n                Swish()\n            )\n            inplanes = expand_planes\n\n        self.depthwise_conv = nn.Sequential(\n            nn.Conv2d(inplanes, expand_planes,\n                      kernel_size=kernel_size, stride=stride, \n                      padding=kernel_size // 2, groups=expand_planes,\n                      bias=False),\n            nn.BatchNorm2d(expand_planes, momentum=0.01, eps=1e-3),\n            Swish()\n        )\n\n        self.squeeze_excitation = SqueezeExcitation(expand_planes, se_planes)\n        \n        self.project_conv = nn.Sequential(\n            nn.Conv2d(expand_planes, planes, \n                      kernel_size=1, stride=1, padding=0, bias=False),\n            nn.BatchNorm2d(planes, momentum=0.01, eps=1e-3),\n        )\n\n        self.with_skip = stride == 1\n        self.drop_connect_rate = torch.tensor(drop_connect_rate, requires_grad=False)\n    \n    def _drop_connect(self, x):        \n        keep_prob = 1.0 - self.drop_connect_rate\n        drop_mask = torch.rand(x.shape[0], 1, 1, 1) + keep_prob\n        drop_mask = drop_mask.type_as(x)\n        drop_mask.floor_()\n        return drop_mask * x / keep_prob\n        \n    def forward(self, x):\n        z = x\n        if self.expansion_conv is not None:\n            x = self.expansion_conv(x)\n\n        x = self.depthwise_conv(x)\n        x = self.squeeze_excitation(x)\n        x = self.project_conv(x)\n        \n        # Add identity skip\n        if x.shape == z.shape and self.with_skip:            \n            if self.training and self.drop_connect_rate is not None:\n                self._drop_connect(x)\n            x += z\n        return x\n\nfrom collections import OrderedDict\nimport math\n\n\ndef init_weights(module):    \n    if isinstance(module, nn.Conv2d):    \n        nn.init.kaiming_normal_(module.weight, a=0, mode='fan_out')\n    elif isinstance(module, nn.Linear):\n        init_range = 1.0 / math.sqrt(module.weight.shape[1])\n        nn.init.uniform_(module.weight, a=-init_range, b=init_range)\n        \n        \nclass EfficientNet(nn.Module):\n        \n    def _setup_repeats(self, num_repeats):\n        return int(math.ceil(self.depth_coefficient * num_repeats))\n    \n    def _setup_channels(self, num_channels):\n        num_channels *= self.width_coefficient\n        new_num_channels = math.floor(num_channels / self.divisor + 0.5) * self.divisor\n        new_num_channels = max(self.divisor, new_num_channels)\n        if new_num_channels < 0.9 * num_channels:\n            new_num_channels += self.divisor\n        return new_num_channels\n\n    def __init__(self, num_classes, \n                 width_coefficient=1.0,\n                 depth_coefficient=1.0,\n                 se_rate=0.25,\n                 dropout_rate=0.2,\n                 drop_connect_rate=0.2):\n        super(EfficientNet, self).__init__()\n        \n        self.width_coefficient = width_coefficient\n        self.depth_coefficient = depth_coefficient\n        self.divisor = 8\n                \n        list_channels = [32, 16, 24, 40, 80, 112, 192, 320, 1280]\n        list_channels = [self._setup_channels(c) for c in list_channels]\n                \n        list_num_repeats = [1, 2, 2, 3, 3, 4, 1]\n        list_num_repeats = [self._setup_repeats(r) for r in list_num_repeats]        \n        \n        expand_rates = [1, 6, 6, 6, 6, 6, 6]\n        strides = [1, 2, 2, 2, 1, 2, 1]\n        kernel_sizes = [3, 3, 5, 3, 5, 5, 3]\n\n        # Define stem:\n        self.stem = nn.Sequential(\n            nn.Conv2d(3, list_channels[0], kernel_size=3, stride=2, padding=1, bias=False),\n            nn.BatchNorm2d(list_channels[0], momentum=0.01, eps=1e-3),\n            Swish()\n        )\n        \n        # Define MBConv blocks\n        blocks = []\n        counter = 0\n        num_blocks = sum(list_num_repeats)\n        for idx in range(7):\n            \n            num_channels = list_channels[idx]\n            next_num_channels = list_channels[idx + 1]\n            num_repeats = list_num_repeats[idx]\n            expand_rate = expand_rates[idx]\n            kernel_size = kernel_sizes[idx]\n            stride = strides[idx]\n            drop_rate = drop_connect_rate * counter / num_blocks\n            \n            name = \"MBConv{}_{}\".format(expand_rate, counter)\n            blocks.append((\n                name,\n                MBConv(num_channels, next_num_channels, \n                       kernel_size=kernel_size, stride=stride, expand_rate=expand_rate, \n                       se_rate=se_rate, drop_connect_rate=drop_rate)\n            ))\n            counter += 1\n            for i in range(1, num_repeats):                \n                name = \"MBConv{}_{}\".format(expand_rate, counter)\n                drop_rate = drop_connect_rate * counter / num_blocks                \n                blocks.append((\n                    name,\n                    MBConv(next_num_channels, next_num_channels, \n                           kernel_size=kernel_size, stride=1, expand_rate=expand_rate, \n                           se_rate=se_rate, drop_connect_rate=drop_rate)                                    \n                ))\n                counter += 1\n        \n        self.blocks = nn.Sequential(OrderedDict(blocks))\n        \n        # Define head\n        self.head = nn.Sequential(\n            nn.Conv2d(list_channels[-2], list_channels[-1], \n                      kernel_size=1, bias=False),\n            nn.BatchNorm2d(list_channels[-1], momentum=0.01, eps=1e-3),\n            Swish(),\n            nn.AdaptiveAvgPool2d(1),\n            Flatten(),\n            nn.Dropout(p=dropout_rate),\n            nn.Linear(list_channels[-1], num_classes)\n        )\n\n        self.apply(init_weights)\n        \n    def forward(self, x):\n        f = self.stem(x)\n        f = self.blocks(f)\n        y = self.head(f)\n        return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_size = 380\nbest_model = EfficientNet(num_classes=6, width_coefficient=1.4, depth_coefficient=1.8) ## B4\nbest_model.load_state_dict(torch.load(model_path))\nbest_model = best_model.cuda().eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from torchvision.transforms import Compose, RandomCrop, Pad, RandomHorizontalFlip, Resize, RandomAffine\nfrom torchvision.transforms import ToTensor, Normalize\n\nfrom torch.utils.data import Subset\nimport torchvision.utils as vutils\n\nimport os\nimport pandas as pd\nfrom sklearn.utils import shuffle\n\nfrom PIL import Image\n\nclass ImageDataset(torch.utils.data.Dataset):\n\n    def __init__(self, root, path_list, targets=None, transform=None, extension='.png'):\n        super().__init__()\n        self.root = root\n        self.path_list = path_list\n        self.targets = targets\n        self.transform = transform\n        self.extension = extension\n        if targets is not None:\n            assert len(self.path_list) == len(self.targets)\n            self.targets = torch.LongTensor(targets)\n\n    def __getitem__(self, index):\n        \"\"\"\n        Args:\n            index (int): Index\n        Returns:\n            tuple: (sample, target) where target is class_index of the target class.\n        \"\"\"\n        path = self.path_list[index]\n        sample = Image.open(os.path.join(self.root, path+self.extension))\n        if self.transform is not None:\n            sample = self.transform(sample)\n\n        if self.targets is not None:\n            return sample, self.targets[index]\n        else:\n            return sample, torch.LongTensor([])\n\n    def __len__(self):\n        return len(self.path_list)\n\nfrom PIL.Image import BICUBIC\n\ntest_transform = Compose([\n    Resize((image_size,image_size), BICUBIC),\n    ToTensor(),\n    Normalize(mean=[0.42, 0.22, 0.075], std=[0.27, 0.15, 0.081])\n])\n\ndf_test = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\ntest_dataset = ImageDataset(root='../input/aptos2019-blindness-detection/test_images', path_list=df_test.id_code.values, transform=test_transform)\n\n# len(test_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import DataLoader\n\nbatch_size = 16\nnum_workers = os.cpu_count()\nprint('num_workers:', num_workers)\n\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, num_workers=num_workers, \n                         shuffle=False, drop_last=False, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm_notebook as tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# def tta(x):\n#     ''' simple 8 fold regressor TTA '''\n#     pred = []\n#     for flip1 in range(2): # flip 1st dim\n#         if flip1: x = x.flip(1)\n#         for flip2 in range(2): # flip 2nd dim\n#             if flip2: x = x.flip(2)\n#             for trans in range(2): # transpose\n#                 if trans: x = x.transpose(-1,-2)\n#                 pred.append(best_model(x)[...,-1].unsqueeze(0))\n#     # concat and calc mean softmax for submission\n#     return torch.cat(pred).mean(dim=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# preds = []\n# with torch.no_grad():\n#     for x,_ in tqdm(test_loader, total=int(len(test_loader))):     \n#         # Let's compute final prediction as a mean of predictions on x and flipped x and/or transposed\n#         x = x.cuda()\n#         pred = tta(x)\n#         preds += pred.cpu().squeeze().tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def tta(x):\n    ''' simple 8 fold classifier TTA '''\n    pred = []\n    for flip1 in range(2): # flip 1st dim\n        if flip1: x = x.flip(1)\n        for flip2 in range(2): # flip 2nd dim\n            if flip2: x = x.flip(2)\n            for trans in range(2): # transpose\n                if trans: x = x.transpose(-1,-2)\n                pred.append(best_model(x)[...,:-1].unsqueeze(0))\n    # concat and calc mean softmax for submission\n    return F.softmax(torch.cat(pred), dim=-1).mean(dim=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = []\nbest_model.eval()\nwith torch.no_grad():\n    for x,_ in tqdm(test_loader, total=int(len(test_loader))):     \n        # Let's compute final prediction as a mean of predictions on x and flipped x and/or transposed\n        x = x.cuda()\n        pred = tta(x)\n        preds += torch.argmax(pred, dim=-1).cpu().squeeze().tolist()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import scipy as sp\n\nclass KappaOptimizer(nn.Module):\n    def __init__(self, coef=[0.5, 1.5, 2.5, 3.5]):\n        super().__init__()\n        self.coef = coef\n        # define score function:\n        self.func = self.quad_kappa\n    \n    def predict(self, preds):\n        return self._predict(self.coef, preds)\n\n    @classmethod\n    def _predict(cls, coef, preds):\n        if type(preds).__name__ == 'Tensor':\n            y_hat = preds.clone().view(-1)\n        else:\n            y_hat = torch.FloatTensor(preds).view(-1)\n\n        for i,pred in enumerate(y_hat):\n            if   pred < coef[0]: y_hat[i] = 0\n            elif pred < coef[1]: y_hat[i] = 1\n            elif pred < coef[2]: y_hat[i] = 2\n            elif pred < coef[3]: y_hat[i] = 3\n            else:                y_hat[i] = 4\n        return y_hat.int()\n    \n    def quad_kappa(self, preds, y):\n        return self._quad_kappa(self.coef, preds, y)\n\n    @classmethod\n    def _quad_kappa(cls, coef, preds, y):\n        y_hat = cls._predict(coef, preds)\n        return cohen_kappa_score(y, y_hat, weights='quadratic')\n\n    def fit(self, preds, y):\n        ''' maximize quad_kappa '''\n        print('Early score:', self.quad_kappa(preds, y))\n        neg_kappa = lambda coef: -self._quad_kappa(coef, preds, y)\n        opt_res = sp.optimize.minimize(neg_kappa, x0=self.coef, method='nelder-mead',\n                                       options={'maxiter':100, 'fatol':1e-20, 'xatol':1e-20})\n        print(opt_res)\n        self.coef = opt_res.x\n        print('New score:', self.quad_kappa(preds, y))\n\n    def forward(self, preds, y):\n        ''' the pytorch loss function '''\n        return torch.tensor(self.quad_kappa(preds, y))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# kappa_opt = KappaOptimizer([0.8, 1.0, 2.5, 3.2])\n# # # fit on validation set\n# # kappa_opt.fit(preds, targets)\n# preds = kappa_opt.predict(preds).tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nprint(len(preds))\n\nsub = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\nsub['diagnosis'] = np.array(preds, dtype=np.int32)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = sub.hist()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n_ = tr.hist()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}