{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":639913,"sourceType":"datasetVersion","datasetId":316567},{"sourceId":650059,"sourceType":"datasetVersion","datasetId":323474},{"sourceId":852244,"sourceType":"datasetVersion","datasetId":300534},{"sourceId":852245,"sourceType":"datasetVersion","datasetId":296300},{"sourceId":956507,"sourceType":"datasetVersion","datasetId":250288}],"dockerImageVersionId":29282,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys\nimport random\nimport math\nimport subprocess\nfrom glob import glob\nfrom collections import OrderedDict\nimport numpy as np\nimport cv2\nfrom skimage import measure\nimport pandas as pd\nfrom tqdm import tqdm_notebook as tqdm\nfrom matplotlib import pyplot as plt\nimport scipy.ndimage as ndi\nfrom PIL import Image\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom sklearn import metrics\nfrom sklearn.linear_model import LinearRegression\n\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\nimport torch.backends.cudnn as cudnn\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import DataLoader\nimport torchvision\nfrom torchvision import datasets, models, transforms\nfrom torch.optim.optimizer import Optimizer, required\n\nsys.path.append(\"../input/pretrained-models/pretrained-models.pytorch-master/\")\nimport pretrainedmodels","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def scale_radius(src, img_size, padding=False):\n    x = src[src.shape[0] // 2, ...].sum(axis=1)\n    r = (x > x.mean() / 10).sum() // 2\n    yx = src.sum(axis=2)\n    region_props = measure.regionprops((yx > yx.mean() / 10).astype('uint8'))\n    yc, xc = np.round(region_props[0].centroid).astype('int')\n    x1 = max(xc - r, 0)\n    x2 = min(xc + r, src.shape[1] - 1)\n    y1 = max(yc - r, 0)\n    y2 = min(yc + r, src.shape[0] - 1)\n    dst = src[y1:y2, x1:x2]\n    dst = cv2.resize(dst, dsize=None, fx=img_size/(2*r), fy=img_size/(2*r))\n    if padding:\n        pad_x = (img_size - dst.shape[1]) // 2\n        pad_y = (img_size - dst.shape[0]) // 2\n        dst = np.pad(dst, ((pad_y, pad_y), (pad_x, pad_x), (0, 0)), 'constant')\n    return dst\n\n    \nclass Dataset(torch.utils.data.Dataset):\n    def __init__(self, img_paths, labels, transform=None, img_size=288, save_img=True):\n        self.img_paths = img_paths\n        self.labels = labels\n        self.transform = transform\n        self.img_size = img_size\n        self.save_img = save_img\n\n    def __getitem__(self, index):\n        img_path, label = self.img_paths[index], self.labels[index]\n        \n        if 'train' in img_path:\n            img = cv2.imread('../input/aptos2019-dataset/images_288_scaled/images_288_scaled/%s' %os.path.basename(img_path))\n        \n        else:\n            if os.path.exists('processed/%s' %os.path.basename(img_path)):\n                img = cv2.imread('processed/%s' %os.path.basename(img_path))\n\n            else:\n                img = cv2.imread(img_path)\n                try:\n                    img = scale_radius(img, img_size=self.img_size, padding=False)\n                except Exception as e:\n                    img = img\n                if self.save_img:\n                    cv2.imwrite('processed/%s' %os.path.basename(img_path), img)\n        \n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = Image.fromarray(img)\n\n        if self.transform is not None:\n            img = self.transform(img)\n\n        return img, label\n\n\n    def __len__(self):\n        return len(self.img_paths)\n\n\ndef get_model(model_name='resnet18', num_outputs=None, pretrained=True,\n              freeze_bn=False, dropout_p=0, **kwargs):\n\n    pretrained = 'imagenet' if pretrained else None\n    model = pretrainedmodels.__dict__[model_name](num_classes=1000,\n                                                  pretrained=pretrained)\n\n    if 'dpn' in model_name:\n        in_channels = model.last_linear.in_channels\n        model.last_linear = nn.Conv2d(in_channels, num_outputs,\n                                      kernel_size=1, bias=True)\n    else:\n        if 'resnet' in model_name:\n            model.avgpool = nn.AdaptiveAvgPool2d(1)\n        else:\n            model.avg_pool = nn.AdaptiveAvgPool2d(1)\n        in_features = model.last_linear.in_features\n        if dropout_p == 0:\n            model.last_linear = nn.Linear(in_features, num_outputs)\n        else:\n            model.last_linear = nn.Sequential(\n                nn.Dropout(p=dropout_p),\n                nn.Linear(in_features, num_outputs),\n            )\n\n    if freeze_bn:\n        for m in model.modules():\n            if isinstance(m, nn.BatchNorm2d):\n                m.weight.requires_grad = False\n                m.bias.requires_grad = False\n\n    return model\n\n\nclass RAdam(Optimizer):\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n        self.buffer = [[None, None, None] for ind in range(10)]\n        super(RAdam, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(RAdam, self).__setstate__(state)\n\n    def step(self, closure=None):\n\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('RAdam does not support sparse gradients')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n                state['step'] += 1\n                buffered = self.buffer[int(state['step'] % 10)]\n                if state['step'] == buffered[0]:\n                    N_sma, step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n\n                    # more conservative since it's an approximated value\n                    if N_sma >= 5:\n                        step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    else:\n                        step_size = group['lr'] / (1 - beta1 ** state['step'])\n                    buffered[2] = step_size\n\n                if group['weight_decay'] != 0:\n                    p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n\n                # more conservative since it's an approximated value\n                if N_sma >= 5:\n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    p_data_fp32.addcdiv_(-step_size, exp_avg, denom)\n                else:\n                    p_data_fp32.add_(-step_size, exp_avg)\n\n                p.data.copy_(p_data_fp32)\n\n        return loss\n\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef quadratic_weighted_kappa(y_pred, y_true):\n    if torch.is_tensor(y_pred):\n        y_pred = y_pred.data.cpu().numpy()\n    if torch.is_tensor(y_true):\n        y_true = y_true.data.cpu().numpy()\n    if y_pred.shape[1] == 1:\n        y_pred = y_pred[:, 0]\n    else:\n        y_pred = np.argmax(y_pred, axis=1)\n    return metrics.cohen_kappa_score(y_pred, y_true, weights='quadratic')\n\n\ndef train(train_loader, model, criterion, optimizer, epoch):\n    losses = AverageMeter()\n    scores = AverageMeter()\n\n    model.train()\n\n    for i, (input, target) in tqdm(enumerate(train_loader), total=len(train_loader)):\n        input = input.cuda()\n        target = target.cuda()\n\n        output = model(input)\n        loss = criterion(output.view(-1), target.float())\n\n        # compute gradient and do optimizing step\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        thrs = [0.5, 1.5, 2.5, 3.5]\n        output[output < thrs[0]] = 0\n        output[(output >= thrs[0]) & (output < thrs[1])] = 1\n        output[(output >= thrs[1]) & (output < thrs[2])] = 2\n        output[(output >= thrs[2]) & (output < thrs[3])] = 3\n        output[output >= thrs[3]] = 4\n        \n        target[target < thrs[0]] = 0\n        target[(target >= thrs[0]) & (target < thrs[1])] = 1\n        target[(target >= thrs[1]) & (target < thrs[2])] = 2\n        target[(target >= thrs[2]) & (target < thrs[3])] = 3\n        target[target >= thrs[3]] = 4\n        \n        score = quadratic_weighted_kappa(output, target)\n\n        losses.update(loss.item(), input.size(0))\n        scores.update(score, input.size(0))\n\n    return losses.avg, scores.avg\n\n\ndef validate(val_loader, model, criterion):\n    losses = AverageMeter()\n    scores = AverageMeter()\n\n    # switch to evaluate mode\n    model.eval()\n\n    with torch.no_grad():\n        for i, (input, target) in tqdm(enumerate(val_loader), total=len(val_loader)):\n            input = input.cuda()\n            target = target.cuda()\n\n            output = model(input)\n            loss = criterion(output.view(-1), target.float())\n        \n            thrs = [0.5, 1.5, 2.5, 3.5]\n            output[output < thrs[0]] = 0\n            output[(output >= thrs[0]) & (output < thrs[1])] = 1\n            output[(output >= thrs[1]) & (output < thrs[2])] = 2\n            output[(output >= thrs[2]) & (output < thrs[3])] = 3\n            output[output >= thrs[3]] = 4\n            score = quadratic_weighted_kappa(output, target)\n\n            losses.update(loss.item(), input.size(0))\n            scores.update(score, input.size(0))\n\n    return losses.avg, scores.avg","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('processed', exist_ok=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = cv2.imread('../input/aptos2019-dataset/images_288_scaled/001639a390f0.png')\n#img = cv2.imread('../input/aptos2019-blindness-detection/train_images/000c1434d8d7.png')\nimg_t = cv2.resize(img, (288,288))\nimg_t = scale_radius(img_t, img_size=288, padding=False)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nimg_t = cv2.cvtColor(img_t, cv2.COLOR_BGR2RGB)\n\nf, axarr = plt.subplots(1,2,figsize = (11,11))\naxarr[0].imshow(img)\naxarr[1].imshow(img_t)\nplt.title('After applying scale_radius')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#img = cv2.imread('../input/aptos2019-dataset/images_288_scaled/001639a390f0.png')\nimg = cv2.imread('../input/aptos2019-blindness-detection/train_images/000c1434d8d7.png')\nimg_t = cv2.resize(img, (288,288))\nimg_t = scale_radius(img_t, img_size=288, padding=False)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nimg_t = cv2.cvtColor(img_t, cv2.COLOR_BGR2RGB)\n\nf, axarr = plt.subplots(1,2,figsize = (11,11))\naxarr[0].imshow(img)\naxarr[1].imshow(img_t)\nplt.title('After applying scale_radius')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pseudo Labeling","metadata":{}},{"cell_type":"code","source":"pseudo_probs = {}\n\naptos2019_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\naptos2019_img_paths = '../input/aptos2019-blindness-detection/train_images/' + aptos2019_df['id_code'].values + '.png'\naptos2019_labels = aptos2019_df['diagnosis'].values\n\ntest_df = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\ndir_name = '../input/aptos2019-blindness-detection/test_images'\ntest_img_paths = dir_name + '/' + test_df['id_code'].values + '.png'\ntest_labels = np.zeros(len(test_img_paths))\n\ntest_transform = transforms.Compose([\n    transforms.Resize((256, 256)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n])\n\ntest_set = Dataset(\n    test_img_paths,\n    test_labels,\n    transform=test_transform)\ntest_loader = torch.utils.data.DataLoader(\n    test_set,\n    batch_size=32,\n    shuffle=False,\n    num_workers=2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## SE-ResNeXt50_32x4d","metadata":{}},{"cell_type":"code","source":"# create model\nmodel = get_model(model_name='se_resnext50_32x4d',\n                  num_outputs=1,\n                  pretrained=False,\n                  freeze_bn=True,\n                  dropout_p=0)\nmodel = model.cuda()\nmodel.eval()\n\nprobs = []\nfor fold in range(5):\n    print('Fold [%d/%d]' %(fold+1, 5))\n\n    model.load_state_dict(torch.load('../input/se-resnext50-32x4d-080922/model_%d.pth' % (fold+1)))\n\n    probs_fold = []\n    with torch.no_grad():\n        for i, (input, _) in tqdm(enumerate(test_loader), total=len(test_loader)):\n            input = input.cuda()\n            output = model(input)\n\n            probs_fold.extend(output.data.cpu().numpy()[:, 0])\n    probs_fold = np.array(probs_fold)\n    probs.append(probs_fold)\n\nprobs = np.mean(probs, axis=0)\npseudo_probs['se_resnext50_32x4d'] = probs\n\ndel model\ntorch.cuda.empty_cache()\n!nvidia-smi","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble","metadata":{}},{"cell_type":"code","source":"preds = pseudo_probs['se_resnext50_32x4d'] #+ 0.5 * l1_probs['se_resnext101_32x4d']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thrs = [0.5, 1.5, 2.5, 3.5]\npreds[preds < thrs[0]] = 0\npreds[(preds >= thrs[0]) & (preds < thrs[1])] = 1\npreds[(preds >= thrs[1]) & (preds < thrs[2])] = 2\npreds[(preds >= thrs[2]) & (preds < thrs[3])] = 3\npreds[preds >= thrs[3]] = 4\npreds = preds.astype('int')\n\ntest_df['diagnosis'] = preds\ntest_df.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm processed/*","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}