{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport torch\nimport os\nimport time\nimport copy\nfrom torch.utils.data import Dataset, ConcatDataset\nfrom torchvision import transforms\nimport torchvision\nimport torch.optim as optim\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom tqdm import tqdm_notebook as tqdm\nfrom joblib import Parallel, delayed\nfrom sklearn import metrics\nfrom sklearn.metrics import confusion_matrix\nfrom scipy.stats import spearmanr\nfrom sklearn.model_selection import KFold, StratifiedKFold\nimport math\nimport uuid\nimport cv2\nimport gc\nimport albumentations\nfrom albumentations import torch as AT\nfrom numba import jit\n!pip install adabound\nimport adabound\nfrom contextlib import contextmanager, redirect_stdout\n\nfrom bisect import bisect_right\nimport numpy as np\nfrom torch.optim import Optimizer\n\nfrom PIL import Image, ImageFile\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\n!pip install efficientnet_pytorch --upgrade\nfrom efficientnet_pytorch import EfficientNet\n\ndevice = torch.device(\"cuda:0\")\n\nclass Logger(object):\n    \"\"\"Save a string line(s) to a file.\"\"\"\n    \n    def __init__(self, file_path, mode='w', verbose=False):\n        self.file_path = file_path\n        self.verbose = verbose\n        open(file_path, mode=mode)\n        \n    def append(self, line, print_line=None):\n        if print_line or self.verbose:\n            print(line)\n        with open(self.file_path, 'a') as f:\n            with redirect_stdout(f):\n                print(line)   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CyclicLR(object):\n    \"\"\"Sets the learning rate of each parameter group according to\n    cyclical learning rate policy (CLR). The policy cycles the learning\n    rate between two boundaries with a constant frequency, as detailed in\n    the paper `Cyclical Learning Rates for Training Neural Networks`_.\n    The distance between the two boundaries can be scaled on a per-iteration\n    or per-cycle basis.\n    Cyclical learning rate policy changes the learning rate after every batch.\n    `batch_step` should be called after a batch has been used for training.\n    To resume training, save `last_batch_iteration` and use it to instantiate `CycleLR`.\n    This class has three built-in policies, as put forth in the paper:\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each\n        cycle iteration.\n    This implementation was adapted from the github repo: `bckenstler/CLR`_\n    Args:\n        optimizer (Optimizer): Wrapped optimizer.\n        base_lr (float or list): Initial learning rate which is the\n            lower boundary in the cycle for eachparam groups.\n            Default: 0.001\n        max_lr (float or list): Upper boundaries in the cycle for\n            each parameter group. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore\n            max_lr may not actually be reached depending on\n            scaling function. Default: 0.006\n        step_size (int): Number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch. Default: 2000\n        mode (str): One of {triangular, triangular2, exp_range}.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n            Default: 'triangular'\n        gamma (float): Constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n            Default: 1.0\n        scale_fn (function): Custom scaling policy defined by a single\n            argument lambda function, where\n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored\n            Default: None\n        scale_mode (str): {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on\n            cycle number or cycle iterations (training\n            iterations since start of cycle).\n            Default: 'cycle'\n        last_batch_iteration (int): The index of the last batch. Default: -1\n    Example:\n        >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)\n        >>> scheduler = torch.optim.CyclicLR(optimizer)\n        >>> data_loader = torch.utils.data.DataLoader(...)\n        >>> for epoch in range(10):\n        >>>     for batch in data_loader:\n        >>>         scheduler.batch_step()\n        >>>         train_batch(...)\n    .. _Cyclical Learning Rates for Training Neural Networks: https://arxiv.org/abs/1506.01186\n    .. _bckenstler/CLR: https://github.com/bckenstler/CLR\n    \"\"\"\n\n    def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,\n                 step_size=2000, mode='triangular', gamma=1.,\n                 scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):\n\n        if not isinstance(optimizer, Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(\n                type(optimizer).__name__))\n        self.optimizer = optimizer\n\n        if isinstance(base_lr, list) or isinstance(base_lr, tuple):\n            if len(base_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} base_lr, got {}\".format(\n                    len(optimizer.param_groups), len(base_lr)))\n            self.base_lrs = list(base_lr)\n        else:\n            self.base_lrs = [base_lr] * len(optimizer.param_groups)\n\n        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n            if len(max_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} max_lr, got {}\".format(\n                    len(optimizer.param_groups), len(max_lr)))\n            self.max_lrs = list(max_lr)\n        else:\n            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n\n        self.step_size = step_size\n\n        if mode not in ['triangular', 'triangular2', 'exp_range'] \\\n                and scale_fn is None:\n            raise ValueError('mode is invalid and scale_fn is None')\n\n        self.mode = mode\n        self.gamma = gamma\n\n        if scale_fn is None:\n            if self.mode == 'triangular':\n                self.scale_fn = self._triangular_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = self._triangular2_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = self._exp_range_scale_fn\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n\n        self.batch_step(last_batch_iteration + 1)\n        self.last_batch_iteration = last_batch_iteration\n\n    def batch_step(self, batch_iteration=None):\n        if batch_iteration is None:\n            batch_iteration = self.last_batch_iteration + 1\n        self.last_batch_iteration = batch_iteration\n        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n            param_group['lr'] = lr\n\n    def _triangular_scale_fn(self, x):\n        return 1.\n\n    def _triangular2_scale_fn(self, x):\n        return 1 / (2. ** (x - 1))\n\n    def _exp_range_scale_fn(self, x):\n        return self.gamma**(x)\n\n    def get_lr(self):\n        step_size = float(self.step_size)\n        cycle = np.floor(1 + self.last_batch_iteration / (2 * step_size))\n        x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)\n\n        lrs = []\n        param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)\n        for param_group, base_lr, max_lr in param_lrs:\n            base_height = (max_lr - base_lr) * np.maximum(0, (1 - x))\n            if self.scale_mode == 'cycle':\n                lr = base_lr + base_height * self.scale_fn(cycle)\n            else:\n                lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)\n            lrs.append(lr)\n        return lrs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# this takes a while, you need to change code a bit to train without apex\n!git clone https://github.com/NVIDIA/apex\n!pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" apex/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -r apex","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from apex.parallel import DistributedDataParallel as DDP\nfrom apex.fp16_utils import *\nfrom apex import amp, optimizers\nfrom apex.multi_tensor_apply import multi_tensor_applier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image, ImageFile, ImageFilter\nimport cv2\n\ndef pil_to_cv2(image):\n    return np.array(image).astype(np.uint8)\n\n\ndef get_train_image(rec, ds=\"train19\", image_size=420):\n    if ds == \"train19\":\n        \n        img_name = os.path.join('../input/aptos2019-blindness-detection/train_images', rec['id_code'] + '.png')\n        diag = rec['diagnosis']\n        src = '2019'\n    elif ds == \"test19\":\n        img_name = os.path.join('../input/aptos2019-blindness-detection/test_images', rec['id_code'] + '.png')\n        diag = 0\n        src = '2019'\n    elif ds == \"train15\":\n        img_name = os.path.join(\"../input/resized-2015-2019-blindness-detection-images/resized train 15\", rec['image'] + '.jpg')\n        diag = rec[\"level\"]\n        src = \"2015\"\n    elif ds == \"test15\":\n        img_name = os.path.join(\"../input/resized-2015-2019-blindness-detection-images/resized test 15\", rec['image'] + '.jpg')\n        diag = rec[\"level\"]\n        src = \"2015\"\n    image = cv2.imread(img_name)\n    img = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    \n    # center retina\n    if img.ndim == 2:\n        mask = img > 7\n        img = img[np.ix_(mask.any(1), mask.any(0))]\n    elif img.ndim == 3:\n        mask = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) > 7\n        check_shape = img[:,:,0][np.ix_(mask.any(1), mask.any(0))].shape[0]\n        if check_shape != 0:\n            image1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            image2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            image3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([image1,image2,image3],axis=-1)\n\n\n    # scale radius\n    scale = image_size\n    \n   # set cropping percentage\n    height, width, _ = img.shape\n    ratio = (width, height)\n    if ratio == (640, 480):\n\n        img = Image.fromarray(img)\n        img = np.clip( (np.array(img, dtype=int) - \\\n                          np.array(img.filter(ImageFilter.GaussianBlur(radius=scale/30)), dtype=int)) \\\n                        * 4 + 128, 0, 255)        \n        \n        new_sz = int(640 * 1.15)\n\n        delta_w = new_sz - img.shape[1]\n        delta_h = new_sz - img.shape[0]\n        top = delta_h//2\n        side = delta_w//2\n\n        img = cv2.copyMakeBorder(img, top, top, side, side, cv2.BORDER_CONSTANT, value=[128, 128, 128])\n        #img = cv2.copyMakeBorder(img, top, top, side, side, cv2.BORDER_CONSTANT, value=[0, 0, 0])\n        \n        r = new_sz / 2\n        s = scale * 1.0 / r\n        img = cv2.resize(img.astype(np.uint8), (0,0), fx=s, fy=s)\n        \n    else:\n        x = img[int(img.shape[0]/2),:,:].sum(1)\n        r = (x>x.mean() / 10).sum() / 2\n        s = scale * 1.0 / r\n        img = cv2.resize(img, (0,0), fx=s, fy=s)\n\n        # remove local mean color & pad\n        #img = cv2.addWeighted(img, 4, cv2.GaussianBlur(img, (0,0), scale/30), -4, 128)\n        img = Image.fromarray(img)\n        img = np.clip( (np.array(img, dtype=int) - \\\n                          np.array(img.filter(ImageFilter.GaussianBlur(radius=scale/30)), dtype=int)) \\\n                        * 4 + 128, 0, 255)\n\n\n\n    # pad\n    height, width, _ = img.shape\n    if height > width:\n        padder = albumentations.Compose([\n                    albumentations.augmentations.transforms.PadIfNeeded(\n                        min_height=height, min_width=height, value=(128, 128, 128), border_mode=0, always_apply=True)])\n        img = padder(image=img)\n        img = img['image'] \n    elif width > height:\n        padder = albumentations.Compose([\n                    albumentations.augmentations.transforms.PadIfNeeded(\n                        min_height=width, min_width=width, value=(128, 128, 128), border_mode=0, always_apply=True)])\n        img = padder(image=img)\n        img = img['image'] \n\n    # circle crop\n    b = np.zeros(img.shape)\n    cv2.circle(b, (img.shape[1]//2, img.shape[0]//2), int(scale*0.9), (1,1,1), -1, 8, 0)\n    img = img*b+128*(1-b)\n\n    img = cv2.resize(img, (image_size, image_size))\n    #img = Image.fromarray(img.astype(np.uint8))\n    img = img.astype(np.uint8)\n\n    return {'image': img, 'diag': diag, 'src': src}\n\n\n\nclass RetinopathyDataset(Dataset):\n    def __init__(self, transform, ds=\"train19\", preload=False):\n        if ds == \"train19\":\n            df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n        elif ds == \"test19\":\n            df = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\n        elif ds == \"train15\":\n            df = pd.read_csv('../input/resized-2015-2019-blindness-detection-images/labels/trainLabels15.csv')\n        elif ds == \"test15\":\n            df = pd.read_csv('../input/resized-2015-2019-blindness-detection-images/labels/testLabels15.csv')\n        self.ds = ds\n        self.df = df\n        self.preload = preload\n        if self.preload:\n            self.data = Parallel(n_jobs=2, temp_folder=\"/tmp\", max_nbytes=None, backend=\"multiprocessing\")\\\n                        (delayed(get_train_image)(df.iloc[idx], ds) for idx in tqdm(df.index))\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        #image = self.transform(self.data[idx]['image'])\n        if self.preload:\n            rec = self.data[idx]\n        else:\n            rec = get_train_image(self.df.iloc[idx], self.ds)\n        if self.transform is not None:\n            image = self.transform(image=rec[\"image\"])[\"image\"]\n            #print(image)\n        else:\n            image = rec[\"image\"]\n        target = np.zeros(5)\n        target[rec['diag']] = 1\n        return {'image': image, 'diag1':rec['diag'], 'diag2': target, 'src': rec[\"src\"]}\n        return self.data[idx]\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def aug_image(is_infer=False, transform_type=0):\n    if is_infer:\n        if transform_type == 0:\n            aug = albumentations.Compose([\n                albumentations.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), p=1.0),\n                AT.ToTensor()\n            ])\n        elif transform_type == 1:\n            aug = albumentations.Compose([\n                albumentations.HorizontalFlip(always_apply=True),\n                albumentations.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), p=1.0),\n                AT.ToTensor()\n            ])            \n        elif transform_type == 2:\n            aug = albumentations.Compose([\n                albumentations.VerticalFlip(always_apply=True),\n                albumentations.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), p=1.0),\n                AT.ToTensor()\n            ])                        \n        elif transform_type == 3:\n            aug = albumentations.Compose([\n                albumentations.VerticalFlip(always_apply=True),\n                albumentations.HorizontalFlip(always_apply=True),\n                albumentations.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), p=1.0),\n                AT.ToTensor()\n            ])            \n    else:\n        aug = albumentations.Compose([\n            albumentations.Rotate(limit=(-120,120)),\n            albumentations.HorizontalFlip(),\n            albumentations.VerticalFlip(),\n            albumentations.OneOf([\n                albumentations.CLAHE(clip_limit=2.0),\n                albumentations.RandomBrightnessContrast(),\n            ], p=0.3),            \n            albumentations.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), p=1.0),\n            AT.ToTensor()\n        ])\n    return aug\n\n# comment pickle loading and uncomment this if you want to change preprocessing\ntrain_dataset_2019 = RetinopathyDataset(transform=aug_image(is_infer=False), ds=\"train19\", preload=True)\n\n# import pickle\n# with open('../input/train-dataset-2019-v1/train_dataset_2019.p', \"rb\") as output_file:\n#     train_dataset_2019 = pickle.load(output_file)\n    \nval_dataset_2019 = copy.copy(train_dataset_2019)\n\ntrain_dataset_2019.transform = aug_image(is_infer=False)\nval_dataset_2019.transform = aug_image(is_infer=True, transform_type=0)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.imshow(val_dataset_2019[120][\"image\"].permute(1, 2, 0)  )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/b5_is=420_bs=24_lr=0.002_seed=123_v5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 0\nn_folds = 4\nchunk_size = 1000\nn_workers = 2\n\nimage_size = (420, 420)\nn_epochs = 4\nn_freeze = 0\npatience = 5\nmodel_weight = 15\n\nbag_size = 3\nn_tta = 4\n\nbatch_size = 8\ngradient_accumulation = 4\n\nlr = 5e-4\nparallel = True\n\npretrain_path = '../input/b5_is=420_bs=24_lr=0.002_seed=123_v5'\nmodel_path = '.'\nmodel_type = \"efficientnet-b5\"\nmodel_name = f\"{model_type}__tile=f_fr={n_freeze}_is={image_size}_bs={batch_size}_ga={gradient_accumulation}_lr={lr}_seed={seed}_v=3\"\npretrained_file_path = f\"{pretrain_path}/weights_{model_weight}.pt\"\n\nif not os.path.exists(model_path):\n    os.makedirs(model_path)\n\n# save parameters        \nlogger = Logger(f\"{model_path}/log.txt\", verbose=True)\nlogger.append(model_name)\nlogger.append(\"\\nParameters:\\n  \" + \"\\n  \".join([\n    f\"seed: {seed}\", f\"n_folds: {n_folds}\", f\"n_workers: {n_workers}\",\n    f\"n_epochs: {n_epochs}\", f\"n_freeze: {n_freeze}\", \n    f\"batch_size: {batch_size}\", f\"image_size: {image_size}\", f\"lr: {lr}\",\n    f\"bag size: {bag_size}\", f\"n_tta: {n_tta}\",\n]))\n\n# frozen_layer_map = {\n#     \"efficientnet-b2\": 299,\n#     \"efficientnet-b3\": 338,\n#     \"efficientnet-b4\": 416,\n#     \"efficientnet-b5\": 504,\n#     \"efficientnet-b6\": 582,\n#     \"efficientnet-b7\": 709,\n# }\nclass OwnSampler(torch.utils.data.sampler.Sampler):\n    def __init__(self, idx):\n        self.idx = idx\n    def __iter__(self):\n        return iter(self.idx)\n    def __len__(self):\n        return len(self.idx)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=42)\n\ncv_targets = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')['diagnosis']\ncv_preds = np.zeros(len(cv_targets))\n\n\n\ncv_results = np.zeros((len(cv_targets), bag_size))\n\ndef pred_to_int(x):\n    return np.round(np.clip(x, 0, 4))\n\nfor bag in range(bag_size):\n    print(f\"Bag {bag}\")\n    \n    for fold, (train_index, valid_index) in enumerate(kf.split(train_dataset_2019, cv_targets)):\n#         if fold > 0:\n#             continue\n        print()\n        print(f\"Fold {fold}\")\n        train_index = train_index\n        valid_index = valid_index\n        \n        train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_index)\n        valid_sampler = OwnSampler(valid_index)\n        \n        train_loader = torch.utils.data.DataLoader(train_dataset_2019, batch_size=batch_size, \n                                                   sampler=train_sampler, num_workers=n_workers)\n       \n\n        model = EfficientNet.from_pretrained(model_type, num_classes=1) \n        for i, param in enumerate(model.parameters()):\n            if i < 502:\n                param.requires_grad = False\n        model = model.cuda()\n    \n        criterion = torch.nn.MSELoss()\n        \n        lr = 0.001\n        #optimizer = adabound.AdaBound(model.parameters(), lr=lr, final_lr=0.1, eps=2e-4, weight_decay=1e-5)\n        optimizer = torch.optim.Adam(model.parameters(), lr=lr, eps=2e-4, weight_decay=1e-5)\n        \n        #scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5)\n        #scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, n_epochs, eta_min=0, last_epoch=-1)\n        scheduler = CyclicLR(optimizer, step_size=len(train_loader)*n_epochs/2, base_lr=0.00001, max_lr=0.0005)\n                \n        model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\", verbosity=0) \n        \n        if parallel:\n            model = torch.nn.DataParallel(model).cuda()\n        \n        model.load_state_dict(torch.load(pretrained_file_path))\n        \n        early_stopping_count = 0\n        best_kappa = -np.Inf\n        for epoch in range(n_epochs):\n            \n            if epoch == n_freeze:\n                for i, param in enumerate(model.parameters()):\n                    param.requires_grad = True \n                del param\n            torch.cuda.empty_cache()\n            \n            start_time = time.time()\n            \n            model.train() \n            avg_loss = 0.0\n            optimizer.zero_grad()\n            all_preds = []\n            all_targets = []\n            for idx, data in enumerate(train_loader):\n                images = data['image']\n                labels = data['diag1'].float()\n                preds = model(images.cuda())\n                loss = criterion(preds.reshape(-1), labels.float().cuda())\n                avg_loss += loss.item() / len(train_loader)\n                # gradient accumulation\n                loss = loss / gradient_accumulation\n                with amp.scale_loss(loss, optimizer) as scaled_loss:\n                    scaled_loss.backward()\n                if (idx+1) % gradient_accumulation == 0:\n                    torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0, norm_type=2)\n                    optimizer.step() \n                    optimizer.zero_grad() \n                    \n                all_preds += list(preds.detach().cpu().numpy())\n                all_targets += list(labels.detach().cpu().numpy())\n                \n                scheduler.batch_step()\n                \n            all_preds = np.array(all_preds)\n            all_targets = np.array(all_targets)\n            \n\n\n            kappa_train = (metrics.cohen_kappa_score(all_targets, pred_to_int(all_preds), weights='quadratic'))\n            mse_train = (metrics.mean_squared_error(all_targets, all_preds))   \n                    \n            model.eval()\n            avg_val_loss = 0.0\n            \n            all_preds = np.zeros(len(valid_index))\n            all_targets = np.zeros(len(valid_index))\n            with torch.no_grad():\n                \n                for i in range(n_tta):\n                    val_dataset_2019.transform = aug_image(is_infer=True, transform_type=i)\n                    valid_loader = torch.utils.data.DataLoader(val_dataset_2019, batch_size=batch_size, shuffle=False,\n                                                  sampler=valid_sampler, num_workers=n_workers)\n                    \n                    for idx, data in enumerate(valid_loader):\n                        images = data['image']\n                        labels = data['diag1'].float()\n                        preds = model(images.cuda())\n                        loss = criterion(preds.reshape(-1), labels.float().cuda())\n                        avg_val_loss += loss.item() / len(valid_loader) / n_tta\n                        \n                        all_preds[idx * batch_size:(idx + 1) * batch_size] += preds.detach().cpu().numpy().reshape(-1) / n_tta\n                        all_targets[idx * batch_size:(idx + 1) * batch_size] = labels.detach().cpu().numpy()\n\n            kappa = (metrics.cohen_kappa_score(all_targets, pred_to_int(all_preds), weights='quadratic'))\n            mse = (metrics.mean_squared_error(all_targets, all_preds))         \n                    \n            elapsed_time = time.time() - start_time \n        \n            # log results\n            logger.append(\n                f\"{epoch:>2d}: time={elapsed_time:0.2f}s  \"\n                f\"lr={scheduler.get_lr()[0]:<8.3g}\"\n                f\"loss={avg_loss:0.5f}  \"\n                f\"kappa={kappa_train:0.5f}  \"  \n                f\"mse={mse_train:0.5f}  \"  \n                f\"val_loss={avg_val_loss:0.5f}  \"   \n                f\"val_kappa={kappa:0.5f}  \"  \n                f\"val_mse={mse:0.5f}  \"  \n            )\n\n            #scheduler.step()\n\n            # check for improvement\n            #if kappa > best_kappa:\n            best_preds = all_preds\n            early_stopping_count = 0\n            best_kappa = kappa\n            torch.save(model.state_dict(), f\"{model_path}/weights_{bag}_{fold}.pt\")\n#             else:\n#                 early_stopping_count += 1\n#                 if early_stopping_count == patience:\n#                     break\n                    \n        cv_preds[valid_index] += best_preds / bag_size\n        print(best_kappa)\n        print(\"Bag kappa:\", metrics.cohen_kappa_score(all_targets, pred_to_int(cv_preds[valid_index]), weights='quadratic'))\n        #break\n        \n        del model\n        del train_loader\n        del valid_loader\n        del images\n        del labels\n        torch.cuda.empty_cache()\n        gc.collect()\n            \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Bag kappa:\", metrics.cohen_kappa_score(cv_targets, pred_to_int(cv_preds), weights='quadratic'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save(f\"{model_path}/cv_preds\", cv_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset_2019 = RetinopathyDataset(transform=aug_image(is_infer=True), ds=\"test19\", preload=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.imshow(test_dataset_2019[120][\"image\"].permute(1, 2, 0)  )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = []\n\nfor i, model_name in enumerate(os.listdir(model_path)):\n    \n    if \"pt\" not in model_name:\n        continue\n    \n    model = EfficientNet.from_name(model_type)\n    model._fc = nn.Linear(model._fc.in_features, 1)\n    if parallel:\n        model = torch.nn.DataParallel(model).cuda()\n        \n    model.load_state_dict(torch.load(f\"{model_path}/{model_name}\"))\n    model = model.cuda()\n    \n    for param in model.parameters():\n        param.requires_grad = False\n\n    model.eval()\n    models.append(model)\n\nprint(len(models))\nprint(\"Done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = np.zeros(len(test_dataset_2019))\nwith torch.no_grad():\n\n    for i in range(n_tta):\n        test_dataset_2019.transform = aug_image(is_infer=True, transform_type=i)\n        test_loader = torch.utils.data.DataLoader(test_dataset_2019, batch_size=batch_size, shuffle=False, num_workers=n_workers)\n\n        for idx, data in enumerate(tqdm(test_loader)):\n            images = data['image']\n            labels = data['diag1'].float()\n            \n            for model in models:\n                preds = model(images.cuda())\n                test_preds[idx * batch_size:(idx + 1) * batch_size] += preds.detach().cpu().numpy().reshape(-1) / n_tta / len(models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save(f\"{model_path}/test_preds\", test_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = pred_to_int(test_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\nsubmission['diagnosis'] = test_preds\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\n%matplotlib inline\n\nsns.distplot(test_preds, kde=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}