{"cells":[{"metadata":{},"cell_type":"markdown","source":"Regression with pre-defined cutpoints just does not feel correct to me for this kind of problem, even with kappa correction. Thanks to the author of this discussion: https://www.kaggle.com/c/aptos2019-blindness-detection/discussion/97893, I found spacecutter and conclude that ordinal regression is a better approach conceptually. Bear in mind that I did not compare the kappa score between regression from 0 - 4 or classification with this ordinal regression model, so performance-wise, I do not really know if it does improve.\n\nIf you want to know more about ordinal regression, please refer to this blog by the author of spacecutter: https://www.ethanrosenthal.com/2018/12/06/spacecutter-ordinal-regression/. Also worth noting that this package offers the ability to determine class weights, which is really handy for imbalance dataset.\n\nI also borrows chunks of code from these excellent kernels:\n- https://www.kaggle.com/ratthachat/aptos-updatedv14-preprocessing-ben-s-cropping\n- https://www.kaggle.com/chanhu/eye-efficientnet-pytorch-lb-0-777\n- https://www.kaggle.com/drhabib/starter-kernel-for-0-79\n\nIn addition to ordinal regression, I also use adaptive learning rate for pytorch (https://towardsdatascience.com/adaptive-and-cyclical-learning-rates-using-pytorch-2bf904d18dee) and TTA. \n\nWhen you see the actual cutpoints (I print them during training), it is indeed the case that the gaps are similar between the classes. Maybe 0-4 regression will be enough to serve the purpose after all."},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nimport sys\nimport os\nimport time\nimport math\nimport gc\nfrom functools import partial\n\nimport cv2\n\nimport numpy as np\nimport scipy as sp\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\n\nfrom tqdm._tqdm_notebook import tqdm_notebook as tqdm\n# tqdm.pandas()\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score, mean_absolute_error, confusion_matrix\n\nimport torch\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\nspacecutter_package_path = '../input/spacecutter/spacecutter-master/spacecutter-master/'\nsys.path.append(spacecutter_package_path)\nfrom spacecutter.models import OrdinalLogisticModel\nfrom spacecutter.losses import CumulativeLinkLoss\n\nenet_package_path = '../input/efficientnet/efficientnet-pytorch/EfficientNet-PyTorch/'\nsys.path.append(enet_package_path)\nfrom efficientnet_pytorch import EfficientNet\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nDEBUG = False    # always change this to 'False' before comitting,\n                # you can change to 'True' during editing to use cache\n                # and make subsequent training faster\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(1336)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PyTorch-style Dataset Class"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class RetinaDataset(Dataset):\n    CACHE_DIR = 'cache'\n    \n    def __init__(self, dataframe, img_size, img_scale, train_transform, use_base_transform, use_cache=False):\n        if use_cache and not os.path.exists(self.CACHE_DIR): os.mkdir(self.CACHE_DIR)\n        self.use_cache = use_cache\n        self.df = dataframe\n        self.train_transform = train_transform\n        self.img_size = img_size\n        self.img_scale = img_scale\n        self.epoch = 0\n        self.use_base_transform = use_base_transform\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        label = self.df.diagnosis.values[idx]\n        label = np.expand_dims(label, -1)\n        img_path = self.df.img_path.values[idx]\n        id_code = self.df.index.values[idx]\n        if self.use_cache:\n            filename = id_code + \".png\"\n            cache_path = os.path.join(self.CACHE_DIR, id_code+\".png\")\n            cached = os.path.exists(cache_path)\n            try:\n                imgpil = Image.open(cache_path)\n                imgpil = self.train_transform(imgpil)\n            except (OSError, IOError) as e:\n                imgpil = self.load_base_transform(img_path)\n                imgpil.save(cache_path,\"PNG\")\n                imgpil = self.train_transform(imgpil)\n        else:\n            imgpil = self.load_base_transform(img_path)\n            imgpil = self.train_transform(imgpil)\n        imgt = transforms.ToTensor()(imgpil)\n        imgt = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])(imgt)\n        return imgt, label\n        \n    def crop_image_from_gray(self, img, tol=7):\n        if img.ndim==2:\n            mask = img>tol\n            return img[np.ix_(mask.any(1),mask.any(0))]\n        elif img.ndim==3:\n            gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n            mask = gray_img>tol\n\n            check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n            if (check_shape == 0): # image is too dark so that we crop out everything,\n                return img # return original image\n            else:\n                img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n                img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n                img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n                img = np.stack([img1,img2,img3],axis=-1)\n            return img\n\n    def load_base_transform(self, img_path):\n        if self.use_base_transform is None or len(self.use_base_transform) == 0:\n            imgpil = Image.open(img_path)\n            w, h = imgpil.size\n            base_size = int(self.img_size * self.img_scale)\n            w_new = base_size if w <= h else int(w * base_size / h)\n            h_new = base_size if h <= w else int(h * base_size / w)\n            return imgpil.resize((w_new, h_new), Image.ANTIALIAS)\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        if 'crop' in self.use_base_transform:\n            img = self.crop_image_from_gray(img)\n        # resize\n        w, h, _ = img.shape\n        base_size = int(self.img_size * self.img_scale)\n        w_new = base_size if w <= h else int(w * base_size / h)\n        h_new = base_size if h <= w else int(h * base_size / w)\n        img = cv2.resize(img, (h_new, w_new))\n        # ben's preprocessing\n        if 'weighted' in self.use_base_transform:\n            img = cv2.addWeighted(img, 4, cv2.GaussianBlur(img, (0,0) , 10), -4 ,128)\n        return transforms.ToPILImage()(img)\n    \n    def show_sample_imgs(self, n, get_original, use_train_transform, per_row=2):\n        samples = self.df.sample(n=n)\n        img_names = samples.index.values\n        rows = (n + per_row - 1)//per_row\n        cols = min(per_row, n)\n        fig, axes = plt.subplots(rows, cols, figsize=(15,15))\n        for ax in axes.flatten(): \n            ax.axis('off')\n        for i,(img_name, ax) in enumerate(zip(img_names, axes.flatten())): \n            img_path = self.df.loc[img_name].img_path\n            if get_original:\n                imgpil = Image.open(img_path)\n            else:\n                imgpil = self.load_base_transform(img_path)\n            if use_train_transform:\n                imgpil = self.train_transform(imgpil)\n            ax.imshow(imgpil)\n            ax.set_title(self.df.loc[img_name].diagnosis)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper class for logging during model training"},{"metadata":{"trusted":true},"cell_type":"code","source":"class NNLogger(object):\n    \n    def __init__(self):\n        # mini-batch-oriented\n        self.y_true = {'train': [], 'val': []}\n        self.y_pred = {'train': [], 'val': []}\n        self.y_true = {'train': [], 'val': []}\n        self.y_pred = {'train': [], 'val': []}\n        self.loss = {'train': [], 'val': []}\n        self.elapsed_time = []\n        self.lr_history = []\n        # epoch-oriented\n        self.current_epoch = 1\n    \n    def step(self):\n        self.current_epoch += 1\n        \n    def record_stat(self, lr, elapsed_time):\n        self.lr_history.append(lr)\n        self.elapsed_time.append(elapsed_time)\n        \n    def add(self, mode, y_true, y_pred, loss):\n        if len(self.y_true[mode]) < self.current_epoch:\n            self.y_true[mode].append([*y_true])\n            self.y_pred[mode].append([*y_pred])\n            self.loss[mode].append([loss])\n        else:\n            self.y_true[mode][self.current_epoch-1] = [*self.y_true[mode][self.current_epoch-1], *y_true]\n            self.y_pred[mode][self.current_epoch-1] = [*self.y_pred[mode][self.current_epoch-1], *y_pred]\n            self.loss[mode][self.current_epoch-1].append(loss)\n        \n    def get_last_kappa_score(self):\n        return (cohen_kappa_score(self.y_true['train'][-1], self.y_pred['train'][-1], weights='quadratic'),\n                cohen_kappa_score(self.y_true['val'][-1], self.y_pred['val'][-1], weights='quadratic'))\n    \n    def get_last_mae(self):\n        return (mean_absolute_error(self.y_true['train'][-1], self.y_pred['train'][-1]),\n                mean_absolute_error(self.y_true['val'][-1], self.y_pred['val'][-1]))\n    \n    def get_last_cm(self):\n        return confusion_matrix(self.y_true['val'][-1], self.y_pred['val'][-1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Trainer class\n\nincluding the training loop."},{"metadata":{"trusted":true},"cell_type":"code","source":"class BlindnessDetectionTrainer(object):\n    # training sets\n    _train_2015_csv = '../input/resized-2015-2019-blindness-detection-images/labels/trainLabels15.csv'\n    _train_2015_img_path = '../input/resized-2015-2019-blindness-detection-images/resized train 15/'\n    _train_2015_ext = '.jpg'\n    _train_2019_csv = '../input/resized-2015-2019-blindness-detection-images/labels/trainLabels19.csv'\n    _train_2019_img_path = '../input/resized-2015-2019-blindness-detection-images/resized train 19/'\n    _train_2019_ext = '.jpg'\n\n    def __init__(self, load_pretrained=True, freeze_pretrained=False, use_cache=False, test_size=0.1):\n        self.use_cache = use_cache\n        # load training set 2015\n        self.train_2015_csv = pd.read_csv(self._train_2015_csv, names=['id_code', 'diagnosis'], skiprows=1)\n        self.train_2015_csv['img_path'] = self.train_2015_csv['id_code'].apply(\n            lambda f: self._train_2015_img_path + f + self._train_2015_ext if os.path.isfile(\n                self._train_2015_img_path + f + self._train_2015_ext) else None)\n        self.train_2015_csv = self.train_2015_csv.set_index('id_code')\n        self.train_2015_df = self.train_2015_csv.dropna()\n        self.train_2015_df['type'] = 'old' # indicate data from old competition (2015)\n        # downsampling class 0\n        class_dist_2015 = [\n            7000,\n            len(self.train_2015_df[self.train_2015_df['diagnosis'] == 1]),\n            len(self.train_2015_df[self.train_2015_df['diagnosis'] == 2]),\n            len(self.train_2015_df[self.train_2015_df['diagnosis'] == 3]),\n            len(self.train_2015_df[self.train_2015_df['diagnosis'] == 4]),\n        ]\n        self.train_2015_df = self.resample(self.train_2015_df, class_dist_2015)\n        # load training set 2019\n        self.train_2019_csv = pd.read_csv(self._train_2019_csv, names=['id_code', 'diagnosis'], skiprows=1)\n        self.train_2019_csv['img_path'] = self.train_2019_csv['id_code'].apply(\n            lambda f: self._train_2019_img_path + f + self._train_2019_ext if os.path.isfile(\n                self._train_2019_img_path + f + self._train_2019_ext) else None)\n        self.train_2019_csv = self.train_2019_csv.set_index('id_code')\n        self.train_2019_df = self.train_2019_csv.dropna()\n        self.train_2019_df['type'] = 'new' # indicate data from new competition (2019)\n        # create validation set from 2019\n        self.train_2019_df, self.val_df = train_test_split(\n            self.train_2019_df, test_size=0.1, stratify=self.train_2019_df.diagnosis, random_state=1337)\n        # combine training set\n        self.train_df = pd.concat([self.train_2015_df, self.train_2019_df], axis=0)\n        self.init_model(load_pretrained, freeze_pretrained)\n        self.best_score = None\n        self.logger = NNLogger()\n        \n    def init_model(self, pretrained, freeze, num_classes=1):\n        enet = EfficientNet.from_name('efficientnet-b0')\n        if pretrained:\n            enet.load_state_dict(torch.load('../input/efficientnet-pytorch/efficientnet-b0-08094119.pth'))\n        if freeze:\n            for parameter in enet.parameters():\n                parameter.requires_grad = True\n        n_fc = enet._fc.in_features\n        enet._fc = nn.Sequential(\n                          nn.Dropout(p=0.5),\n                          nn.Linear(in_features=n_fc, out_features=n_fc, bias=True),\n                          nn.ReLU(),\n                          nn.BatchNorm1d(n_fc, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n                          nn.Dropout(p=0.5),\n                          nn.Linear(in_features=n_fc, out_features=1, bias=True),\n                         )\n        self.model = OrdinalLogisticModel(enet, num_classes=5)\n        self.model.to(device)\n        \n    def load_best_state_dict(self):\n        self.model.load_state_dict(self.best_state_dict)\n        \n    def unfreeze(self):\n        for parameter in self.model.parameters():\n            parameter.requires_grad = True\n    \n    def freeze_except_fc(self):\n        for _, child in trainer.model.named_children():\n            for name, params in child.named_parameters():\n                if '_fc.' not in name and name != 'cutpoints':\n                    params.required_grad = False\n                    \n    def resample(self, df, class_dist):\n        resample_df = []\n        for label, n_sample in enumerate(class_dist):\n            if len(df[df['diagnosis'] == label]) < n_sample:\n                resample_df.append(df[df['diagnosis'] == label].sample(n=n_sample, replace=True))\n            else:\n                resample_df.append(df[df['diagnosis'] == label].sample(n=n_sample))\n        return pd.concat(resample_df, axis=0)\n                    \n    def adjusted_weights(self, weights):\n        return [weight * len(weights) / sum(weights) for weight in weights]\n    \n    def lr_finder(self, lr_find_epochs, start_lr, end_lr, train_type,\n                  img_size, img_scale, train_transform, use_base_transform, batch_size, \n                  class_weights, num_workers=4):\n        dataset = RetinaDataset(self.train_df[self.train_df['type'] == train_type],\n                                img_size=img_size, img_scale=img_scale,\n                                train_transform=train_transform,\n                                use_base_transform=use_base_transform,\n                                use_cache=self.use_cache)\n        dataloader = DataLoader(dataset, batch_size=batch_size, \n                                shuffle=True, num_workers=num_workers)\n        lr_lambda = lambda x: math.exp(x * math.log(end_lr / start_lr) / (lr_find_epochs * len(dataloader)))\n        param_list = [{\"params\": self.model.predictor._conv_stem.parameters(), \"lr\":start_lr * 0.1},\n         {\"params\": self.model.predictor._bn0.parameters(), \"lr\":start_lr * 0.1},\n         {\"params\": self.model.predictor._blocks.parameters(), \"lr\":start_lr * 0.1},\n         {\"params\": self.model.predictor._conv_head.parameters(), \"lr\":start_lr * 0.1},\n         {\"params\": self.model.predictor._bn1.parameters(), \"lr\":start_lr * 0.1},\n         {\"params\": self.model.predictor._fc.parameters()},\n         {\"params\": self.model.link.parameters()}]\n        optimizer = torch.optim.Adam(param_list, lr=start_lr, weight_decay=1e-5)\n        #optimizer = torch.optim.SGD(param_list, lr=start_lr, momentum=0.9, nesterov=True, weight_decay=1e-5)\n        criterion = CumulativeLinkLoss(class_weights=self.adjusted_weights(class_weights))\n        scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)\n        # Make lists to capture the logs\n        lr_find_loss = []\n        lr_find_lr = []\n        iter = 0\n        smoothing = 0.05\n        for i in range(lr_find_epochs):\n            for imgs, labels in tqdm(dataloader, total=len(dataloader), disable=(not DEBUG)):\n                # Send to device\n                imgs, labels = imgs.to(device), labels.to(device)\n                # Training mode and zero gradients\n                self.model.train()\n                optimizer.zero_grad()\n                # Get outputs to calc loss\n                outputs = self.model(imgs)\n                loss = criterion(outputs, labels)\n                # Backward pass\n                loss.backward()\n                optimizer.step()\n                # Update LR\n                scheduler.step()\n                lr_step = optimizer.state_dict()[\"param_groups\"][-1][\"lr\"]\n                lr_find_lr.append(lr_step)\n                # smooth the loss\n                if iter==0:\n                    lr_find_loss.append(loss)\n                else:\n                    loss = smoothing  * loss + (1 - smoothing) * lr_find_loss[-1]\n                    lr_find_loss.append(loss)\n            iter += 1\n        return lr_find_loss, lr_find_lr\n    \n    def cosine_lr(self, stepsize, min_lr, max_lr, lr_scale):\n        \n        def _cosine_lr(it):\n            cycle_no = it // stepsize\n            lr_scale_cycle = lr_scale ** cycle_no\n            scaled_max_lr, scaled_min_lr = max_lr * lr_scale_cycle, min_lr * lr_scale_cycle\n            fraction_to_restart = (it % stepsize) / stepsize\n            lr = scaled_min_lr + 0.5 * (scaled_max_lr - scaled_min_lr) * (1 + np.cos(fraction_to_restart * np.pi))\n            return lr\n        \n        return _cosine_lr\n        \n    def train_loop(self, n_epochs, img_size, img_scale, batch_size, class_weights, lr, lr_scale,\n                   step_size, train_transform, use_base_transform, train_type,\n                   best_score=None, save_model=True, num_workers=4):\n        self.train_dataset = RetinaDataset(self.train_df[self.train_df['type'] == train_type], \n                                           img_size=img_size, img_scale=img_scale,\n                                           train_transform=train_transform,\n                                           use_base_transform=use_base_transform,\n                                           use_cache=self.use_cache)\n        self.val_dataset = RetinaDataset(self.val_df, img_size=img_size, img_scale=img_scale,\n                                         train_transform=train_transform,\n                                         use_base_transform=use_base_transform,\n                                         use_cache=self.use_cache)\n        self.train_dl = DataLoader(self.train_dataset, batch_size=batch_size, \n                                   shuffle=True, num_workers=num_workers)\n        self.val_dl = DataLoader(self.val_dataset, batch_size=batch_size, \n                                 shuffle=True, num_workers=num_workers)\n        self.criterion = CumulativeLinkLoss(class_weights=self.adjusted_weights(class_weights))\n        self.best_score = best_score if best_score is not None else self.best_score\n        epoch_lr_size = step_size * len(self.train_dl)\n        param_list = [{\"params\": self.model.predictor._conv_stem.parameters()},\n         {\"params\": self.model.predictor._bn0.parameters()},\n         {\"params\": self.model.predictor._blocks.parameters()},\n         {\"params\": self.model.predictor._conv_head.parameters()},\n         {\"params\": self.model.predictor._bn1.parameters()},\n         {\"params\": self.model.predictor._fc.parameters()},\n         {\"params\": self.model.link.parameters()}]\n        lr_fn = self.cosine_lr\n        lr_params = [\n            # _conv_stem\n            lr_fn(epoch_lr_size, lr*0.01/6, lr*0.1, lr_scale=lr_scale),\n            #_bn0\n            lr_fn(epoch_lr_size, lr*0.01/6, lr*0.1, lr_scale=lr_scale),\n            #_blocks\n            lr_fn(epoch_lr_size, lr*0.1/6, lr*0.2, lr_scale=lr_scale),\n            #_conv_head\n            lr_fn(epoch_lr_size, lr*0.1/6, lr*0.2, lr_scale=lr_scale),\n            #_bn1\n            lr_fn(epoch_lr_size, lr*0.1/6, lr*0.2, lr_scale=lr_scale),\n            #_fc\n            lr_fn(epoch_lr_size, lr/6, lr, lr_scale=lr_scale),\n            #link\n            lr_fn(epoch_lr_size, lr/6, lr, lr_scale=lr_scale)\n        ]\n        self.optimizer = torch.optim.Adam(param_list, lr=1.)\n        self.scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lr_params)\n        # self.model, self.optimizer = amp.initialize(self.model, self.optimizer, opt_level=\"O1\", verbosity=0)\n        for epoch in range(n_epochs):\n            print('Epoch: {}/{} \\t LR: {}'.format(epoch + 1, n_epochs, self.scheduler.get_lr()[-1]))\n            avg_loss = self.train_model()\n            avg_val_loss = self.eval_model()\n            train_kappa, val_kappa = self.logger.get_last_kappa_score()\n            train_mae, val_mae = self.logger.get_last_mae()\n            print('loss={:.4f} \\t val_loss={:.4f}'.format(avg_loss, avg_val_loss))\n            print('kappa={:.4f} \\t val_kappa={:.4f}'.format(train_kappa, val_kappa))\n            print('MAE={:.4f} \\t val_MAE={:.4f}'.format(train_mae, val_mae))\n            print('cutpoints={}'.format(str(self.model.link.cutpoints.detach().cpu().numpy())))\n            print('Validation CM:')\n            print(self.logger.get_last_cm())\n            self.logger.step()\n            if save_model and (self.best_score is None or val_kappa > self.best_score):\n                print('best score improved to {:.4f}. Saving model.'.format(val_kappa))\n                self.best_score = val_kappa\n                self.best_state_dict = self.model.state_dict()\n                save_score = str(int(self.best_score * 1000))\n                torch.save(self.model.state_dict(), '../working/weight_best_kappa.pt')\n        \n    def train_model(self):\n        self.model.train() \n        avg_loss = 0.\n        self.optimizer.zero_grad()\n        train_iter = tqdm(enumerate(self.train_dl), total=len(self.train_dl), disable=(not DEBUG))\n        for idx, (imgs, labels) in train_iter:\n            start_time = time.time()\n            imgs_train, labels_train = imgs.to(device), labels.to(device)\n            output_train = self.model(imgs_train)\n            pred_diagnosis = output_train.detach().cpu().numpy().argmax(axis=1)\n            loss = self.criterion(output_train, labels_train)\n            self.logger.add('train', labels.numpy().ravel(), pred_diagnosis, loss.item())\n            elapsed_time = time.time() - start_time\n            self.logger.record_stat(self.scheduler.get_lr(), elapsed_time)\n            train_iter.set_postfix(loss=\"{:.6f}\".format(loss.item()))\n#             with amp.scale_loss(loss, self.optimizer) as scaled_loss:\n#                 scaled_loss.backward()\n            loss.backward()\n            self.optimizer.step() \n            self.scheduler.step()\n            self.optimizer.zero_grad() \n            avg_loss += loss.item() / len(self.train_dl)\n        return avg_loss\n    \n    def eval_model(self):\n        self.model.eval()\n        avg_val_loss = 0.\n        with torch.no_grad():\n            for idx, (imgs, labels) in tqdm(enumerate(self.val_dl), total=len(self.val_dl), disable=(not DEBUG)):\n                imgs_val, labels_val = imgs.to(device), labels.to(device)\n                output_val = self.model(imgs_val)\n                val_diagnosis = output_val.detach().cpu().numpy().argmax(axis=1)\n                loss = self.criterion(output_val, labels_val)\n                self.logger.add('val', labels.numpy().ravel(), val_diagnosis, loss.item())\n                avg_val_loss += loss.item() / len(self.val_dl)\n        return avg_val_loss\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_SIZE = 224\ntrain_params = {\n    'n_epochs': 2,\n    'img_size': IMG_SIZE,\n    'img_scale': 1.2,\n    'batch_size': 32,\n    'class_weights': [1, 2, 1, 2, 1],\n    'lr': 2e-3,\n    'lr_scale': 0.8,\n    'step_size': 1,\n    'train_type': 'old',\n    'use_base_transform' : ['weighted'],\n    'train_transform': transforms.Compose([\n            transforms.RandomHorizontalFlip(),\n            transforms.RandomVerticalFlip(),\n            transforms.RandomRotation(90),\n            transforms.ColorJitter(contrast=.1, saturation=.1, brightness=.1),\n            transforms.RandomCrop(IMG_SIZE)\n        ])\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Learning Rate Finder"},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer = BlindnessDetectionTrainer(freeze_pretrained=False, use_cache=DEBUG)\nlr_find_loss, lr_find_lr = trainer.lr_finder(lr_find_epochs=6, start_lr=3e-5, end_lr=3e-2, train_type='old',\n                                             img_size=train_params['img_size'], img_scale=train_params['img_scale'],\n                                             train_transform=train_params['train_transform'], \n                                             use_base_transform=train_params['use_base_transform'], batch_size=train_params['batch_size'], \n                                             class_weights=[1,2,1,2,1])\nfig, ax = plt.subplots(1,1)\n_ = ax.plot(lr_find_lr, lr_find_loss)\n_ = ax.set_xscale('log')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Loop 1: 2015 data train fully-connected only"},{"metadata":{"trusted":true},"cell_type":"code","source":"# warm start by freezing pretrained layers\ntrainer = BlindnessDetectionTrainer(freeze_pretrained=True, use_cache=DEBUG)\ntrainer.train_loop(**train_params)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Show sample of augmented training images from 2015"},{"metadata":{},"cell_type":"markdown","source":"original images"},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.train_dataset.show_sample_imgs(6, get_original=True, use_train_transform=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"augmented images"},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.train_dataset.show_sample_imgs(6, get_original=False, use_train_transform=train_params['train_transform'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Loop 2: 2015 data train all layers"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_params['n_epochs'] = 4\ntrain_params['lr'] = 3e-3\ntrain_params['step_size'] = 2\ntrain_params['train_type'] = 'old'\ntrainer.load_best_state_dict()\ntrainer.unfreeze()\ntrainer.train_loop(**train_params)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Loop 3: 2019 data train fully-connected only"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_params['n_epochs'] = 2\ntrain_params['step_size'] = 1\ntrain_params['lr'] = 1e-3\ntrain_params['train_type'] = 'new'\ntrainer.load_best_state_dict()\ntrainer.freeze_except_fc()\ntrainer.train_loop(**train_params)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Show sample of augmented training images from 2019"},{"metadata":{},"cell_type":"markdown","source":"original images"},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.train_dataset.show_sample_imgs(6, get_original=True, use_train_transform=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"augmented images"},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.train_dataset.show_sample_imgs(6, get_original=False, use_train_transform=train_params['train_transform'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Loop 4: 2019 data train all layers"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_params['n_epochs'] = 8\ntrain_params['step_size'] = 2\ntrain_params['use_base_transform'] = ['weighted']\ntrainer.load_best_state_dict()\ntrainer.unfreeze()\ntrainer.train_loop(**train_params)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluation"},{"metadata":{},"cell_type":"markdown","source":"## Learning Rate Scheduler"},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = plt.plot([lr[-1] for lr in trainer.logger.lr_history])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Losses"},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = plt.plot([np.mean(loss) for loss in trainer.logger.loss['train']])\n_ = plt.plot([np.mean(loss) for loss in trainer.logger.loss['val']])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"class BlindnessDetectionPredictor(object):\n    \n    def __init__(self, subm_df, state_dict_file, transform, use_base_transform, img_size, \n                 img_scale, n_TTA, batch_size):\n        self.subm_df = subm_df\n        self.subm_df['img_path'] = self.subm_df['id_code'].apply(\n            lambda f: _subm_2019_img_path + f + _subm_2019_ext if os.path.isfile(\n                _subm_2019_img_path + f + _subm_2019_ext) else None)\n        self.subm_df = self.subm_df.set_index('id_code')\n        self.subm_dataset = RetinaDataset(self.subm_df, img_size=img_size, img_scale=img_scale,\n                                          train_transform=transform, use_base_transform=use_base_transform)\n        self.subm_dl = DataLoader(self.subm_dataset, batch_size=batch_size, shuffle=False)\n        self.batch_size = batch_size\n        self.n_TTA = n_TTA\n        self.init_model(state_dict_file)\n        \n    def init_model(self, state_dict_file):\n        enet = EfficientNet.from_name('efficientnet-b0')\n        n_fc = enet._fc.in_features\n        enet._fc = nn.Sequential(\n                          nn.Dropout(p=0.5),\n                          nn.Linear(in_features=n_fc, out_features=n_fc, bias=True),\n                          nn.ReLU(),\n                          nn.BatchNorm1d(n_fc, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),\n                          nn.Dropout(p=0.25),\n                          nn.Linear(in_features=n_fc, out_features=1, bias=True),\n                         )\n        self.model = OrdinalLogisticModel(enet, num_classes=5)\n        self.model.load_state_dict(torch.load(state_dict_file))\n        self.model.to(device)\n        \n    def del_model(self):\n        del self.model\n        gc.collect()\n\n    def predict(self):\n        preds = np.zeros((len(self.subm_df), 5))\n        self.model.eval()\n        start_time = time.time()\n        for tta_no in range(self.n_TTA):\n            len_batch = len(self.subm_dl)\n            with torch.no_grad():\n                for i, data in enumerate(self.subm_dl):\n                    images, _ = data\n                    images = images.to(device)\n                    output_subm = self.model(images)\n                    preds[i * self.batch_size:(i + 1) * self.batch_size] += output_subm.detach().cpu().squeeze().numpy()\n                    elapsed_time = time.time() - start_time\n                    print('TTA={}/{}, Batch={}/{}, total_elapsed_time={:.2f}s'.format(\n                        tta_no+1, self.n_TTA, i+1, len_batch, elapsed_time))\n        output = preds / self.n_TTA\n        self.del_model()\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission set\n_subm_2019_csv = '../input/aptos2019-blindness-detection/sample_submission.csv'\n_subm_2019_img_path = '../input/aptos2019-blindness-detection/test_images/'\n_subm_2019_ext = '.png'\nsubmit_df = pd.read_csv(_subm_2019_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_SIZE = 224\nsubm_params = {\n    'subm_df': submit_df,\n    'state_dict_file' : '../working/weight_best_kappa.pt',\n    'batch_size': 32,\n    'n_TTA': 3,\n    'img_size': IMG_SIZE,\n    'img_scale': 1.2,\n    'use_base_transform': ['crop', 'weighted'],\n    'transform': transforms.Compose([\n        transforms.RandomHorizontalFlip(),\n        transforms.RandomVerticalFlip(),\n        transforms.RandomRotation(90),\n        # no jitter for submission\n        transforms.RandomCrop(IMG_SIZE)\n    ])\n}\nenet_predictor = BlindnessDetectionPredictor(**subm_params)\nenet_subm_preds = np.argmax(enet_predictor.predict(), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame(\n    {\n        'id_code': submit_df.id_code.values,\n        'diagnosis': enet_subm_preds\n    }\n)\nprint(submission.head())\nprint(submission.diagnosis.value_counts())\nsubmission.to_csv('submission.csv', index=False)\nprint(os.listdir('./'))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}