{"cells":[{"metadata":{},"cell_type":"markdown","source":"Credit to Lex and his references\nhttps://www.kaggle.com/lextoumbourou/blindness-detection-resnet34-ordinal-targets"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"%matplotlib inline  \n# Ignore  the warnings\nimport warnings\nwarnings.filterwarnings('always')\nwarnings.filterwarnings('ignore')\n\n# data visualisation and manipulation\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib import style\nimport seaborn as sns\n\n# style.use('fivethirtyeight')\n# sns.set(style='whitegrid',color_codes=True)\n\nfrom sklearn.metrics import confusion_matrix\nfrom fastai import *\nfrom fastai.vision import *\nfrom fastai.callbacks import *\n\n# specifically for manipulating zipped images and getting numpy arrays of pixel values of images.\nimport cv2                  \nimport numpy as np  \nfrom tqdm import tqdm\nimport os                   \nfrom random import shuffle  \nfrom zipfile import ZipFile\nfrom PIL import Image\nfrom sklearn.utils import shuffle\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"# def seed_everything(seed):\n#     random.seed(seed)\n#     os.environ['PYTHONHASHSEED'] = str(seed)\n#     np.random.seed(seed)\n#     torch.manual_seed(seed)\n#     torch.cuda.manual_seed(seed)\n#     torch.backends.cudnn.deterministic = True\n\n# SEED = 88\n# seed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# copy pretrained weights for resnet34 to the folder fastai will search by default\nPath('/tmp/.cache/torch/checkpoints/').mkdir(exist_ok=True, parents=True)\n!cp '../input/resnet50/resnet50.pth' '/tmp/.cache/torch/checkpoints/resnet50-19c8e357.pth'","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"df_train = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ndf_test = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\n\nx_train = df_train['id_code']\ny_train = df_train['diagnosis']","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_train.diagnosis.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import torch\nimport torch.utils.data\nimport torchvision","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(os.listdir(\"../input/aptos2019-blindness-detection/\")) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def get_label(diagnosis):\n    return ','.join([str(i) for i in range(diagnosis + 1)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_train['label'] = df_train.diagnosis.apply(get_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_train.head(10)\ntorch.cuda.manual_seed_all(13)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"tfms = ([RandTransform(tfm=TfmCrop (crop_pad), kwargs={'row_pct': (0.4, 1), 'col_pct': (0.1, 0.9), 'padding_mode': 'reflection'}, p=1.0, resolved={}, do_run=True, is_random=True, use_on_y=True),\n  RandTransform(tfm=TfmPixel (rgb_randomize), kwargs={'channel':2, 'thresh':0.1}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),\n  RandTransform(tfm=TfmPixel (rgb_randomize), kwargs={'channel':1, 'thresh':0.1}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),\n  RandTransform(tfm=TfmAffine (dihedral_affine), kwargs={}, p=1.0, resolved={}, do_run=True, is_random=True, use_on_y=True),\n  RandTransform(tfm=TfmAffine (rotate), kwargs={'degrees': (-15.0, 15.0)}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),\n  RandTransform(tfm=TfmAffine (zoom), kwargs={'scale': (1.0, 1.05), 'row_pct': (0, 1), 'col_pct': (0, 1)}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),\n  RandTransform(tfm=TfmLighting (brightness), kwargs={'change': (0.4, 0.6)}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),\n  RandTransform(tfm=TfmLighting (contrast), kwargs={'scale': (0.8, 1.25)}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True)],\n [RandTransform(tfm=TfmCrop (crop_pad), kwargs={}, p=1.0, resolved={}, do_run=True, is_random=True, use_on_y=True)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# create image data bunch\ndata = ImageDataBunch.from_df('./', \n                              df=df_train, \n                              valid_pct=0.25,\n                              folder=\"../input/aptos2019-blindness-detection/train_images\",\n                              suffix=\".png\",\n                              ds_tfms=tfms,\n                              size=224,\n                              bs=156, \n                              num_workers=32,\n                             label_col='label', label_delim=',').normalize(imagenet_stats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# check classes\nprint(f'Classes: \\n {data.classes}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# show some sample images\n# data.show_batch(rows=3, figsize=(7,6))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":false},"cell_type":"code","source":"def get_preds(arr):\n    mask = arr == 0\n    return np.clip(np.where(mask.any(1), mask.argmax(1), 5) - 1, 0, 4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"last_output = torch.tensor([\n    [1.7226, 1.7226, 1.7226, 1.7226, 1.7226],\n    [0, 0, 0, 0, 1.7226],\n    [0.12841, -7.6266, -6.3899, -2.1333, -0.48995],\n    [0.68119, 1.7226, -1.9895, -0.097746, 0.53576]\n])\narr = (torch.sigmoid(last_output) > 0.5).numpy(); arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Test output\nassert (get_preds(arr) == np.array([4, 0, 0, 1])).all()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class ConfusionMatrix(Callback):\n    \"Computes the confusion matrix.\"\n\n    def on_train_begin(self, **kwargs):\n        self.n_classes = 0\n\n    def on_epoch_begin(self, **kwargs):\n        self.cm = None\n\n    def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):\n        preds = torch.tensor(get_preds((torch.sigmoid(last_output) > 0.5).cpu().numpy()))\n        \n        targs = torch.tensor(get_preds(last_target.cpu().numpy()))\n\n        if self.n_classes == 0:\n            self.n_classes = last_output.shape[-1]\n            self.x = torch.arange(0, self.n_classes)\n        \n        cm = ((preds==self.x[:, None]) & (targs==self.x[:, None, None])).sum(dim=2, dtype=torch.float32)\n        if self.cm is None: self.cm =  cm\n        else:               self.cm += cm\n\n    def on_epoch_end(self, **kwargs):\n        self.metric = self.cm\n        \n\n@dataclass\nclass KappaScore(ConfusionMatrix):\n    \"Compute the rate of agreement (Cohens Kappa).\"\n    weights:Optional[str]=None      # None, `linear`, or `quadratic`\n\n    def on_epoch_end(self, last_metrics, **kwargs):\n        sum0 = self.cm.sum(dim=0)\n        sum1 = self.cm.sum(dim=1)\n        expected = torch.einsum('i,j->ij', (sum0, sum1)) / sum0.sum()\n        if self.weights is None:\n            w = torch.ones((self.n_classes, self.n_classes))\n            w[self.x, self.x] = 0\n        elif self.weights == \"linear\" or self.weights == \"quadratic\":\n            w = torch.zeros((self.n_classes, self.n_classes))\n            w += torch.arange(self.n_classes, dtype=torch.float)\n            w = torch.abs(w - torch.t(w)) if self.weights == \"linear\" else (w - torch.t(w)) ** 2\n        else: raise ValueError('Unknown weights. Expected None, \"linear\", or \"quadratic\".')\n        k = torch.sum(w * self.cm) / torch.sum(w * expected)\n        return add_metrics(last_metrics, 1-k)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# from torch.utils.data.sampler import WeightedRandomSampler\n# class OverSamplingCallback(LearnerCallback):\n#     def __init__(self,learn:Learner):\n#         super().__init__(learn)\n#         self.labels = self.learn.data.train_dl.dataset.y.items\n#         _, counts = np.unique(self.labels,return_counts=True)\n#         self.weights = torch.DoubleTensor((1/counts)[self.labels])\n#         self.label_counts = np.bincount([self.learn.data.train_dl.dataset.y[i].data \\\n#                                          for i in range(len(self.learn.data.train_dl.dataset))])\n#         self.total_len_oversample = int(self.learn.data.c*np.max(self.label_counts))\n        \n#     def on_train_begin(self, **kwargs):\n#         self.learn.data.train_dl.dl.batch_sampler = \\\n#         BatchSampler(WeightedRandomSampler(self.weights,self.total_len_oversample), self.learn.data.train_dl.batch_size,False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import math\nimport torch\nfrom torch.optim.optimizer import Optimizer, required\nimport itertools as it\n#from torch.optim import Optimizer\n#credit - Lookahead implementation from LonePatient - https://github.com/lonePatient/lookahead_pytorch/blob/master/optimizer.py\n#credit2 - RAdam code by https://github.com/LiyuanLucasLiu/RAdam/blob/master/radam.py\n\n\nclass Ranger(Optimizer):\n    \n    def __init__(self, params, lr=1e-2, alpha=0.5, k=8, betas=(.9,0.999), eps=1e-8, weight_decay=0.1):\n        #parameter checks\n        if not 0.0 <= alpha <= 1.0:\n            raise ValueError(f'Invalid slow update rate: {alpha}')\n        if not 1 <= k:\n            raise ValueError(f'Invalid lookahead steps: {k}')\n        if not lr > 0:\n            raise ValueError(f'Invalid Learning Rate: {lr}')\n        if not eps > 0:\n            raise ValueError(f'Invalid eps: {eps}')\n        \n        #prep defaults and init torch.optim base\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n        super().__init__(params,defaults)\n        \n        #now we can get to work...\n        for group in self.param_groups:\n            group[\"step_counter\"] = 0\n            #print(\"group step counter init\")\n                      \n        #look ahead params\n        self.alpha = alpha\n        self.k = k \n        \n        #radam buffer for state\n        self.radam_buffer = [[None,None,None] for ind in range(10)]\n        \n        #lookahead weights\n        self.slow_weights = [[p.clone().detach() for p in group['params']]\n                                for group in self.param_groups]\n        \n        #don't use grad for lookahead weights\n        for w in it.chain(*self.slow_weights):\n            w.requires_grad = False\n        \n    def __setstate__(self, state):\n        print(\"set state called\")\n        super(Ranger, self).__setstate__(state)\n       \n        \n    def step(self, closure=None):\n        loss = None\n        #note - below is commented out b/c I have other work that passes back the loss as a float, and thus not a callable closure.  \n        #Uncomment if you need to use the actual closure...\n        \n        #if closure is not None:\n            #loss = closure()\n            \n        #------------ radam\n        for group in self.param_groups:\n    \n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('RAdam does not support sparse gradients')\n    \n                p_data_fp32 = p.data.float()\n    \n                state = self.state[p]\n    \n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n    \n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n    \n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n    \n                state['step'] += 1\n                buffered = self.radam_buffer[int(state['step'] % 10)]\n                if state['step'] == buffered[0]:\n                    N_sma, step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n                    if N_sma > 5:\n                        step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    else:\n                        step_size = group['lr'] / (1 - beta1 ** state['step'])\n                    buffered[2] = step_size\n    \n                if group['weight_decay'] != 0:\n                    p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n    \n                if N_sma > 5:                    \n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    p_data_fp32.addcdiv_(-step_size, exp_avg, denom)\n                else:\n                    p_data_fp32.add_(-step_size, exp_avg)\n    \n                p.data.copy_(p_data_fp32)\n        \n        \n        #---------------- end radam step\n        \n        #look ahead tracking and updating if latest batch = k\n        for group,slow_weights in zip(self.param_groups,self.slow_weights):\n            group['step_counter'] += 1\n            if group['step_counter'] % self.k != 0:\n                continue\n            for p,q in zip(group['params'],slow_weights):\n                if p.grad is None:\n                    continue\n                q.data.add_(self.alpha,p.data - q.data)\n                p.data.copy_(q.data)\n            \n        \n            \n        return loss\n\noptar = partial(Ranger)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"kappa = KappaScore(weights=\"quadratic\")\n\n# build model (use resnet34)\nlearn = cnn_learner(data, models.resnet50, metrics=[kappa, accuracy_thresh], \n                   opt_func = optar,\n                   callback_fns = [\n                                partial(EarlyStoppingCallback, monitor='kappa_score', min_delta=0.001, patience=3),\n                                partial(ReduceLROnPlateauCallback),\n#                               partial(GradientClipping, clip=0.2),\n                                partial(SaveModelCallback, every = 'improvement', monitor='kappa_score', name='bestordinal')],\n                   model_dir=\"/tmp\").to_fp16().mixup(stack_y=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot()\nlrs = learn.recorder.lrs\nlosses = learn.recorder.losses\nmg = (np.gradient(np.array(losses))).argmin()\nml = np.argmin(losses[1:])\nmin_grad_lr = lrs[mg]\nprint(min_grad_lr)\nmin_loss_lr = lrs[ml]/10\nprint(min_loss_lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#first time learning\nlr = min_loss_lr\nlearn.fit_one_cycle(10, lr)\ntorch.cuda.manual_seed_all(18)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# data = ImageDataBunch.from_df('./', \n#                               df=df_train, \n#                               valid_pct=0.2,\n#                               folder=\"../input/aptos2019-blindness-detection/train_images\",\n#                               suffix=\".png\",\n#                               ds_tfms=tfms,\n#                               size=224,\n#                               bs=128, \n#                               num_workers=32,\n#                              label_col='label', label_delim=',').normalize(imagenet_stats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# learn.data = data\n# learn.to_fp16()\nlearn.unfreeze()\nlearn.lr_find\nlrs = learn.recorder.lrs\nlosses = learn.recorder.losses\nmg = (np.gradient(np.array(losses))).argmin()\nml = np.argmin(losses[1:])\nmin_grad_lr = lrs[mg]\nprint(min_grad_lr)\nmin_loss_lr = lrs[ml]/10\nprint(min_loss_lr)\nlr2 = min_loss_lr\nlearn.unfreeze()\nlearn.fit_one_cycle(10, max_lr = lr2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# #Adding Weight decay for regulization\n# learn.fit_one_cycle(5,wd=1e-1)\n# learn.save('stage-1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# learn.unfreeze()\n# learn.lr_find()\n# learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.load('bestordinal')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.freeze()\nlearn.fit_one_cycle(15, max_lr=lr2/50,wd=1e-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.load('bestordinal')\nsample_df = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\nsample_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.data.add_test(ImageList.from_df(sample_df,'../input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"preds, y = learn.get_preds(DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"sample_df.diagnosis = get_preds((preds > 0.5).cpu().numpy())\nsample_df.diagnosis.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"sample_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"sample_df.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"!mv {learn.model_dir}/*.pth .\nos.listdir()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}