{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Blindness Detection: EfficientNet B3\n\n* Img size: 256x256\n* Batch size: 64\n* Data: concat 2019 + 2015 training sets. Downsample class 0 to match class 2. Each epoch change sample of 0 class.\n* Validation: 2015 test set with class 0 downsampled to match class 2.\n* Preprocess: Preprocessing copied from [this](https://www.kaggle.com/joorarkesteijn/fast-cropping-preprocessing-and-augmentation) kernel which used ideas from [this](https://www.kaggle.com/ratthachat/aptos-updated-preprocessing-ben-s-cropping) kernel.\n* Model head: multiclass (ordinal regression) outputs.\n* Loss: BCEWithLogitsLoss with modified label smoothing: convert `[1, 1, 0, 0, 0]` labels into `[0.95, 0.95, 0.05, 0.05, 0.05]`\n* Opt: Adam (fast.ai default)\n* Pseudo-labelling: add all test labels from submission.csv with 0.834 LB.\n* Augmentations: flip_lr, brightness, contrast, rotate(360)\n* Train: train just head for one epoch, train 15 epochs using [one cycle](https://arxiv.org/pdf/1803.09820)."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\n\nimport time\nimport warnings\nimport os                   \nfrom random import shuffle  \nfrom zipfile import ZipFile\n\nwarnings.filterwarnings('always')\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nfrom matplotlib import style\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix, cohen_kappa_score\nfrom fastai import *\nfrom fastai.vision import *\nfrom fastai.callbacks import Callback\nfrom fastai.callbacks import SaveModelCallback, EarlyStoppingCallback, ReduceLROnPlateauCallback\nfrom fastai.data_block import MultiCategoryList\nimport cv2 \nfrom tqdm import tqdm\nfrom sklearn.utils import shuffle\n\nsys.path.append('../input/efficientnet-pytorch/efficientnet-pytorch/EfficientNet-PyTorch-master')\nimport efficientnet_pytorch\n\nstyle.use('fivethirtyeight')\nsns.set(style='whitegrid',color_codes=True)\n\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hyperparams"},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL_NAME = 'efficientnetb03_change_zeros_ord_reg_label_smoothing'\n\nIMG_SIZE = 256\nBS = 64\nSEED = 425","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"max_zoom = 1.5\np_affine = 0.75\nmax_lighting = 0.2\np_lighting = 0.75\nscale = 2.\nmax_rotate = 360\n\ntrain_tfms, val_tfms = [\n    flip_lr(),\n    brightness(change=(0.5 * (1-max_lighting), 0.5 * (1 + max_lighting)), p=p_lighting),\n    contrast(scale=(1-max_lighting, 1/(1-max_lighting)), p=p_lighting),\n    rotate(degrees=(-max_rotate, max_rotate), p=p_affine)\n], []","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data downsampling"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_label(diagnosis):\n        return ','.join([str(i) for i in range(diagnosis + 1)])\n\n\ndef get_train_df(seed, num_zeros=4000):\n    val_preds_id = pd.read_csv('../input/bd-peter-and-lex-validation-set/val.csv')['id_code']\n\n    df_train = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n    df_test = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\n\n    df_train['is_valid'] = False\n    # df_train.loc[df_train.id_code.isin(val_preds_id), 'is_valid'] = True\n    df_train.id_code = '../input/aptos2019-blindness-detection/train_images/' + df_train.id_code + '.png'\n\n    df_train.columns = ['image_path', 'diagnosis', 'is_valid']\n\n    extra_training_df = pd.read_csv('../input/diabetic-retinopathy-resized/trainLabels.csv')\n    extra_training_df['is_valid'] = False\n    # extra_training_df.loc[extra_training_df.image.isin(val_preds_id), 'is_valid'] = True\n    extra_training_df.image = '../input/diabetic-retinopathy-resized/resized_train/resized_train/' + extra_training_df.image + '.jpeg'\n    extra_training_df.columns = ['image_path', 'diagnosis', 'is_valid']\n    \n    pseudo = pd.read_csv('../input/linearstackingblendedwithbestlbsub/linear-stacking-blended-with-best-lb-as-reg-v2-submission.csv')\n    pseudo.id_code = '../input/aptos2019-blindness-detection/test_images/' + pseudo.id_code + '.png'\n    pseudo['is_valid'] = False\n    pseudo.columns = ['image_path', 'diagnosis', 'is_valid']\n    \n    test_labels_15_df = pd.read_csv('../input/resized-2015-2019-blindness-detection-images/labels/testLabels15.csv')\n    del test_labels_15_df['Usage']\n    test_labels_15_df.columns = ['image_id', 'diagnosis']\n    test_labels_15_df['dataset_id'] = 'test_labels_15'\n    test_labels_15_df['image_path'] = '../input/resized-2015-2019-blindness-detection-images/resized test 15/' + test_labels_15_df.image_id + '.jpg'\n    test_labels_15_df['is_valid'] = True\n    test_labels_15_df = test_labels_15_df[['image_path', 'diagnosis', 'is_valid']]\n\n    df_train = pd.concat([\n        df_train,\n        extra_training_df[(extra_training_df.diagnosis == 0) & (extra_training_df.is_valid)],\n        extra_training_df[(extra_training_df.diagnosis == 0) & ~(extra_training_df.is_valid)].sample(n=num_zeros, random_state=seed),\n        extra_training_df[extra_training_df.diagnosis == 1],\n        extra_training_df[extra_training_df.diagnosis == 2],\n        extra_training_df[extra_training_df.diagnosis == 3],\n        extra_training_df[extra_training_df.diagnosis == 4],\n        pseudo,\n        pd.concat([\n            test_labels_15_df[test_labels_15_df.diagnosis == 0].sample(n=7900, random_state=420),\n            test_labels_15_df[test_labels_15_df.diagnosis != 0]\n        ]).sample(n=10_000, random_state=420),\n    ]).sample(frac=1, random_state=seed)\n\n    df_train['label'] = df_train.diagnosis.apply(get_label)\n    \n    return df_train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Fixes bug with `show_batch` with multi label labels."},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass ReconstructFixMultiCategoryList(MultiCategoryList):\n    def reconstruct(self, t):\n        try:\n            return super().reconstruct(t)\n        except Exception as e:\n            return FloatItem(np.log(t))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image preprocessing\n\nAll image preprocessing this kernel taken from [this]() wonderful kernel."},{"metadata":{"trusted":true},"cell_type":"code","source":"# To remove irregularities along the circular boundary of the image\nPARAM = 96\n\ndef Radius_Reduction(img,PARAM):\n    h,w,c=img.shape\n    Frame=np.zeros((h,w,c),dtype=np.uint8)\n    cv2.circle(Frame,(int(math.floor(w/2)),int(math.floor(h/2))),int(math.floor((h*PARAM)/float(2*100))), (255,255,255), -1)\n    Frame1=cv2.cvtColor(Frame, cv2.COLOR_BGR2GRAY)\n    img1 =cv2.bitwise_and(img,img,mask=Frame1)\n    return img1\n\n\ndef info_image(im):\n    # Compute the center (cx, cy) and radius of the eye\n    cy = im.shape[0]//2\n    midline = im[cy,:]\n    midline = np.where(midline>midline.mean()/3)[0]\n    if len(midline)>im.shape[1]//2:\n        x_start, x_end = np.min(midline), np.max(midline)\n    else: # This actually rarely happens p~1/10000\n        x_start, x_end = im.shape[1]//10, 9*im.shape[1]//10\n    cx = (x_start + x_end)/2\n    r = (x_end - x_start)/2\n    return cx, cy, r\n\n\ndef resize_image(im, img_size, augmentation=False):\n    # Crops, resizes and potentially augments the image to IMG_SIZE\n    cx, cy, r = info_image(im)\n    scaling = img_size/(2*r)\n    rotation = 0\n    if augmentation:\n        scaling *= 1 + 0.3 * (np.random.rand()-0.5)\n        rotation = 360 * np.random.rand()\n    M = cv2.getRotationMatrix2D((cx,cy), rotation, scaling)\n    M[0,2] -= cx - img_size/2\n    M[1,2] -= cy - img_size/2\n    return cv2.warpAffine(im, M, (img_size, img_size)) # This is the most important line\n\n\ndef open_img(self, fn, size):\n    \"Open image in `fn`, subclass and overwrite for custom behavior.\"\n    image = cv2.imread(fn)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = resize_image(image, size)\n    # image = subtract_median_bg_image(image)\n    image = Radius_Reduction(image, PARAM)\n    return Image(pil2tensor(image, np.float32).div_(255))\n    \n\nImageList.open = lambda self, fn: open_img(self, fn, size=IMG_SIZE)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Metrics and callbacks"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_preds(arr):\n    mask = arr == 0\n    return np.clip(np.where(mask.any(1), mask.argmax(1), 5) - 1, 0, 4)\n\nclass ConfusionMatrix(Callback):\n    \"Computes the confusion matrix.\"\n\n    def on_train_begin(self, **kwargs):\n        self.n_classes = 0\n\n    def on_epoch_begin(self, **kwargs):\n        self.cm = None\n\n    def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):\n        preds = torch.tensor(get_preds((torch.sigmoid(last_output) > 0.5).cpu().numpy()))\n        \n        targs = torch.tensor(get_preds(last_target.cpu().numpy()))\n\n        if self.n_classes == 0:\n            self.n_classes = last_output.shape[-1]\n            self.x = torch.arange(0, self.n_classes)\n        \n        cm = ((preds==self.x[:, None]) & (targs==self.x[:, None, None])).sum(dim=2, dtype=torch.float32)\n        if self.cm is None: self.cm =  cm\n        else:               self.cm += cm\n\n    def on_epoch_end(self, **kwargs):\n        self.metric = self.cm\n        \n\n@dataclass\nclass KappaScore(ConfusionMatrix):\n    \"Compute the rate of agreement (Cohens Kappa).\"\n    weights:Optional[str]=None      # None, `linear`, or `quadratic`\n\n    def on_epoch_end(self, last_metrics, **kwargs):\n        sum0 = self.cm.sum(dim=0)\n        sum1 = self.cm.sum(dim=1)\n        expected = torch.einsum('i,j->ij', (sum0, sum1)) / sum0.sum()\n        if self.weights is None:\n            w = torch.ones((self.n_classes, self.n_classes))\n            w[self.x, self.x] = 0\n        elif self.weights == \"linear\" or self.weights == \"quadratic\":\n            w = torch.zeros((self.n_classes, self.n_classes))\n            w += torch.arange(self.n_classes, dtype=torch.float)\n            w = torch.abs(w - torch.t(w)) if self.weights == \"linear\" else (w - torch.t(w)) ** 2\n        else: raise ValueError('Unknown weights. Expected None, \"linear\", or \"quadratic\".')\n        k = torch.sum(w * self.cm) / torch.sum(w * expected)\n        return add_metrics(last_metrics, 1-k)\n\n\n@dataclass\nclass ChangeDataOnEpoch(Callback):\n    learn:Learner\n    i:int\n        \n    def on_epoch_end(self, **kwargs):\n        print(f'Data seed {self.i}')\n        self.learn.data = get_data(seed=self.i)\n        self.learn.data.add_tfm(batch_to_half)\n        self.i += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss function"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FlattenedLoss():\n    \"Same as `func`, but flattens input and target.\"\n    def __init__(self, func, *args, axis:int=-1, floatify:bool=False, is_2d:bool=True, **kwargs):\n        self.func,self.axis,self.floatify,self.is_2d = func(*args,**kwargs),axis,floatify,is_2d\n        functools.update_wrapper(self, self.func)\n\n    def __repr__(self): return f\"FlattenedLoss of {self.func}\"\n    @property\n    def reduction(self): return self.func.reduction\n    @reduction.setter\n    def reduction(self, v): self.func.reduction = v\n\n    def __call__(self, input:Tensor, target:Tensor, **kwargs)->Rank0Tensor:\n        input = input.transpose(self.axis,-1).contiguous()\n        target = target.transpose(self.axis,-1).contiguous()\n        if self.floatify: target = target.float()\n            \n        # Label smoothing experiment\n        target = (target * 0.9 + 0.05)\n        target[:,0] = 1\n\n        input = input.view(-1,input.shape[-1]) if self.is_2d else input.view(-1)\n        return self.func.__call__(input, target.view(-1), **kwargs)\n\n    \ndef LabelSmoothBCEWithLogitsFlat(*args, axis:int=-1, floatify:bool=True, **kwargs):\n    \"Same as `nn.BCEWithLogitsLoss`, but flattens input and target.\"\n    return FlattenedLoss(nn.BCEWithLogitsLoss, *args, axis=axis, floatify=floatify, is_2d=False, **kwargs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get data"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data(seed, size=IMG_SIZE):\n    df_train = get_train_df(seed)\n    data = (\n        ImageList.from_df(\n            path='./',\n            df=df_train,\n            folder='.'\n        )\n    )\n    data = (data.split_from_df('is_valid')\n            .label_from_df('label', label_delim=',', label_cls=ReconstructFixMultiCategoryList)\n            .transform(\n                (train_tfms, val_tfms),\n                resize_method=ResizeMethod.NO,\n                padding_mode='zeros')\n            .databunch(bs=BS)\n            .normalize(imagenet_stats))\n    data.add_test(ImageList.from_df(sample_df, '../input/aptos2019-blindness-detection', folder='test_images', suffix='.png'))\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = get_data(seed=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.show_batch(figsize=(20, 16))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.show_batch(figsize=(20, 16), ds_type=DatasetType.Valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.show_batch(figsize=(20, 16), ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_efficientnet(name, pretrained, model_path):\n    \"\"\"Constructs a EfficientNetB5 model for FastAI.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = efficientnet_pytorch.EfficientNet.from_name(f'efficientnet-{name}', override_params={'num_classes': 5})\n    if pretrained:\n        model_state = torch.load(model_path)\n        # load original weights apart from its head\n        if '_fc.weight' in model_state.keys():\n            model_state.pop('_fc.weight')\n            model_state.pop('_fc.bias')\n            res = model.load_state_dict(model_state, strict=False)\n            print('Loaded pretrained')\n            assert str(res.missing_keys) == str(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'\n        else:\n            # A basic remapping is required\n            from collections import OrderedDict\n            mapping = { i:o for i,o in zip(model_state.keys(), model.state_dict().keys()) }\n            mapped_model_state = OrderedDict([\n                (mapping[k], v) for k,v in model_state.items() if not mapping[k].startswith('_fc')\n            ])\n            res = model.load_state_dict(mapped_model_state, strict=False)\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kappa = KappaScore(weights=\"quadratic\")\nchange_data_cb = partial(ChangeDataOnEpoch, i=SEED)\n\nmodel = get_efficientnet('b3', True, '../input/efficientnet-pytorch/efficientnet-b3-c8376fa2.pth')\n\nlearn = Learner(data, model, metrics=[kappa, accuracy_thresh], model_dir=\".\", callback_fns=[change_data_cb, BnFreeze])\nlearn.loss_func = LabelSmoothBCEWithLogitsFlat()\nlearn.split(lambda m: (m._conv_head,) );\nlearn = learn.to_fp16()\nlearn.freeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.lr_find()\n# learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(1, 1e-2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot_losses()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.lr_find()\n# learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(15, 1e-3, callbacks=[SaveModelCallback(learn, name='best_model')])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot_losses()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.load('best_model');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.validate()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"duration = time.time() - start_time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Trained one fold in {duration} seconds')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_items = learn.data.valid_dl.dataset.items ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_preds, val_y = learn.get_preds(ds_type=DatasetType.Valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_preds.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_preds_df = pd.concat([\n    pd.DataFrame({'id_code': [\n        v.split('/')[-1].split('.')[0] for v in val_items\n    ], 'diagnosis': val_y.argmax(1).numpy(), 'preds': get_preds((val_preds > 0.5).numpy())}),\n    pd.DataFrame(val_preds.numpy())\n], axis=1); val_preds_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_preds_df.to_csv(f'{MODEL_NAME}_val_preds.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"metric = cohen_kappa_score(val_preds_df['diagnosis'], val_preds_df['preds'], weights='quadratic')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Val kappa score: {metric}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds, y = learn.get_preds(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"duration = time.time() - start_time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Made test predictions in {duration} seconds')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df.diagnosis = get_preds((preds > 0.5).cpu().numpy())\nsample_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds_df = pd.concat([\n    sample_df,\n    pd.DataFrame(preds.numpy())\n], axis=1)\ntest_preds_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds_df.to_csv('test_preds.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}