{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Blindness Detection: DenseNet201\n\n* Img size: 320x320\n* Batch size: 64\n* Data: concat 2019 + 2015 training sets. Downsample class 0 to match class 2. Each epoch change sample of 0 class.\n* Validation: 2015 test set with class 0 downsampled to match class 2.\n* Preprocess: Preprocessing copied from [this](https://www.kaggle.com/joorarkesteijn/fast-cropping-preprocessing-and-augmentation) kernel which used ideas from [this](https://www.kaggle.com/ratthachat/aptos-updated-preprocessing-ben-s-cropping) kernel. Also used GaussianBlur subtraction from Ben's preprocessing.\n* Model head: multiclass (ordinal regression) outputs.\n* Loss: BCEWithLogitsLoss with modified label smoothing: convert `[1, 1, 0, 0, 0]` labels into `[0.95, 0.95, 0.05, 0.05, 0.05]`\n* Opt: Adam (fast.ai default)\n* Pseudo-labelling: add all test labels from submission.csv with 0.834 LB.\n* Augmentations: flip_lr, brightness, contrast, rotate(360)\n* Train: train just head for one epoch, train 15 epochs using [one cycle](https://arxiv.org/pdf/1803.09820)."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport time\n# Ignore  the warnings\nimport warnings\nwarnings.filterwarnings('always')\nwarnings.filterwarnings('ignore')\n\n# data visualisation and manipulation\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib import style\nimport seaborn as sns\n\nstyle.use('fivethirtyeight')\nsns.set(style='whitegrid',color_codes=True)\n\nfrom sklearn.metrics import confusion_matrix, cohen_kappa_score\nfrom fastai import *\nfrom fastai.vision import *\nfrom fastai.callbacks import Callback\nfrom sklearn.metrics import classification_report\nfrom fastai.data_block import MultiCategoryList\nfrom fastai.callbacks import SaveModelCallback, EarlyStoppingCallback, ReduceLROnPlateauCallback\n\n# specifically for manipulating zipped images and getting numpy arrays of pixel values of images.\nimport cv2                  \nimport numpy as np  \nfrom tqdm import tqdm\nimport os                   \nfrom random import shuffle  \nfrom zipfile import ZipFile\nfrom sklearn.utils import shuffle\n\nprint(os.listdir(\"../input\"))\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# copy pretrained weights for resnet34 to the folder fastai will search by default\nPath('/tmp/.cache/torch/checkpoints/').mkdir(exist_ok=True, parents=True)\n!cp '../input/densenet201/densenet201.pth' '/tmp/.cache/torch/checkpoints/densenet201-c1103571.pth'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hyperparams"},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL_NAME = 'densenet201_change_zeros_ord_reg_label_smoothing'\n\nSIZE = 320\nIMG_SIZE = 320\n\nBS = 64\n\nSEED = 424\n\nSIGMA_X = 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"max_zoom = 1.5\np_affine = 0.75\nmax_lighting = 0.2\np_lighting = 0.75\nscale = 2.\nmax_rotate = 60\n\ntrain_tfms, val_tfms = [\n    flip_lr(),\n    # zoom(scale=(1., max_zoom), p=p_affine),\n    brightness(change=(0.5 * (1-max_lighting), 0.5 * (1 + max_lighting)), p=p_lighting),\n    contrast(scale=(1-max_lighting, 1/(1-max_lighting)), p=p_lighting),\n    # jitter(magnitude=0.003, p=0.4),\n    rotate(degrees=(-max_rotate, max_rotate), p=p_affine)\n], []","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data downsampling"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_label(diagnosis):\n        return ','.join([str(i) for i in range(diagnosis + 1)])\n\ndef get_train_df(seed, num_zeros=4000):\n    val_preds_id = pd.read_csv('../input/bd-peter-and-lex-validation-set/val.csv')['id_code']\n\n    df_train = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n    df_test = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\n\n    df_train['is_valid'] = False\n    # df_train.loc[df_train.id_code.isin(val_preds_id), 'is_valid'] = True\n    df_train.id_code = '../input/aptos2019-blindness-detection/train_images/' + df_train.id_code + '.png'\n\n    df_train.columns = ['image_path', 'diagnosis', 'is_valid']\n\n    extra_training_df = pd.read_csv('../input/diabetic-retinopathy-resized/trainLabels.csv')\n    extra_training_df['is_valid'] = False\n    # extra_training_df.loc[extra_training_df.image.isin(val_preds_id), 'is_valid'] = True\n    extra_training_df.image = '../input/diabetic-retinopathy-resized/resized_train/resized_train/' + extra_training_df.image + '.jpeg'\n    extra_training_df.columns = ['image_path', 'diagnosis', 'is_valid']\n    \n    pseudo = pd.read_csv('../input/bd-best-model-blend-v1-densenet101/submission.csv')\n    pseudo.id_code = '../input/aptos2019-blindness-detection/test_images/' + pseudo.id_code + '.png'\n    pseudo['is_valid'] = False\n    pseudo.columns = ['image_path', 'diagnosis', 'is_valid']\n    \n    test_labels_15_df = pd.read_csv('../input/resized-2015-2019-blindness-detection-images/labels/testLabels15.csv')\n    del test_labels_15_df['Usage']\n    test_labels_15_df.columns = ['image_id', 'diagnosis']\n    test_labels_15_df['dataset_id'] = 'test_labels_15'\n    test_labels_15_df['image_path'] = '../input/resized-2015-2019-blindness-detection-images/resized test 15/' + test_labels_15_df.image_id + '.jpg'\n    test_labels_15_df['is_valid'] = True\n    test_labels_15_df = test_labels_15_df[['image_path', 'diagnosis', 'is_valid']]\n\n    df_train = pd.concat([\n        df_train,\n        extra_training_df[(extra_training_df.diagnosis == 0) & (extra_training_df.is_valid)],\n        extra_training_df[(extra_training_df.diagnosis == 0) & ~(extra_training_df.is_valid)].sample(n=num_zeros, random_state=seed),\n        extra_training_df[extra_training_df.diagnosis == 1],\n        extra_training_df[extra_training_df.diagnosis == 2],\n        extra_training_df[extra_training_df.diagnosis == 3],\n        extra_training_df[extra_training_df.diagnosis == 4],\n        pseudo,\n        pd.concat([\n            test_labels_15_df[test_labels_15_df.diagnosis == 0].sample(n=7900, random_state=420),\n            test_labels_15_df[test_labels_15_df.diagnosis != 0]\n        ]).sample(n=10_000, random_state=420),\n    ]).sample(frac=1, random_state=seed)\n\n    df_train['label'] = df_train.diagnosis.apply(get_label)\n    \n    return df_train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image preprocessing"},{"metadata":{},"cell_type":"markdown","source":"From: https://www.kaggle.com/joorarkesteijn/fast-cropping-preprocessing-and-augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# To remove irregularities along the circular boundary of the image\nPARAM = 96\n\ndef Radius_Reduction(img,PARAM):\n    h,w,c=img.shape\n    Frame=np.zeros((h,w,c),dtype=np.uint8)\n    cv2.circle(Frame,(int(math.floor(w/2)),int(math.floor(h/2))),int(math.floor((h*PARAM)/float(2*100))), (255,255,255), -1)\n    Frame1=cv2.cvtColor(Frame, cv2.COLOR_BGR2GRAY)\n    img1 =cv2.bitwise_and(img,img,mask=Frame1)\n    return img1\n\ndef info_image(im):\n    # Compute the center (cx, cy) and radius of the eye\n    cy = im.shape[0]//2\n    midline = im[cy,:]\n    midline = np.where(midline>midline.mean()/3)[0]\n    if len(midline)>im.shape[1]//2:\n        x_start, x_end = np.min(midline), np.max(midline)\n    else: # This actually rarely happens p~1/10000\n        x_start, x_end = im.shape[1]//10, 9*im.shape[1]//10\n    cx = (x_start + x_end)/2\n    r = (x_end - x_start)/2\n    return cx, cy, r\n\n\ndef resize_image(im, img_size, augmentation=False):\n    # Crops, resizes and potentially augments the image to IMG_SIZE\n    cx, cy, r = info_image(im)\n    scaling = img_size/(2*r)\n    rotation = 0\n    if augmentation:\n        scaling *= 1 + 0.3 * (np.random.rand()-0.5)\n        rotation = 360 * np.random.rand()\n    M = cv2.getRotationMatrix2D((cx,cy), rotation, scaling)\n    M[0,2] -= cx - img_size/2\n    M[1,2] -= cy - img_size/2\n    return cv2.warpAffine(im, M, (img_size, img_size)) # This is the most important line\n\n\ndef subtract_median_bg_image(im):\n    k = np.max(im.shape)//20*2+1\n    bg = cv2.medianBlur(im, k)\n    return cv2.addWeighted (im, 4, bg, -4, 128)\n\n\ndef subtract_gaussian_bg_image(im):\n    # k = np.max(im.shape)/10\n    bg = cv2.GaussianBlur(im ,(0,0) , SIGMA_X)\n    return cv2.addWeighted (im, 4, bg, -4, 128)\n\n\ndef open_img(self, fn, size):\n    \"Open image in `fn`, subclass and overwrite for custom behavior.\"\n    image = cv2.imread(fn)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = resize_image(image, size)\n\n    # changing line here.\n    image = subtract_gaussian_bg_image(image)\n    image = Radius_Reduction(image, PARAM)\n    return Image(pil2tensor(image, np.float32).div_(255))\n\ndef get_data(seed, size=IMG_SIZE, bs=BS):\n    df_train = get_train_df(seed)\n    \n    ImageList.open = lambda self, fn: open_img(self, fn, size=size)\n\n    data = (\n        ImageList.from_df(\n            path='./',\n            df=df_train,\n            folder='.'\n        )\n    )\n    data = (data.split_from_df('is_valid')\n            .label_from_df('label', label_delim=',', label_cls=ReconstructFixMultiCategoryList)\n            .transform(\n                (train_tfms, val_tfms),\n                resize_method=ResizeMethod.NO,\n                padding_mode='zeros')\n            .databunch(bs=bs)\n            .normalize(imagenet_stats))\n    data.add_test(ImageList.from_df(sample_df, '../input/aptos2019-blindness-detection', folder='test_images', suffix='.png'))\n    return data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Fixes bug with `show_batch` with multi label."},{"metadata":{"trusted":true},"cell_type":"code","source":"class ReconstructFixMultiCategoryList(MultiCategoryList):\n    def reconstruct(self, t):\n        try:\n            return super().reconstruct(t)\n        except Exception as e:\n            return FloatItem(np.log(t))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Metrics and callbacks"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_preds(arr):\n    mask = arr == 0\n    return np.clip(np.where(mask.any(1), mask.argmax(1), 5) - 1, 0, 4)\n\nclass ConfusionMatrix(Callback):\n    \"Computes the confusion matrix.\"\n\n    def on_train_begin(self, **kwargs):\n        self.n_classes = 0\n\n    def on_epoch_begin(self, **kwargs):\n        self.cm = None\n\n    def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):\n        preds = torch.tensor(get_preds((torch.sigmoid(last_output) > 0.5).cpu().numpy()))\n        \n        targs = torch.tensor(get_preds(last_target.cpu().numpy()))\n\n        if self.n_classes == 0:\n            self.n_classes = last_output.shape[-1]\n            self.x = torch.arange(0, self.n_classes)\n        \n        cm = ((preds==self.x[:, None]) & (targs==self.x[:, None, None])).sum(dim=2, dtype=torch.float32)\n        if self.cm is None: self.cm =  cm\n        else:               self.cm += cm\n\n    def on_epoch_end(self, **kwargs):\n        self.metric = self.cm\n        \n\n@dataclass\nclass KappaScore(ConfusionMatrix):\n    \"Compute the rate of agreement (Cohens Kappa).\"\n    weights:Optional[str]=None      # None, `linear`, or `quadratic`\n\n    def on_epoch_end(self, last_metrics, **kwargs):\n        sum0 = self.cm.sum(dim=0)\n        sum1 = self.cm.sum(dim=1)\n        expected = torch.einsum('i,j->ij', (sum0, sum1)) / sum0.sum()\n        if self.weights is None:\n            w = torch.ones((self.n_classes, self.n_classes))\n            w[self.x, self.x] = 0\n        elif self.weights == \"linear\" or self.weights == \"quadratic\":\n            w = torch.zeros((self.n_classes, self.n_classes))\n            w += torch.arange(self.n_classes, dtype=torch.float)\n            w = torch.abs(w - torch.t(w)) if self.weights == \"linear\" else (w - torch.t(w)) ** 2\n        else: raise ValueError('Unknown weights. Expected None, \"linear\", or \"quadratic\".')\n        k = torch.sum(w * self.cm) / torch.sum(w * expected)\n        return add_metrics(last_metrics, 1-k)\n    \n@dataclass\nclass ChangeDataOnEpoch(Callback):\n    learn:Learner\n    i:int\n    size:int\n    bs:int\n        \n    def on_epoch_end(self, **kwargs):\n        print(f'Data seed {self.i}, size: {self.size}, bs: {self.bs}')\n        self.learn.data = get_data(seed=self.i, size=self.size, bs=self.bs)\n        self.learn.data.add_tfm(batch_to_half)\n        self.i += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss function"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FlattenedLoss():\n    \"Same as `func`, but flattens input and target.\"\n    def __init__(self, func, *args, axis:int=-1, floatify:bool=False, is_2d:bool=True, **kwargs):\n        self.func,self.axis,self.floatify,self.is_2d = func(*args,**kwargs),axis,floatify,is_2d\n        functools.update_wrapper(self, self.func)\n\n    def __repr__(self): return f\"FlattenedLoss of {self.func}\"\n    @property\n    def reduction(self): return self.func.reduction\n    @reduction.setter\n    def reduction(self, v): self.func.reduction = v\n\n    def __call__(self, input:Tensor, target:Tensor, **kwargs)->Rank0Tensor:\n        input = input.transpose(self.axis,-1).contiguous()\n        target = target.transpose(self.axis,-1).contiguous()\n        if self.floatify: target = target.float()\n            \n        # Label smoothing experiment\n        target = (target * 0.9 + 0.05)\n        target[:,0] = 1\n\n        input = input.view(-1,input.shape[-1]) if self.is_2d else input.view(-1)\n        return self.func.__call__(input, target.view(-1), **kwargs)\n\n    \ndef LabelSmoothBCEWithLogitsFlat(*args, axis:int=-1, floatify:bool=True, **kwargs):\n    \"Same as `nn.BCEWithLogitsLoss`, but flattens input and target.\"\n    return FlattenedLoss(nn.BCEWithLogitsLoss, *args, axis=axis, floatify=floatify, is_2d=False, **kwargs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get data"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = get_data(seed=SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show some sample images\ndata.show_batch(figsize=(20, 16))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.show_batch(figsize=(20, 16), ds_type=DatasetType.Valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.show_batch(figsize=(20, 16), ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kappa = KappaScore(weights=\"quadratic\")\n\ndata = get_data(seed=0, size=IMG_SIZE, bs=BS)\nlearn = cnn_learner(data, models.densenet201, metrics=[kappa, accuracy_thresh], model_dir=\".\", lin_ftrs=[2048], callback_fns=[BnFreeze])\nlearn.loss_func = LabelSmoothBCEWithLogitsFlat()\nlearn = learn.to_fp16()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(1, 1e-02)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot_losses()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.lr_find()\n# learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(\n    20,\n    max_lr=slice(5e-5, 5e-4),\n    callbacks=[\n        SaveModelCallback(learn, monitor='kappa_score', mode='max', name='best_model'),\n        ChangeDataOnEpoch(learn=learn, i=SEED, size=IMG_SIZE, bs=BS)\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot_losses()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.load('best_model');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.validate()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.data.bs = 32","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"duration = time.time() - start_time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Trained one fold in {duration} seconds')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_items = learn.data.valid_dl.dataset.items ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_preds, val_y = learn.get_preds(ds_type=DatasetType.Valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_preds.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_preds_df = pd.concat([\n    pd.DataFrame({'id_code': [\n        v.split('/')[-1].split('.')[0] for v in val_items\n    ], 'diagnosis': val_y.argmax(1).numpy(), 'preds': get_preds((val_preds > 0.5).numpy())}),\n    pd.DataFrame(val_preds.numpy())\n], axis=1); val_preds_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_preds_df.to_csv(f'{MODEL_NAME}_val_preds.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"metric = cohen_kappa_score(val_preds_df['diagnosis'], val_preds_df['preds'], weights='quadratic')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Val kappa score: {metric}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_names = ['0', '1', '2', '3', '4']\nprint(classification_report(val_preds_df['diagnosis'], val_preds_df['preds'], target_names=target_names))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds, y = learn.get_preds(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"duration = time.time() - start_time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Made test predictions in {duration} seconds')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df.diagnosis = get_preds((preds > 0.5).cpu().numpy())\nsample_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds_df = pd.concat([\n    sample_df,\n    pd.DataFrame(preds.numpy())\n], axis=1)\ntest_preds_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds_df.to_csv('test_preds.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}