{"cells":[{"metadata":{},"cell_type":"markdown","source":"## EfficientNet architecture \n\nPre-trained weights from: https://www.kaggle.com/hmendonca/efficientnet-pytorch-ignite and [Ignite examples](https://github.com/pytorch/ignite/tree/master/examples):\n> Recently new ConvNets architectures have been proposed in \"[EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/pdf/1905.11946.pdf)\" paper. According to the paper, model's compound scaling starting from a 'good' baseline provides an network that achieves state-of-the-art on ImageNet, while being 8.4x smaller and 6.1x faster on inference than the best existing ConvNet.\n![efficientnets](https://raw.githubusercontent.com/pytorch/ignite/c22609796031f5831f054036895696c7e4df07ce/examples/notebooks/assets/efficientnets.png)\n\nThis kernel borrowed some of its code from: https://www.kaggle.com/kageyama/fork-of-fastai-blindness-detection-resnet34 and https://www.kaggle.com/demonplus/fast-ai-starter-with-resnet-50 Many thanks to the authors!\n\nIf you liked it, please upvote and leave questions or any feedback below (for me and other kagglers learning).\n\nCheers!"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"import time\n# For keeping time. GPU limit for this competition is set to ± 9 hours.\nt_start = time.time()\n\n\n# Ignore  the warnings\nimport warnings\nwarnings.filterwarnings('always')\nwarnings.filterwarnings('ignore')\n\n# data visualisation and manipulation\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib import style\nimport seaborn as sns\n \n#configure\n# sets matplotlib to inline and displays graphs below the corressponding cell.\n%matplotlib inline  \nstyle.use('fivethirtyeight')\nsns.set(style='whitegrid', color_codes=True)\n\nfrom sklearn.metrics import confusion_matrix\n\n# specifically for manipulating zipped images and getting numpy arrays of pixel values of images.\nimport cv2                  \nimport numpy as np  \nfrom tqdm import tqdm, tqdm_notebook\nimport os, random\nfrom random import shuffle  \nfrom zipfile import ZipFile\nfrom PIL import Image\nfrom sklearn.utils import shuffle\n\n!ls ../input/*","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import fastai\nfrom fastai import *\nfrom fastai.vision import *\nfrom fastai.callbacks import *\nfrom fastai.basic_train import *\nfrom fastai.vision.learner import *\n\nfastai.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check if the kernel is running in interactive/edit/debug mode: https://www.kaggle.com/masterscrat/detect-if-notebook-is-running-interactively\ndef is_interactive():\n   return 'runtime' in get_ipython().config.IPKernelApp.connection_file\n\nprint('Interactive?', is_interactive())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# copy pretrained weights to the folder fastai will search by default\nPath('/tmp/.cache/torch/checkpoints/').mkdir(exist_ok=True, parents=True)\nmodel_path = '/tmp/.cache/torch/checkpoints/efficientNet.pth'\n!cp ../input/efficientnet*/efficientNet_*.pth {model_path}","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"PATH = Path('../input/aptos2019-blindness-detection')\nPATH_train = Path('../input/drd-newold/drd')\n\ndf_train = pd.read_csv('../input/oldandnew/new_train_data.csv')\ndf_test = pd.read_csv(PATH/'test.csv')\n\n# if is_interactive():\n#     df_train = df_train.sample(800)\n\n_ = df_train.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SIZE=380\ndef crop_image_from_gray(img, tol=7):\n    \"\"\"\n    Applies masks to the orignal image and \n    returns the a preprocessed image with \n    3 channels\n    \"\"\"\n    # If for some reason we only have two channels\n    if img.ndim == 2:\n        mask = img > tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    # If we have a normal RGB images\n    elif img.ndim == 3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img > tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([img1,img2,img3],axis=-1)\n        return img\ndef preprocess_image(image):\n    \"\"\"\n    The whole preprocessing pipeline:\n    1. Read in image\n    2. Apply masks\n    3. Resize image to desired size\n    4. Add Gaussian noise to increase Robustness\n    \"\"\"\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = crop_image_from_gray(image)\n    image = cv2.resize(image, (IMG_WIDTH, IMG_HEIGHT))\n    image = cv2.addWeighted (image,4, cv2.GaussianBlur(image, (0,0) ,10), -4, 128)\n    return image\n\ndef open_aptos2019_image(fn, convert_mode, after_open)->Image:\n    image = cv2.imread(fn)\n    image =cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = crop_image_from_gray(image)\n    image = cv2.resize(image, (SIZE, SIZE))\n    image = cv2.addWeighted (image,4, cv2.GaussianBlur(image, (0,0) ,10), -4, 128)\n    return Image(pil2tensor(image, np.float32).div_(255))\n\nvision.data.open_image = open_aptos2019_image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create image data bunch\naptos19_stats = ([0.42, 0.22, 0.075], [0.27, 0.15, 0.081])\ndata = ImageDataBunch.from_df(df=df_train,\n                              path=PATH_train, folder='aptos_drd_jpeg', suffix='.jpeg',\n                              valid_pct=0.1,\n                              ds_tfms=get_transforms(flip_vert=True, max_warp=0.05, max_rotate=20.),\n                              #size=380,\n                              bs=4, \n                              num_workers=os.cpu_count()\n                             ).normalize(aptos19_stats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check classes\nprint(f'Classes: \\n {data.classes}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show some sample images\ndata.show_batch(rows=3, figsize=(7,6))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define model"},{"metadata":{"trusted":true},"cell_type":"code","source":"package_path = '../input/efficientnet-pytorch/efficientnet-pytorch/EfficientNet-PyTorch-master'\nsys.path.append(package_path)\n\nfrom efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# FastAI adapators to retrain our model without lossing its old head ;)\ndef EfficientNetB4(pretrained=True):\n    \"\"\"Constructs a EfficientNet model for FastAI.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = EfficientNet.from_name('efficientnet-b4', override_params={'num_classes': 5 })\n\n    if pretrained:\n        model_state = torch.load(model_path)\n        # load original weights apart from its head\n        if '_fc.weight' in model_state.keys():\n            model_state.pop('_fc.weight')\n            model_state.pop('_fc.bias')\n            res = model.load_state_dict(model_state, strict=False)\n            assert str(res.missing_keys) == str(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'\n        else:\n            # A basic remapping is required\n            from collections import OrderedDict\n            mapping = { i:o for i,o in zip(model_state.keys(), model.state_dict().keys()) }\n            mapped_model_state = OrderedDict([\n                (mapping[k], v) for k,v in model_state.items() if not mapping[k].startswith('_fc')\n            ])\n            res = model.load_state_dict(mapped_model_state, strict=False)\n            print(res)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# create model\nmodel = EfficientNetB4(pretrained=True)\n# print model structure (hidden)\n#model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss and Learner"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, gamma=3., reduction='mean'):\n        super().__init__()\n        self.gamma = gamma\n        self.reduction = reduction\n\n    def forward(self, inputs, targets):\n        CE_loss = nn.CrossEntropyLoss(reduction='none')(inputs, targets)\n        pt = torch.exp(-CE_loss)\n        F_loss = ((1 - pt)**self.gamma) * CE_loss\n        if self.reduction == 'sum':\n            return F_loss.sum()\n        elif self.reduction == 'mean':\n            return F_loss.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# from FastAI master\nfrom torch.utils.data.sampler import WeightedRandomSampler\n\nclass OverSamplingCallback(LearnerCallback):\n    def __init__(self,learn:Learner, weights:torch.Tensor=None):\n        super().__init__(learn)\n        self.labels = self.learn.data.train_dl.dataset.y.items\n        _, counts = np.unique(self.labels, return_counts=True)\n        self.weights = (weights if weights is not None else\n                        torch.DoubleTensor((1/counts)[self.labels]))\n\n    def on_train_begin(self, **kwargs):\n        self.learn.data.train_dl.dl.batch_sampler = BatchSampler(\n            WeightedRandomSampler(self.weights, len(self.learn.data.train_dl.dataset)),\n            self.learn.data.train_dl.batch_size,False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# build model (using EfficientNet)\nlearn = Learner(data, model,\n                loss_func=FocalLoss(),\n                metrics=[accuracy, KappaScore(weights=\"quadratic\")],\n                callback_fns=[BnFreeze,\n#                               OverSamplingCallback,\n#                               partial(GradientClipping, clip=0.2),\n                              partial(SaveModelCallback, monitor='kappa_score', name='best_kappa')]\n               )\nlearn.split( lambda m: (model._conv_head,) )\nlearn.freeze()\nlearn.model_dir = '/tmp/'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train head first\nlearn.freeze()\nlearn.lr_find(start_lr=1e-6, end_lr=1e1, wd=5e-3)\nlearn.recorder.plot(suggestion=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(2, max_lr=3e-2, div_factor=10, final_div=100, wd=5e-3)\nlearn.save('stage-1')\nlearn.recorder.plot_losses()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# unfreeze and search appropriate learning rate for full training\nlearn.unfreeze()\n#learn.lr_find(start_lr=slice(1e-6, 1e-5), end_lr=slice(1e-2, 1e-1), wd=1e-3)\n#learn.recorder.plot(suggestion=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train all layers\nlearn.fit_one_cycle(3, max_lr=slice(1e-4, 1e-3), div_factor=10, final_div=100, wd=1e-3)\nlearn.save('stage-2')\n#learn.recorder.plot_losses()\n# schedule of the lr (left) and momentum (right) that the 1cycle policy uses\n#learn.recorder.plot_lr(show_moms=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# _ = learn.load('best_kappa')\n\n# learn.lr_find(start_lr=slice(1e-7, 1e-6), end_lr=slice(1e-2, 1e-1), wd=1e-3)\n# learn.recorder.plot(suggestion=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train all layers\nlearn.fit_one_cycle(cyc_len=5, max_lr=slice(5e-5, 5e-4), pct_start=0, wd=1e-3) # warm restart: pct_start=0\nlearn.save('stage-3')\n#learn.recorder.plot_losses()\n# # schedule of the lr (left) and momentum (right) that the 1cycle policy uses\n#learn.recorder.plot_lr(show_moms=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# learn.load('best_kappa')\n\n# # retrain only head\n# learn.freeze()\n# learn.lr_find(start_lr=1e-7, end_lr=1e-1, wd=1e-2)\n# learn.recorder.plot(suggestion=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.fit_one_cycle(6, max_lr=1e-3, div_factor=100, wd=1e-2)\n# learn.save('stage-4')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.load('best_kappa')\n\ninterp = ClassificationInterpretation.from_learner(learn)\ninterp.plot_confusion_matrix(figsize=(8,8), dpi=60)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# interp.plot_top_losses(5, figsize=(15,11))  ## TODO: fix loss function reduction topk","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TTA and submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"# remove zoom from FastAI TTA\ntta_params = {'beta':0.12, 'scale':1.0}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')\nsample_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.data.add_test(ImageList.from_df(\n    sample_df, PATH,\n    folder='test_images',\n    suffix='.png'\n))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds,y = learn.TTA(ds_type=DatasetType.Test, **tta_params)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df.diagnosis = preds.argmax(1)\nsample_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df.to_csv('submission.csv',index=False)\n_ = sample_df.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"#move models back to root folder\n!mv {learn.model_dir}/*.pth .\nos.listdir()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check kernels run-time. GPU limit for this competition is set to ± 9 hours.\nt_finish = time.time()\ntotal_time = round((t_finish-t_start) / 3600, 4)\nprint('Kernel runtime = {} hours ({} minutes)'.format(total_time, \n                                                      int(total_time*60)))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}