{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in ok\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n #   for filename in filenames:\n  #      print(os.path.join(dirname, filename))\n#for dirname, _, filenames in os.walk('/kaggle/input/effnet'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\nbase_dir = \"../input/aptos2019-blindness-detection/\"\n\ntrain_csv = base_dir+\"train.csv\"\ntest_csv = base_dir+\"test.csv\"\ntest_dir = base_dir+\"test_images/\"\n\n#test_dir_processed = base_dir+'test_dir_processed'\n#train_dir = base_dir+\"train_data_cropped\"\n\ntest_dir_processed = 'test_dir_processed'\ntrain_dir = \"train_data_cropped\"\n\nIMG_SIZE = 224\n\nSEED = 72\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#pip install -U git+http://github.com/qubvel/efficientnet\n#!pip install git+https://github.com/qubvel/efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://github.com/qubvel/efficientnet#installation","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import sys\n# Repository source: https://github.com/qubvel/efficientnet\nsys.path.append(os.path.abspath('../input/efficientnet/efficientnet-master/efficientnet-master/'))\nfrom efficientnet import EfficientNetB5\n#from efficientnet import EfficientNetB4\n#from efficientnet import EfficientNetB3\n#from efficientnet import EfficientNetB2\n#from efficientnet import EfficientNetB1\n#from efficientnet import EfficientNetB0\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2 \nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import class_weight, shuffle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image = cv2.imread('../input/aptos2019-blindness-detection/train_images/295fdc964f6e.png')\nplt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\n# commenting for kernel run else there will be error\n#shutil.rmtree('train_data_cropped')\n#shutil.rmtree('test_dir_processed')\nos.mkdir('train_data_cropped')\nos.mkdir('test_dir_processed')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        \n        gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img         # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Converting images to grayscale , gaussian blur and then cropping"},{"metadata":{"trusted":true},"cell_type":"code","source":"image_path = '../input/aptos2019-blindness-detection/train_images/'\n\nfor fileName in os.listdir(image_path):\n    \n    #Ignore the file which are not png\n    # some file with .DS_Store will be there\n    # created by jupyter and caused issue as not images\n    if fileName.endswith('png'):\n        #image = cv2.imread('train_data'+'/'+fileName)\n        image = cv2.imread(image_path+fileName)\n        \n        #convert into gray images\n        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n        \n        #Crop image so that there is less black around retina image\n        image = crop_image_from_gray(image)\n        \n        # resize image ,default started with 512\n        image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n        \n        #This line of code enhance image \n        #image=cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , sigmaX) ,-4 ,128)\n        #Please refer to about Gaussian\n        #https://www.tutorialkart.com/opencv/python/opencv-python-gaussian-image-smoothing/ .\n        image=cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , 10) ,-4 ,128)\n        \n        # save image on disk\n        cv2.imwrite('train_data_cropped/'+fileName,image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_path = '../input/aptos2019-blindness-detection/test_images/'\n\nfor fileName in os.listdir(image_path):\n    \n    #Ignore the file which are not png\n    # some file with .DS_Store will be there\n    # created by jupyter and caused issue as not images\n    if fileName.endswith('png'):\n        #image = cv2.imread('train_data'+'/'+fileName)\n        image = cv2.imread(image_path+fileName)\n        \n        #convert into gray images\n        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n        \n        #Crop image so that there is less black around retina image\n        image = crop_image_from_gray(image)\n        \n        # resize image ,default started with 512\n        image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n        \n        #This line of code enhance image \n        #image=cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , sigmaX) ,-4 ,128)\n        #Please refer to about Gaussian\n        #https://www.tutorialkart.com/opencv/python/opencv-python-gaussian-image-smoothing/ .\n        image=cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , 10) ,-4 ,128)\n        \n        # save image on disk\n        cv2.imwrite('test_dir_processed/'+fileName,image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let's view processed images\ni = 0\nfor fileName in os.listdir(\"train_data_cropped/\"):\n    i = i + 1\n\nprint(i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let's view processed images\ni = 0\nfor fileName in os.listdir(\"test_dir_processed/\"):\n    i = i + 1\n\nprint(i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image1 = cv2.imread('../input/aptos2019-blindness-detection/train_images/295fdc964f6e.png')\nimage1 = cv2.cvtColor(image1, cv2.COLOR_BGR2RGB)\nplt.imshow(image1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n#295fdc964f6e.png  c8905b8d5cf1.png\nfig = plt.figure(figsize=(25, 16))\nax = fig.add_subplot(5, 5, 5, xticks=[], yticks=[])\nimage2 = cv2.imread('train_data_cropped/295fdc964f6e.png')\nimage2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)\nplt.imshow(image2,cmap='gray')\nimage2.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# constants for \nWORKERS = 2\nCHANNEL = 3\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n\nNUM_CLASSES = 5\nSEED = 72\nTRAIN_NUM = 1000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def df_train_test_split_preprocess(df):\n    \n    image_ids = df[\"id_code\"].values.tolist()\n    labels = df[\"diagnosis\"].values.tolist()\n    \n    for i in range(len(image_ids)):\n        imgname = image_ids[i]\n        newname = str(imgname) + \".png\"\n        image_ids[i] = newname\n    \n    xtrain, xval, ytrain, yval = train_test_split(image_ids, labels, test_size = 0.15)\n    \n    df_train = pd.DataFrame({\"id_code\":xtrain, \"diagnosis\":ytrain})\n    df_val = pd.DataFrame({\"id_code\":xval, \"diagnosis\":yval})\n    \n    df_train[\"diagnosis\"] = df_train[\"diagnosis\"].astype('str')\n    df_val[\"diagnosis\"] = df_val[\"diagnosis\"].astype('str')\n    \n    print(\"Length of Training Data :\",len(df_train))\n    print(\"Length of Validation Data :\",len(df_val))\n    \n    return df_train, df_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(train_csv)\ndf_train, df_val = df_train_test_split_preprocess(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score, fbeta_score, cohen_kappa_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"ImageDataGenerator (Training data)"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\n#   --- TO DO ----\n#   zca_epsilon: epsilon for ZCA whitening. Default is 1e-6.\n#   zca_whitening: Boolean. Apply ZCA whitening.\n#\ntrain_aug = ImageDataGenerator(rescale=1./255,\n                               horizontal_flip = True,\n                               zoom_range = 0.15,\n                               vertical_flip = True,\n                               shear_range=0.1,\n                               rotation_range = 90\n                               )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator = train_aug.flow_from_dataframe(dataframe = df_train,\n                                               directory = train_dir,\n                                               x_col = \"id_code\",\n                                               y_col = \"diagnosis\",\n                                               batch_size = 16,\n                                               target_size =  (IMG_SIZE, IMG_SIZE),\n                                               #color_mode = 'grayscale',\n                                               class_mode = \"categorical\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"ImageDataGenerator ( Validation data )"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Using same as for training\nvalidation_generator = train_aug.flow_from_dataframe(dataframe = df_val,\n                                                    directory = train_dir,\n                                                    x_col = \"id_code\",\n                                                    y_col = \"diagnosis\",\n                                                    batch_size = 16, \n                                                    target_size = (IMG_SIZE, IMG_SIZE),\n                                                    #color_mode = 'grayscale',\n                                                    class_mode = \"categorical\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Kappa Cohen using Keras\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\nimport tensorflow as tf\nfrom keras import backend as K\nfrom sklearn.metrics import cohen_kappa_score\n# build model...(not shown)\n\n# custom metric with TF\ndef cohens_kappa(y_true, y_pred):\n    y_true_classes = tf.argmax(y_true, 1)\n    y_pred_classes = tf.argmax(y_pred, 1)\n    ck_val = tf.contrib.metrics.cohen_kappa(y_true_classes, y_pred_classes, 5)[1]\n    print(ck_val)\n    return ck_val\n\n# custom metric with TF\n#def quad_cohens_kappa(y_true, y_pred):\n#    y_true_classes = tf.argmax(y_true, 1)\n#    y_pred_classes = tf.argmax(y_pred, 1)\n#    print(y_true_classes)\n    #ck_val = cohen_kappa_score(y_true_classes, y_pred_classes, weights='quadratic')\n#    ck_val = 0\n#    print(ck_val)\n#    return ck_val\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"from keras.callbacks import Callback, ModelCheckpoint\nclass Metrics(Callback):\n    def on_train_begin(self, logs={}):\n        self.val_kappas = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        print(self.validation_data)\n        X_val, y_val = self.validation_data[:2]\n        y_val = y_val.sum(axis=1) - 1\n        \n        y_pred = self.model.predict(X_val) > 0.5\n        y_pred = y_pred.astype(int).sum(axis=1) - 1\n\n        _val_kappa = cohen_kappa_score(\n            y_val,\n            y_pred, \n            weights='quadratic'\n        )\n\n        self.val_kappas.append(_val_kappa)\n\n        print(f\"val_kappa: {_val_kappa:.4f}\")\n        \n        if _val_kappa == max(self.val_kappas):\n            print(\"Validation Kappa has improved. Saving model.\")\n            self.model.save('model.h5')\n\n        return"},{"metadata":{},"cell_type":"markdown","source":"Test data "},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df_orig = pd.read_csv(test_csv)\n\ndef process_test_df(test_df):\n    test_ids = test_df[\"id_code\"].values.tolist()\n    for i in range(len(test_ids)):\n        imgname = test_ids[i]\n        newname = str(imgname) + \".png\"\n        test_ids[i] = newname\n    test_df[\"id_code\"] = test_ids\n    return test_df\n\ntest_df = process_test_df(test_df_orig)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Test Data augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# No need to augment only rescale pixel values\ntest_aug = ImageDataGenerator(rescale = 1./255 )\n\ntest_generator = test_aug.flow_from_dataframe(dataframe = test_df, \n                                              directory = test_dir_processed,\n                                              x_col = \"id_code\",\n                                              batch_size = 1,\n                                              target_size =  (IMG_SIZE, IMG_SIZE), # to be changed as ???\n                                              shuffle = False,\n                                              class_mode = None)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**RADAM Implementation**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Code Source: https://github.com/CyberZHG/keras-radam/blob/master/keras_radam/optimizers.py\nclass RAdam(keras.optimizers.Optimizer):\n    \"\"\"RAdam optimizer.\n    # Arguments\n        lr: float >= 0. Learning rate.\n        beta_1: float, 0 < beta < 1. Generally close to 1.\n        beta_2: float, 0 < beta < 1. Generally close to 1.\n        epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`.\n        decay: float >= 0. Learning rate decay over each update.\n        weight_decay: float >= 0. Weight decay for each param.\n        amsgrad: boolean. Whether to apply the AMSGrad variant of this\n            algorithm from the paper \"On the Convergence of Adam and\n            Beyond\".\n        total_steps: int >= 0. Total number of training steps. Enable warmup by setting a positive value.\n        warmup_proportion: 0 < warmup_proportion < 1. The proportion of increasing steps.\n        min_lr: float >= 0. Minimum learning rate after warmup.\n    # References\n        - [Adam - A Method for Stochastic Optimization](https://arxiv.org/abs/1412.6980v8)\n        - [On the Convergence of Adam and Beyond](https://openreview.net/forum?id=ryQu7f-RZ)\n        - [On The Variance Of The Adaptive Learning Rate And Beyond](https://arxiv.org/pdf/1908.03265v1.pdf)\n    \"\"\"\n\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., weight_decay=0., amsgrad=False,\n                 total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):\n        super(RAdam, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n            self.weight_decay = K.variable(weight_decay, name='weight_decay')\n            self.total_steps = K.variable(total_steps, name='total_steps')\n            self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')\n            self.min_lr = K.variable(lr, name='min_lr')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.initial_weight_decay = weight_decay\n        self.initial_total_steps = total_steps\n        self.amsgrad = amsgrad\n\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n\n        lr = self.lr\n\n        if self.initial_decay > 0:\n            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay))))\n\n        t = K.cast(self.iterations, K.floatx()) + 1\n\n        if self.initial_total_steps > 0:\n            warmup_steps = self.total_steps * self.warmup_proportion\n            decay_steps = self.total_steps - warmup_steps\n            lr = K.switch(\n                t <= warmup_steps,\n                lr * (t / warmup_steps),\n                lr * (1.0 - K.minimum(t, decay_steps) / decay_steps),\n            )\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for (i, p) in enumerate(params)]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for (i, p) in enumerate(params)]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for (i, p) in enumerate(params)]\n        else:\n            vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params))]\n\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        beta_1_t = K.pow(self.beta_1, t)\n        beta_2_t = K.pow(self.beta_2, t)\n\n        sma_inf = 2.0 / (1.0 - self.beta_2) - 1.0\n        sma_t = sma_inf - 2.0 * t * beta_2_t / (1.0 - beta_2_t)\n\n        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n\n            m_corr_t = m_t / (1.0 - beta_1_t)\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                v_corr_t = K.sqrt(vhat_t / (1.0 - beta_2_t) + self.epsilon)\n                self.updates.append(K.update(vhat, vhat_t))\n            else:\n                v_corr_t = K.sqrt(v_t / (1.0 - beta_2_t) + self.epsilon)\n\n            r_t = K.sqrt((sma_t - 4.0) / (sma_inf - 4.0) *\n                         (sma_t - 2.0) / (sma_inf - 2.0) *\n                         sma_inf / sma_t)\n\n            p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)\n\n            if self.initial_weight_decay > 0:\n                p_t += self.weight_decay * p\n\n            p_t = p - lr * p_t\n\n            self.updates.append(K.update(m, m_t))\n            self.updates.append(K.update(v, v_t))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {\n            'lr': float(K.get_value(self.lr)),\n            'beta_1': float(K.get_value(self.beta_1)),\n            'beta_2': float(K.get_value(self.beta_2)),\n            'decay': float(K.get_value(self.decay)),\n            'weight_decay': float(K.get_value(self.weight_decay)),\n            'epsilon': self.epsilon,\n            'amsgrad': self.amsgrad,\n            'total_steps': float(K.get_value(self.total_steps)),\n            'warmup_proportion': float(K.get_value(self.warmup_proportion)),\n            'min_lr': float(K.get_value(self.min_lr)),\n        }\n        base_config = super(RAdam, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications import ResNet50\nfrom keras.models import Model\nfrom keras.layers import Dense, GlobalAveragePooling2D, Dropout, Input\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom sklearn.model_selection import train_test_split\nimport keras\nfrom keras.engine import Layer,InputSpec\n#from keras.applications import DenseNet121,DenseNet169,DenseNet201","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GroupNormalization(Layer):\n    \"\"\"Group normalization layer\n    Group Normalization divides the channels into groups and computes within each group\n    the mean and variance for normalization. GN's computation is independent of batch sizes,\n    and its accuracy is stable in a wide range of batch sizes\n    # Arguments\n        groups: Integer, the number of groups for Group Normalization.\n        axis: Integer, the axis that should be normalized\n            (typically the features axis).\n            For instance, after a `Conv2D` layer with\n            `data_format=\"channels_first\"`,\n            set `axis=1` in `BatchNormalization`.\n        epsilon: Small float added to variance to avoid dividing by zero.\n        center: If True, add offset of `beta` to normalized tensor.\n            If False, `beta` is ignored.\n        scale: If True, multiply by `gamma`.\n            If False, `gamma` is not used.\n            When the next layer is linear (also e.g. `nn.relu`),\n            this can be disabled since the scaling\n            will be done by the next layer.\n        beta_initializer: Initializer for the beta weight.\n        gamma_initializer: Initializer for the gamma weight.\n        beta_regularizer: Optional regularizer for the beta weight.\n        gamma_regularizer: Optional regularizer for the gamma weight.\n        beta_constraint: Optional constraint for the beta weight.\n        gamma_constraint: Optional constraint for the gamma weight.\n    # Input shape\n        Arbitrary. Use the keyword argument `input_shape`\n        (tuple of integers, does not include the samples axis)\n        when using this layer as the first layer in a model.\n    # Output shape\n        Same shape as input.\n    # References\n        - [Group Normalization](https://arxiv.org/abs/1803.08494)\n    \"\"\"\n\n    def __init__(self,\n                 groups=32,\n                 axis=-1,\n                 epsilon=1e-5,\n                 center=True,\n                 scale=True,\n                 beta_initializer='zeros',\n                 gamma_initializer='ones',\n                 beta_regularizer=None,\n                 gamma_regularizer=None,\n                 beta_constraint=None,\n                 gamma_constraint=None,\n                 **kwargs):\n        super(GroupNormalization, self).__init__(**kwargs)\n        self.supports_masking = True\n        self.groups = groups\n        self.axis = axis\n        self.epsilon = epsilon\n        self.center = center\n        self.scale = scale\n        self.beta_initializer = initializers.get(beta_initializer)\n        self.gamma_initializer = initializers.get(gamma_initializer)\n        self.beta_regularizer = regularizers.get(beta_regularizer)\n        self.gamma_regularizer = regularizers.get(gamma_regularizer)\n        self.beta_constraint = constraints.get(beta_constraint)\n        self.gamma_constraint = constraints.get(gamma_constraint)\n\n    def build(self, input_shape):\n        dim = input_shape[self.axis]\n\n        if dim is None:\n            raise ValueError('Axis ' + str(self.axis) + ' of '\n                             'input tensor should have a defined dimension '\n                             'but the layer received an input with shape ' +\n                             str(input_shape) + '.')\n\n        if dim < self.groups:\n            raise ValueError('Number of groups (' + str(self.groups) + ') cannot be '\n                             'more than the number of channels (' +\n                             str(dim) + ').')\n\n        if dim % self.groups != 0:\n            raise ValueError('Number of groups (' + str(self.groups) + ') must be a '\n                             'multiple of the number of channels (' +\n                             str(dim) + ').')\n\n        self.input_spec = InputSpec(ndim=len(input_shape),\n                                    axes={self.axis: dim})\n        shape = (dim,)\n\n        if self.scale:\n            self.gamma = self.add_weight(shape=shape,\n                                         name='gamma',\n                                         initializer=self.gamma_initializer,\n                                         regularizer=self.gamma_regularizer,\n                                         constraint=self.gamma_constraint)\n        else:\n            self.gamma = None\n        if self.center:\n            self.beta = self.add_weight(shape=shape,\n                                        name='beta',\n                                        initializer=self.beta_initializer,\n                                        regularizer=self.beta_regularizer,\n                                        constraint=self.beta_constraint)\n        else:\n            self.beta = None\n        self.built = True\n\n    def call(self, inputs, **kwargs):\n        input_shape = K.int_shape(inputs)\n        tensor_input_shape = K.shape(inputs)\n\n        # Prepare broadcasting shape.\n        reduction_axes = list(range(len(input_shape)))\n        del reduction_axes[self.axis]\n        broadcast_shape = [1] * len(input_shape)\n        broadcast_shape[self.axis] = input_shape[self.axis] // self.groups\n        broadcast_shape.insert(1, self.groups)\n\n        reshape_group_shape = K.shape(inputs)\n        group_axes = [reshape_group_shape[i] for i in range(len(input_shape))]\n        group_axes[self.axis] = input_shape[self.axis] // self.groups\n        group_axes.insert(1, self.groups)\n\n        # reshape inputs to new group shape\n        group_shape = [group_axes[0], self.groups] + group_axes[2:]\n        group_shape = K.stack(group_shape)\n        inputs = K.reshape(inputs, group_shape)\n\n        group_reduction_axes = list(range(len(group_axes)))\n        group_reduction_axes = group_reduction_axes[2:]\n\n        mean = K.mean(inputs, axis=group_reduction_axes, keepdims=True)\n        variance = K.var(inputs, axis=group_reduction_axes, keepdims=True)\n\n        inputs = (inputs - mean) / (K.sqrt(variance + self.epsilon))\n\n        # prepare broadcast shape\n        inputs = K.reshape(inputs, group_shape)\n        outputs = inputs\n\n        # In this case we must explicitly broadcast all parameters.\n        if self.scale:\n            broadcast_gamma = K.reshape(self.gamma, broadcast_shape)\n            outputs = outputs * broadcast_gamma\n\n        if self.center:\n            broadcast_beta = K.reshape(self.beta, broadcast_shape)\n            outputs = outputs + broadcast_beta\n\n        outputs = K.reshape(outputs, tensor_input_shape)\n\n        return outputs\n\n    def get_config(self):\n        config = {\n            'groups': self.groups,\n            'axis': self.axis,\n            'epsilon': self.epsilon,\n            'center': self.center,\n            'scale': self.scale,\n            'beta_initializer': initializers.serialize(self.beta_initializer),\n            'gamma_initializer': initializers.serialize(self.gamma_initializer),\n            'beta_regularizer': regularizers.serialize(self.beta_regularizer),\n            'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),\n            'beta_constraint': constraints.serialize(self.beta_constraint),\n            'gamma_constraint': constraints.serialize(self.gamma_constraint)\n        }\n        base_config = super(GroupNormalization, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n\n    def compute_output_shape(self, input_shape):\n        return input_shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(IMG_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_layer = Input(shape = (IMG_SIZE,IMG_SIZE,3))\n\nbase_model = EfficientNetB5(weights = None,\n                       include_top = False,\n                       input_tensor = input_layer)\n\nbase_model.load_weights('../input/effnet/efficientnetb5notop.h5')\n# all are false\n#for layer in base_model.layers:\n#    layer.trainable = False\n# top 5 are fasle    \n#for layer in base_model.layers[:180]:\n#    layer.trainable = False\n#--- v1 with 90 , it was CH = .7435 & ACC = .89 start with .43   \n#--- v2 with 71 , it was CH = .7435 & ACC = .89 start with .43   \n\n# last 4 are false\n#for layer in vgg_conv.layers[:-4]:\n#    layer.trainable = False\n    \nx = GlobalAveragePooling2D()(base_model.output)\nx = Dense(1024, activation='relu')(x)\nx = Dropout(0.40)(x)\nx = Dense(512, activation='relu')(x)\nx = Dropout(0.3)(x)\nout = Dense(5, activation = 'softmax')(x)\n\nmodel = Model(inputs = input_layer, outputs = out)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Image size \n* efficientnet-b0-224\n* efficientnet-b1-240\n* efficientnet-b2-260\n* efficientnet-b3-300\n* efficientnet-b4-380\n* efficientnet-b5-456\n* efficientnet-b6-528\n* efficientnet-b7-600"},{"metadata":{"trusted":true},"cell_type":"code","source":"#model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for layer in model.layers:\n#    print(layer.name,layer.trainable)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#optimizer = keras.optimizers.Adam(lr=2e-4)\n#optimizer = keras.optimizers.Adam(lr=0.0005)\n\noptimizer = RAdam(lr=0.0005)\n#optimizer = RAdam(lr=2e-4)\n#es = EarlyStopping(monitor='val_loss', mode='min', patience = 9, restore_best_weights=True)\n#rlrop = ReduceLROnPlateau(monitor='val_loss', mode='min', patience = 3, factor = 0.5, min_lr=1e-6)\n\nes = EarlyStopping(monitor='cohens_kappa', mode='auto', verbose=1, patience=3,restore_best_weights=True)\nrlrop = ReduceLROnPlateau(monitor='cohens_kappa', \n                        factor=0.2, \n                        patience=5, \n                        verbose=1, \n                        mode='auto', \n                        min_lr=1e-6)\n#kappa_metrics = Metrics()\ncallback_list = [ rlrop ]\n\nmodel.compile(optimizer = optimizer, loss = \"categorical_crossentropy\", metrics = [\"accuracy\",cohens_kappa]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"K.get_session().run(tf.local_variables_initializer())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#import gc\n#gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(generator = train_generator, \n                    steps_per_epoch = len(train_generator), \n                    epochs = 18, \n                    validation_data = validation_generator, \n                    validation_steps = len(validation_generator),\n                    callbacks =  callback_list  )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"acc = history.history['acc']\nval_acc = history.history['val_acc']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\ncohens_kappa = history.history['cohens_kappa']\nval_cohens_kappa = history.history['val_cohens_kappa']\n\nepochs = range(len(acc))\n \nplt.plot(epochs, acc, 'b', label='Training acc')\nplt.plot(epochs, val_acc, 'r', label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\n \nplt.figure()\n \nplt.plot(epochs, loss, 'b', label='Training loss')\nplt.plot(epochs, val_loss, 'r', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.figure()\n \nplt.plot(epochs, cohens_kappa, 'b', label='Training Cohen-kappa')\nplt.plot(epochs, val_cohens_kappa, 'r', label='Validation Cohen-kappa')\nplt.title('Cohen Kappa - Training and validation score')\nplt.legend()\n\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**PREDICTIO ON TEST**"},{"metadata":{"trusted":true},"cell_type":"code","source":"predprobs = model.predict_generator(test_generator, steps=len(test_generator))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**CLEANING of files generated during pre-processing**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Cleaning all processed file\n# Else you will get \"TOO MANY FILES\" Error\nshutil.rmtree('train_data_cropped')\nshutil.rmtree('test_dir_processed')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**select prediction of highest probability**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# select prediction of highest probability\npredictions = []\nfor i in predprobs:\n    predictions.append(np.argmax(i))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create dataframe for submitting result Need to take care that submit file should not have PNG in id_codes column"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df_orig.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create new column and assign prediction class\ntest_df_orig[\"diagnosis\"] = predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df_orig.head(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ids = test_df_orig[\"id_code\"].values.tolist()\nfor i in range(len(test_ids)):\n    imgname = test_ids[i]\n    newname = imgname.split('.')[0]\n    test_ids[i] = newname\n    test_df_orig[\"id_code\"] = test_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df_orig.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**SUBMIT FILE CREATION**"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df_orig.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subfile = pd.read_csv('submission.csv')\nsubfile.head(3)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}