{"cells":[{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"import os\nimport sys\nsys.path.append(os.path.abspath('../input/efficientnet/efficientnet-master/efficientnet-master/'))\nfrom efficientnet import EfficientNetB5\n\nimport cv2\nimport time\nimport scipy as sp\nimport numpy as np\nimport random as rn\nimport pandas as pd\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom functools import partial\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport keras\nfrom keras import initializers\nfrom keras import regularizers\nfrom keras import constraints\nfrom keras import backend as K\nfrom keras.activations import elu\nfrom keras.optimizers import Adam\nfrom keras.models import Sequential\nfrom keras.engine import Layer, InputSpec\nfrom keras.utils.generic_utils import get_custom_objects\nfrom keras.callbacks import Callback, EarlyStopping, ReduceLROnPlateau\nfrom keras.layers import Dense, Conv2D, Flatten, GlobalAveragePooling2D, Dropout\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.metrics import cohen_kappa_score\n\nKAGGLE_DIR = '../input/aptos2019-blindness-detection/'\nTRAIN_DF_PATH = KAGGLE_DIR + \"train.csv\"\nTEST_DF_PATH = KAGGLE_DIR + 'test.csv'\nTRAIN_IMG_PATH = KAGGLE_DIR + \"train_images/\"\nTEST_IMG_PATH = KAGGLE_DIR + 'test_images/'\n\nSAVED_MODEL_NAME = 'effnet_modelB5.h5'\n\nseed = 1234\nrn.seed(seed)\nnp.random.seed(seed)\ntf.set_random_seed(seed)\nos.environ['PYTHONHASHSEED'] = str(seed)\nt_start = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"tf.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('\\n# Files and file sizes')\nfor file in os.listdir(KAGGLE_DIR):\n    print('{}| {} MB'.format(file.ljust(30), \n                             str(round(os.path.getsize(KAGGLE_DIR + file) / 1000000, 2))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Image IDs and Labels (TRAIN)\")\ntrain_df = pd.read_csv(TRAIN_DF_PATH)\n\ntrain_df['id_code'] = train_df['id_code'] + \".png\"\nprint(f\"Training images: {train_df.shape[0]}\")\ndisplay(train_df.head())\nprint(\"Image IDs (TEST)\")\ntest_df = pd.read_csv(TEST_DF_PATH)\n\ntest_df['id_code'] = test_df['id_code'] + \".png\"\nprint(f\"Testing Images: {test_df.shape[0]}\")\ndisplay(test_df.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_WIDTH = 456\nIMG_HEIGHT = 456\nCHANNELS = 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_preds_and_labels(model, generator):   \n    preds = []\n    labels = []\n    for _ in range(int(np.ceil(generator.samples / BATCH_SIZE))):\n        x, y = next(generator)\n        preds.append(model.predict(x))\n        labels.append(y)   \n    return np.concatenate(preds).ravel(), np.concatenate(labels).ravel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class Metrics(Callback):\n    \"\"\"\n    A custom Keras callback for saving the best model\n    according to the Quadratic Weighted Kappa (QWK) metric\n    \"\"\"\n    def on_train_begin(self, logs={}):\n        \"\"\"\n        Initialize list of QWK scores on validation data\n        \"\"\"\n        self.val_kappas = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        \"\"\"\n        Gets QWK score on the validation data\n        \n        :param epoch: The current epoch number\n        \"\"\"\n        # Get predictions and convert to integers\n        y_pred, labels = get_preds_and_labels(model, val_generator)\n        y_pred = np.rint(y_pred).astype(np.uint8).clip(0, 4)\n        # We can use sklearns implementation of QWK straight out of the box\n        # as long as we specify weights as 'quadratic'\n        _val_kappa = cohen_kappa_score(labels, y_pred, weights='quadratic')\n        self.val_kappas.append(_val_kappa)\n        print(f\"val_kappa: {round(_val_kappa, 4)}\")\n        if _val_kappa == max(self.val_kappas):\n            print(\"Validation Kappa has improved. Saving model.\")\n            self.model.save(SAVED_MODEL_NAME)\n        return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['diagnosis'].value_counts().sort_index().plot(kind=\"bar\", \n                                                       figsize=(12,5), \n                                                       rot=0)\nplt.title(\"Label Distribution (Training Set)\", \n          weight='bold', \n          fontsize=18)\nplt.xticks(fontsize=15)\nplt.yticks(fontsize=15)\nplt.xlabel(\"Label\", fontsize=17)\nplt.ylabel(\"Frequency\", fontsize=17);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Приклад для кожного ступеню захворювання(label)\n\nfig, ax = plt.subplots(1, 5, figsize=(15, 6))\nfor i in range(5):\n    sample = train_df[train_df['diagnosis'] == i].sample(1)\n    image_name = sample['id_code'].item()\n    X = cv2.imread(f\"{TRAIN_IMG_PATH}{image_name}\")\n    ax[i].set_title(f\"Image: {image_name}\\n Label = {sample['diagnosis'].item()}\", \n                    weight='bold', fontsize=10)\n    ax[i].axis('off')\n    RGB_img = cv2.cvtColor(X, cv2.COLOR_BGR2RGB)\n    ax[i].imshow(RGB_img);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"def crop_image_from_gray(img, tol=7):\n    \"\"\"\n    Applies masks to the orignal image and \n    returns the a preprocessed image with \n    3 channels\n    \n    :param img: A NumPy Array that will be cropped\n    :param tol: The tolerance used for masking\n    \n    :return: A NumPy array containing the cropped image\n    \"\"\"\n    # If for some reason we only have two channels\n    if img.ndim == 2:\n        mask = img > tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    # If we have a normal RGB images\n    elif img.ndim == 3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img > tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([img1,img2,img3],axis=-1)\n        return img\n\ndef preprocess_image(image, sigmaX=10):\n    \"\"\"\n    The whole preprocessing pipeline:\n    1. Read in image\n    2. Apply masks\n    3. Resize image to desired size\n    4. Add Gaussian noise to increase Robustness\n    \n    :param img: A NumPy Array that will be cropped\n    :param sigmaX: Value used for add GaussianBlur to the image\n    \n    :return: A NumPy array containing the preprocessed image\n    \"\"\"\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = crop_image_from_gray(image)\n    image = cv2.resize(image, (IMG_WIDTH, IMG_HEIGHT))\n    image = cv2.addWeighted (image,4, cv2.GaussianBlur(image, (0,0) ,sigmaX), -4, 128)\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 5, figsize=(15, 6))\nfor i in range(5):\n    sample = train_df[train_df['diagnosis'] == i].sample(1)\n    image_name = sample['id_code'].item()\n    X = preprocess_image(cv2.imread(f\"{TRAIN_IMG_PATH}{image_name}\"))\n    ax[i].set_title(f\"Image: {image_name}\\n Label = {sample['diagnosis'].item()}\", \n                    weight='bold', fontsize=10)\n    ax[i].axis('off')\n    ax[i].imshow(X);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_labels = train_df['diagnosis'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 4\n\ntrain_datagen = ImageDataGenerator(rotation_range=360,\n                                   horizontal_flip=True,\n                                   vertical_flip=True,\n                                   validation_split=0.15,\n                                   preprocessing_function=preprocess_image, \n                                   rescale=1 / 128.)\n\ntrain_generator = train_datagen.flow_from_dataframe(train_df, \n                                                    x_col='id_code', \n                                                    y_col='diagnosis',\n                                                    directory = TRAIN_IMG_PATH,\n                                                    target_size=(IMG_WIDTH, IMG_HEIGHT),\n                                                    batch_size=BATCH_SIZE,\n                                                    class_mode='other', \n                                                    subset='training')\n\nval_generator = train_datagen.flow_from_dataframe(train_df, \n                                                  x_col='id_code', \n                                                  y_col='diagnosis',\n                                                  directory = TRAIN_IMG_PATH,\n                                                  target_size=(IMG_WIDTH, IMG_HEIGHT),\n                                                  batch_size=BATCH_SIZE,\n                                                  class_mode='other',\n                                                  subset='validation')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"#https://github.com/CyberZHG/keras-radam/blob/master/keras_radam/optimizers.py\nclass RAdam(keras.optimizers.Optimizer):\n    \"\"\"RAdam optimizer.\n    # Arguments\n        lr: float >= 0. Learning rate.\n        beta_1: float, 0 < beta < 1. Generally close to 1.\n        beta_2: float, 0 < beta < 1. Generally close to 1.\n        epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`.\n        decay: float >= 0. Learning rate decay over each update.\n        weight_decay: float >= 0. Weight decay for each param.\n        amsgrad: boolean. Whether to apply the AMSGrad variant of this\n            algorithm from the paper \"On the Convergence of Adam and\n            Beyond\".\n        total_steps: int >= 0. Total number of training steps. Enable warmup by setting a positive value.\n        warmup_proportion: 0 < warmup_proportion < 1. The proportion of increasing steps.\n        min_lr: float >= 0. Minimum learning rate after warmup.\n    # References\n        - [Adam - A Method for Stochastic Optimization](https://arxiv.org/abs/1412.6980v8)\n        - [On the Convergence of Adam and Beyond](https://openreview.net/forum?id=ryQu7f-RZ)\n        - [On The Variance Of The Adaptive Learning Rate And Beyond](https://arxiv.org/pdf/1908.03265v1.pdf)\n    \"\"\"\n\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., weight_decay=0., amsgrad=False,\n                 total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):\n        super(RAdam, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n            self.weight_decay = K.variable(weight_decay, name='weight_decay')\n            self.total_steps = K.variable(total_steps, name='total_steps')\n            self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')\n            self.min_lr = K.variable(lr, name='min_lr')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.initial_weight_decay = weight_decay\n        self.initial_total_steps = total_steps\n        self.amsgrad = amsgrad\n\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n\n        lr = self.lr\n\n        if self.initial_decay > 0:\n            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay))))\n\n        t = K.cast(self.iterations, K.floatx()) + 1\n\n        if self.initial_total_steps > 0:\n            warmup_steps = self.total_steps * self.warmup_proportion\n            decay_steps = self.total_steps - warmup_steps\n            lr = K.switch(\n                t <= warmup_steps,\n                lr * (t / warmup_steps),\n                lr * (1.0 - K.minimum(t, decay_steps) / decay_steps),\n            )\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for (i, p) in enumerate(params)]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for (i, p) in enumerate(params)]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for (i, p) in enumerate(params)]\n        else:\n            vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params))]\n\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        beta_1_t = K.pow(self.beta_1, t)\n        beta_2_t = K.pow(self.beta_2, t)\n\n        sma_inf = 2.0 / (1.0 - self.beta_2) - 1.0\n        sma_t = sma_inf - 2.0 * t * beta_2_t / (1.0 - beta_2_t)\n\n        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n\n            m_corr_t = m_t / (1.0 - beta_1_t)\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                v_corr_t = K.sqrt(vhat_t / (1.0 - beta_2_t) + self.epsilon)\n                self.updates.append(K.update(vhat, vhat_t))\n            else:\n                v_corr_t = K.sqrt(v_t / (1.0 - beta_2_t) + self.epsilon)\n\n            r_t = K.sqrt((sma_t - 4.0) / (sma_inf - 4.0) *\n                         (sma_t - 2.0) / (sma_inf - 2.0) *\n                         sma_inf / sma_t)\n\n            p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)\n\n            if self.initial_weight_decay > 0:\n                p_t += self.weight_decay * p\n\n            p_t = p - lr * p_t\n\n            self.updates.append(K.update(m, m_t))\n            self.updates.append(K.update(v, v_t))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {\n            'lr': float(K.get_value(self.lr)),\n            'beta_1': float(K.get_value(self.beta_1)),\n            'beta_2': float(K.get_value(self.beta_2)),\n            'decay': float(K.get_value(self.decay)),\n            'weight_decay': float(K.get_value(self.weight_decay)),\n            'epsilon': self.epsilon,\n            'amsgrad': self.amsgrad,\n            'total_steps': float(K.get_value(self.total_steps)),\n            'warmup_proportion': float(K.get_value(self.warmup_proportion)),\n            'min_lr': float(K.get_value(self.min_lr)),\n        }\n        base_config = super(RAdam, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class GroupNormalization(Layer):\n    \"\"\"Group normalization layer\n    Group Normalization divides the channels into groups and computes within each group\n    the mean and variance for normalization. GN's computation is independent of batch sizes,\n    and its accuracy is stable in a wide range of batch sizes\n    # Arguments\n        groups: Integer, the number of groups for Group Normalization.\n        axis: Integer, the axis that should be normalized\n            (typically the features axis).\n            For instance, after a `Conv2D` layer with\n            `data_format=\"channels_first\"`,\n            set `axis=1` in `BatchNormalization`.\n        epsilon: Small float added to variance to avoid dividing by zero.\n        center: If True, add offset of `beta` to normalized tensor.\n            If False, `beta` is ignored.\n        scale: If True, multiply by `gamma`.\n            If False, `gamma` is not used.\n            When the next layer is linear (also e.g. `nn.relu`),\n            this can be disabled since the scaling\n            will be done by the next layer.\n        beta_initializer: Initializer for the beta weight.\n        gamma_initializer: Initializer for the gamma weight.\n        beta_regularizer: Optional regularizer for the beta weight.\n        gamma_regularizer: Optional regularizer for the gamma weight.\n        beta_constraint: Optional constraint for the beta weight.\n        gamma_constraint: Optional constraint for the gamma weight.\n    # Input shape\n        Arbitrary. Use the keyword argument `input_shape`\n        (tuple of integers, does not include the samples axis)\n        when using this layer as the first layer in a model.\n    # Output shape\n        Same shape as input.\n    # References\n        - [Group Normalization](https://arxiv.org/abs/1803.08494)\n    \"\"\"\n\n    def __init__(self,\n                 groups=32,\n                 axis=-1,\n                 epsilon=1e-5,\n                 center=True,\n                 scale=True,\n                 beta_initializer='zeros',\n                 gamma_initializer='ones',\n                 beta_regularizer=None,\n                 gamma_regularizer=None,\n                 beta_constraint=None,\n                 gamma_constraint=None,\n                 **kwargs):\n        super(GroupNormalization, self).__init__(**kwargs)\n        self.supports_masking = True\n        self.groups = groups\n        self.axis = axis\n        self.epsilon = epsilon\n        self.center = center\n        self.scale = scale\n        self.beta_initializer = initializers.get(beta_initializer)\n        self.gamma_initializer = initializers.get(gamma_initializer)\n        self.beta_regularizer = regularizers.get(beta_regularizer)\n        self.gamma_regularizer = regularizers.get(gamma_regularizer)\n        self.beta_constraint = constraints.get(beta_constraint)\n        self.gamma_constraint = constraints.get(gamma_constraint)\n\n    def build(self, input_shape):\n        dim = input_shape[self.axis]\n\n        if dim is None:\n            raise ValueError('Axis ' + str(self.axis) + ' of '\n                             'input tensor should have a defined dimension '\n                             'but the layer received an input with shape ' +\n                             str(input_shape) + '.')\n\n        if dim < self.groups:\n            raise ValueError('Number of groups (' + str(self.groups) + ') cannot be '\n                             'more than the number of channels (' +\n                             str(dim) + ').')\n\n        if dim % self.groups != 0:\n            raise ValueError('Number of groups (' + str(self.groups) + ') must be a '\n                             'multiple of the number of channels (' +\n                             str(dim) + ').')\n\n        self.input_spec = InputSpec(ndim=len(input_shape),\n                                    axes={self.axis: dim})\n        shape = (dim,)\n\n        if self.scale:\n            self.gamma = self.add_weight(shape=shape,\n                                         name='gamma',\n                                         initializer=self.gamma_initializer,\n                                         regularizer=self.gamma_regularizer,\n                                         constraint=self.gamma_constraint)\n        else:\n            self.gamma = None\n        if self.center:\n            self.beta = self.add_weight(shape=shape,\n                                        name='beta',\n                                        initializer=self.beta_initializer,\n                                        regularizer=self.beta_regularizer,\n                                        constraint=self.beta_constraint)\n        else:\n            self.beta = None\n        self.built = True\n\n    def call(self, inputs, **kwargs):\n        input_shape = K.int_shape(inputs)\n        tensor_input_shape = K.shape(inputs)\n\n        # Prepare broadcasting shape.\n        reduction_axes = list(range(len(input_shape)))\n        del reduction_axes[self.axis]\n        broadcast_shape = [1] * len(input_shape)\n        broadcast_shape[self.axis] = input_shape[self.axis] // self.groups\n        broadcast_shape.insert(1, self.groups)\n\n        reshape_group_shape = K.shape(inputs)\n        group_axes = [reshape_group_shape[i] for i in range(len(input_shape))]\n        group_axes[self.axis] = input_shape[self.axis] // self.groups\n        group_axes.insert(1, self.groups)\n\n        # reshape inputs to new group shape\n        group_shape = [group_axes[0], self.groups] + group_axes[2:]\n        group_shape = K.stack(group_shape)\n        inputs = K.reshape(inputs, group_shape)\n\n        group_reduction_axes = list(range(len(group_axes)))\n        group_reduction_axes = group_reduction_axes[2:]\n\n        mean = K.mean(inputs, axis=group_reduction_axes, keepdims=True)\n        variance = K.var(inputs, axis=group_reduction_axes, keepdims=True)\n\n        inputs = (inputs - mean) / (K.sqrt(variance + self.epsilon))\n\n        # prepare broadcast shape\n        inputs = K.reshape(inputs, group_shape)\n        outputs = inputs\n\n        # In this case we must explicitly broadcast all parameters.\n        if self.scale:\n            broadcast_gamma = K.reshape(self.gamma, broadcast_shape)\n            outputs = outputs * broadcast_gamma\n\n        if self.center:\n            broadcast_beta = K.reshape(self.beta, broadcast_shape)\n            outputs = outputs + broadcast_beta\n\n        outputs = K.reshape(outputs, tensor_input_shape)\n\n        return outputs\n\n    def get_config(self):\n        config = {\n            'groups': self.groups,\n            'axis': self.axis,\n            'epsilon': self.epsilon,\n            'center': self.center,\n            'scale': self.scale,\n            'beta_initializer': initializers.serialize(self.beta_initializer),\n            'gamma_initializer': initializers.serialize(self.gamma_initializer),\n            'beta_regularizer': regularizers.serialize(self.beta_regularizer),\n            'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),\n            'beta_constraint': constraints.serialize(self.beta_constraint),\n            'gamma_constraint': constraints.serialize(self.gamma_constraint)\n        }\n        base_config = super(GroupNormalization, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n\n    def compute_output_shape(self, input_shape):\n        return input_shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"effnet = EfficientNetB5(weights=None,\n                        include_top=False,\n                        input_shape=(IMG_WIDTH, IMG_HEIGHT, CHANNELS))\neffnet.load_weights('../input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, layer in enumerate(effnet.layers):\n    if \"batch_normalization\" in layer.name:\n        effnet.layers[i] = GroupNormalization(groups=32, axis=-1, epsilon=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model():  \n    model = Sequential()\n    model.add(effnet)\n    model.add(GlobalAveragePooling2D())\n    model.add(Dropout(0.5))\n    model.add(Dense(5, activation=elu))\n    model.add(Dense(1, activation=\"linear\"))\n    model.compile(loss='mse',\n                  optimizer=RAdam(lr=0.00005), \n                  metrics=['mse', 'acc'])\n    print(model.summary())\n    return model\n\nmodel = build_model()\nfrom tensorflow.keras.utils import plot_model\n#plot_model(effnet, to_file='model.pdf', show_shapes=True)\nfrom IPython.display import Image\nmodel.save(\"model.h5\")\n\n#from efficientnet.keras import EfficientNetB0\n#from efficientnet.keras import center_crop_and_resize, preprocess_input\n#model_test = EfficientNetB0(weights='imagenet')\n#Image(filename='model.pdf') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(effnet.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kappa_metrics = Metrics()\n\nes = EarlyStopping(monitor='val_loss', mode='auto', verbose=1, patience=12)\nrlr = ReduceLROnPlateau(monitor='val_loss', \n                        factor=0.5, \n                        patience=4, \n                        verbose=1, \n                        mode='auto', \n                        epsilon=0.0001)\n\nmodel.fit_generator(train_generator,\n                    steps_per_epoch=train_generator.samples // BATCH_SIZE,\n                    epochs=35,\n                    validation_data=val_generator,\n                    validation_steps = val_generator.samples // BATCH_SIZE,\n                    callbacks=[kappa_metrics, es, rlr])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_df = pd.DataFrame(model.history.history)\nhistory_df[['loss', 'val_loss']].plot(figsize=(12,5))\nplt.title(\"Loss (MSE)\", fontsize=16, weight='bold')\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss (MSE)\")\nhistory_df[['acc', 'val_acc']].plot(figsize=(12,5))\nplt.title(\"Accuracy\", fontsize=16, weight='bold')\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"% Accuracy\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights(SAVED_MODEL_NAME)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#random prediction\nfig, ax = plt.subplots(1, figsize=(15, 6))\nsample = train_df.sample(1)\nimage_name = sample['id_code'].item()\nx = cv2.imread(f\"{TRAIN_IMG_PATH}{image_name}\")\nx = preprocess_image(x)\n\n#rescale to 1 / 128.\nfor A in range(0, 456):\n    for B in range(0, 456):\n        for C in range(0, 3):\n            x[A,B,C] = x[A,B,C] / 128.\n\nx = np.expand_dims(x, axis=0)\n\npred = model.predict(x)[0,0]\npred = str(round(pred, 2))\n\nX = cv2.imread(f\"{TRAIN_IMG_PATH}{image_name}\")\nax.set_title(f\"Image: {image_name}\\n Label = {sample['diagnosis'].item()}\\n Pred = {pred}\", \n                    weight='bold', fontsize=10)\nax.axis('off')\nRGB_img = cv2.cvtColor(X, cv2.COLOR_BGR2RGB)\nax.imshow(RGB_img);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataGenerator(keras.utils.Sequence):\n    def __init__(self, X, batch_size = 16, img_size = (512, 512, 3), *args, **kwargs):\n        self.X = X\n        self.indices = np.arange(len(self.X))\n        self.batch_size = batch_size\n        self.img_size = img_size\n                    \n    def __len__(self):\n        return int(ceil(len(self.X) / self.batch_size))\n\n    def __getitem__(self, index):\n        indices = self.indices[index*self.batch_size:(index+1)*self.batch_size]\n        X = self.__data_generation(indices)\n        return X\n    \n    def __data_generation(self, indices):\n        X = np.empty((self.batch_size, *self.img_size))\n        \n        for i, index in enumerate(indices):\n            image = self.X[index]\n            image = np.stack((image,)*CHANNELS, axis=-1)\n            image = image.reshape(-1, HEIGHT_NEW, WIDTH_NEW, CHANNELS)\n            \n            X[i,] = image\n        \n        return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"The Training Cohen Kappa Score is: {round(train_score, 5)}\")\nprint(f\"The Validation Cohen Kappa Score is: {round(val_score, 5)}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['diagnosis'] = np.zeros(test_df.shape[0]) \ntest_generator = ImageDataGenerator(preprocessing_function=preprocess_image, \n                                    rescale=1 / 128.).flow_from_dataframe(test_df, \n                                                                          x_col='id_code', \n                                                                          y_col='diagnosis',\n                                                                          directory=TEST_IMG_PATH,\n                                                                          target_size=(IMG_WIDTH, IMG_HEIGHT),\n                                                                          batch_size=BATCH_SIZE,\n                                                                          class_mode='other',\n                                                                          shuffle=False)\ny_test, _ = get_preds_and_labels(model, test_generator)\ny_test = np.rint(y_train_preds).astype(np.uint8).clip(0, 4)\ntest_df['diagnosis'] = y_test","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class OptimizedRounder(object):\n    \"\"\"\n    An optimizer for rounding thresholds\n    to maximize Quadratic Weighted Kappa score\n    \"\"\"\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        \"\"\"\n        Get loss according to\n        using current coefficients\n        \n        :param coef: A list of coefficients that will be used for rounding\n        :param X: The raw predictions\n        :param y: The ground truth labels\n        \"\"\"\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n\n        ll = cohen_kappa_score(y, X_p, weights='quadratic')\n        return -ll\n\n    def fit(self, X, y):\n        \"\"\"\n        Optimize rounding thresholds\n        \n        :param X: The raw predictions\n        :param y: The ground truth labels\n        \"\"\"\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5]\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n\n    def predict(self, X, coef):\n        \"\"\"\n        Make predictions with specified thresholds\n        \n        :param X: The raw predictions\n        :param coef: A list of coefficients that will be used for rounding\n        \"\"\"\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n        return X_p\n\n    def coefficients(self):\n        \"\"\"\n        Return the optimized coefficients\n        \"\"\"\n        return self.coef_['x']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_val_preds, val_labels = get_preds_and_labels(model, val_generator)\noptR = OptimizedRounder()\noptR.fit(y_val_preds, val_labels)\ncoefficients = optR.coefficients()\nopt_val_predictions = optR.predict(y_val_preds, coefficients)\nnew_val_score = cohen_kappa_score(val_labels, opt_val_predictions, weights=\"quadratic\")\n\ntest_df['diagnosis'] = np.zeros(test_df.shape[0]) \n\ntest_generator = ImageDataGenerator(preprocessing_function=preprocess_image, \n                                    rescale=1 / 128.).flow_from_dataframe(test_df, \n                                                                          x_col='id_code', \n                                                                          y_col='diagnosis',\n                                                                          directory=TEST_IMG_PATH,\n                                                                          target_size=(IMG_WIDTH, IMG_HEIGHT),\n                                                                          batch_size=BATCH_SIZE,\n                                                                          class_mode='other',\n                                                                          shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test, _ = get_preds_and_labels(model, test_generator)\ny_test = optR.predict(y_test, coefficients).astype(np.uint8)\ntest_df['diagnosis'] = y_test\ntest_df['id_code'] = test_df['id_code'].str.replace(r'.png$', '')\ntest_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['diagnosis'].value_counts().sort_index().plot(kind=\"bar\", \n                                                       figsize=(12,5), \n                                                       rot=0)\nplt.title(\"Label Distribution (Training Set)\", \n          weight='bold', \n          fontsize=18)\nplt.xticks(fontsize=15)\nplt.yticks(fontsize=15)\nplt.xlabel(\"Label\", fontsize=17)\nplt.ylabel(\"Frequency\", fontsize=17);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['diagnosis'].value_counts().sort_index().plot(kind=\"bar\", \n                                                      figsize=(12,5), \n                                                      rot=0)\nplt.title(\"Label Distribution (Predictions)\", \n          weight='bold', \n          fontsize=18)\nplt.xticks(fontsize=15)\nplt.yticks(fontsize=15)\nplt.xlabel(\"Label\", fontsize=17)\nplt.ylabel(\"Frequency\", fontsize=17);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t_finish = time.time()\ntotal_time = round((t_finish-t_start) / 3600, 4)\nprint('runtime = {} hours ({} minutes)'.format(total_time, \n                                                      int(total_time*60)))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}