{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":187731,"sourceType":"datasetVersion","datasetId":80814},{"sourceId":556303,"sourceType":"datasetVersion","datasetId":266957},{"sourceId":556726,"sourceType":"datasetVersion","datasetId":267272}],"dockerImageVersionId":29188,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Standard dependencies\nimport cv2\nimport time\nimport scipy as sp\nimport numpy as np\nimport random as rn\nimport pandas as pd\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom functools import partial\nimport matplotlib.pyplot as plt\n\n# Machine Learning\nimport tensorflow as tf\nimport keras\nfrom keras import initializers\nfrom keras import regularizers\nfrom keras import constraints\nfrom keras import backend as K\nfrom keras.activations import elu\nfrom keras.optimizers import Adam\nfrom keras.models import Sequential\nfrom keras.engine import Layer, InputSpec\nfrom keras.utils.generic_utils import get_custom_objects\nfrom keras.callbacks import Callback, EarlyStopping, ReduceLROnPlateau\nfrom keras.layers import Dense, Conv2D, Flatten, GlobalAveragePooling2D, Dropout\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.metrics import cohen_kappa_score\n\n# Path specifications\nKAGGLE_DIR = '../input/aptos2019-blindness-detection/'\nTRAIN_DF_PATH = KAGGLE_DIR + \"train.csv\"\nTEST_DF_PATH = KAGGLE_DIR + 'test.csv'\nTRAIN_IMG_PATH = KAGGLE_DIR + \"train_images/\"\nTEST_IMG_PATH = KAGGLE_DIR + 'test_images/'\n\n# Specify title of our final model\nSAVED_MODEL_NAME = 'model.h5'\n\n","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:05:31.366450Z","iopub.execute_input":"2024-11-02T18:05:31.366751Z","iopub.status.idle":"2024-11-02T18:05:32.203855Z","shell.execute_reply.started":"2024-11-02T18:05:31.366708Z","shell.execute_reply":"2024-11-02T18:05:32.202696Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Image IDs and Labels (TRAIN)\")\ntrain_df = pd.read_csv(TRAIN_DF_PATH)\n# Add extension to id_code\ntrain_df['id_code'] = train_df['id_code'] + \".png\"\nprint(f\"Training images: {train_df.shape[0]}\")\ndisplay(train_df.head())\n\nprint(\"Image IDs (TEST)\")\ntest_df = pd.read_csv(TEST_DF_PATH)\n# Add extension to id_code\ntest_df['id_code'] = test_df['id_code'] + \".png\"\nprint(f\"Testing Images: {test_df.shape[0]}\")\ndisplay(test_df.head())","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:05:32.207395Z","iopub.execute_input":"2024-11-02T18:05:32.207715Z","iopub.status.idle":"2024-11-02T18:05:32.335007Z","shell.execute_reply.started":"2024-11-02T18:05:32.207665Z","shell.execute_reply":"2024-11-02T18:05:32.334124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Specify image size\nIMG_WIDTH = 224\nIMG_HEIGHT = 224\nCHANNELS = 3","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:05:32.339466Z","iopub.execute_input":"2024-11-02T18:05:32.339700Z","iopub.status.idle":"2024-11-02T18:05:32.343140Z","shell.execute_reply.started":"2024-11-02T18:05:32.339660Z","shell.execute_reply":"2024-11-02T18:05:32.342405Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_preds_and_labels(model, generator):\n    \"\"\"\n    Get predictions and labels from the generator\n    \n    :param model: A Keras model object\n    :param generator: A Keras ImageDataGenerator object\n    \n    :return: A tuple with two Numpy Arrays. One containing the predictions\n    and one containing the labels\n    \"\"\"\n    preds = []\n    labels = []\n    for _ in range(int(np.ceil(generator.samples / BATCH_SIZE))):\n        x, y = next(generator)\n        preds.append(model.predict(x))\n        labels.append(y)\n    # Flatten list of numpy arrays\n    return np.concatenate(preds).ravel(), np.concatenate(labels).ravel()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:05:33.070103Z","iopub.execute_input":"2024-11-02T18:05:33.070405Z","iopub.status.idle":"2024-11-02T18:05:33.077092Z","shell.execute_reply.started":"2024-11-02T18:05:33.070361Z","shell.execute_reply":"2024-11-02T18:05:33.076334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Metrics(Callback):\n\n    def on_train_begin(self, logs={}):\n\n        self.val_kappas = []\n\n    def on_epoch_end(self, epoch, logs={}):\n\n        # Get predictions and convert to integers\n        y_pred, labels = get_preds_and_labels(model, val_generator)\n        y_pred = np.rint(y_pred).astype(np.uint8).clip(0, 4)\n        # We can use sklearns implementation of QWK straight out of the box\n        # as long as we specify weights as 'quadratic'\n        _val_kappa = cohen_kappa_score(labels, y_pred, weights='quadratic')\n        self.val_kappas.append(_val_kappa)\n        print(f\"val_kappa: {round(_val_kappa, 4)}\")\n        if _val_kappa == max(self.val_kappas):\n            print(\"Validation Kappa has improved. Saving model.\")\n            self.model.save(SAVED_MODEL_NAME)\n        return","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:05:33.246521Z","iopub.execute_input":"2024-11-02T18:05:33.246852Z","iopub.status.idle":"2024-11-02T18:05:33.255482Z","shell.execute_reply.started":"2024-11-02T18:05:33.246795Z","shell.execute_reply":"2024-11-02T18:05:33.254617Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Label distribution\ntrain_df['diagnosis'].value_counts().sort_index().plot(kind=\"bar\", \n                                                       figsize=(12,5), \n                                                       rot=0)\nplt.title(\"Label Distribution (Training Set)\", \n          weight='bold', \n          fontsize=18)\nplt.xticks(fontsize=15)\nplt.yticks(fontsize=15)\nplt.xlabel(\"Label\", fontsize=17)\nplt.ylabel(\"Frequency\", fontsize=17);","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:05:34.048672Z","iopub.execute_input":"2024-11-02T18:05:34.048985Z","iopub.status.idle":"2024-11-02T18:05:34.398763Z","shell.execute_reply.started":"2024-11-02T18:05:34.048940Z","shell.execute_reply":"2024-11-02T18:05:34.397701Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Example from every label\nfig, ax = plt.subplots(1, 5, figsize=(15, 6))\nfor i in range(5):\n    sample = train_df[train_df['diagnosis'] == i].sample(1)\n    image_name = sample['id_code'].item()\n    X = cv2.imread(f\"{TRAIN_IMG_PATH}{image_name}\")\n    ax[i].set_title(f\"Image: {image_name}\\n Label = {sample['diagnosis'].item()}\", \n                    weight='bold', fontsize=10)\n    ax[i].axis('off')\n    ax[i].imshow(X);","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:05:34.856538Z","iopub.execute_input":"2024-11-02T18:05:34.856841Z","iopub.status.idle":"2024-11-02T18:05:36.278542Z","shell.execute_reply.started":"2024-11-02T18:05:34.856799Z","shell.execute_reply":"2024-11-02T18:05:36.277506Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def crop_image_from_gray(img, tol=7):\n\n    # If for some reason we only have two channels\n    if img.ndim == 2:\n        mask = img > tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    # If we have a normal RGB images\n    elif img.ndim == 3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img > tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([img1,img2,img3],axis=-1)\n        return img\n\ndef preprocess_image(image, sigmaX=10):\n    \"\"\"\n    The whole preprocessing pipeline:\n    1. Read in image\n    2. Apply masks\n    3. Resize image to desired size\n    4. Add Gaussian noise to increase Robustness\n    \n    :param img: A NumPy Array that will be cropped\n    :param sigmaX: Value used for add GaussianBlur to the image\n    \n    :return: A NumPy array containing the preprocessed image\n    \"\"\"\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = crop_image_from_gray(image)\n    image = cv2.resize(image, (IMG_WIDTH, IMG_HEIGHT))\n    image = cv2.addWeighted (image,4, cv2.GaussianBlur(image, (0,0) ,sigmaX), -4, 128)\n    return image","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:05:36.280126Z","iopub.execute_input":"2024-11-02T18:05:36.280400Z","iopub.status.idle":"2024-11-02T18:05:36.294598Z","shell.execute_reply.started":"2024-11-02T18:05:36.280348Z","shell.execute_reply":"2024-11-02T18:05:36.293579Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Example of preprocessed images from every label\nfig, ax = plt.subplots(1, 5, figsize=(15, 6))\nfor i in range(5):\n    sample = train_df[train_df['diagnosis'] == i].sample(1)\n    image_name = sample['id_code'].item()\n    X = preprocess_image(cv2.imread(f\"{TRAIN_IMG_PATH}{image_name}\"))\n    ax[i].set_title(f\"Image: {image_name}\\n Label = {sample['diagnosis'].item()}\", \n                    weight='bold', fontsize=10)\n    ax[i].axis('off')\n    ax[i].imshow(X);","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:05:36.816418Z","iopub.execute_input":"2024-11-02T18:05:36.816711Z","iopub.status.idle":"2024-11-02T18:05:38.222423Z","shell.execute_reply.started":"2024-11-02T18:05:36.816668Z","shell.execute_reply":"2024-11-02T18:05:38.221492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['diagnosis'] = train_df['diagnosis'].astype(str)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:05:38.224048Z","iopub.execute_input":"2024-11-02T18:05:38.224412Z","iopub.status.idle":"2024-11-02T18:05:38.232689Z","shell.execute_reply.started":"2024-11-02T18:05:38.224342Z","shell.execute_reply":"2024-11-02T18:05:38.231972Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Labels for training data\ny_labels = train_df['diagnosis'].values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:05:38.234291Z","iopub.execute_input":"2024-11-02T18:05:38.234620Z","iopub.status.idle":"2024-11-02T18:05:38.243137Z","shell.execute_reply.started":"2024-11-02T18:05:38.234562Z","shell.execute_reply":"2024-11-02T18:05:38.242443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# We use a small batch size so we can handle large images easily\nBATCH_SIZE = 32\n\n# Add Image augmentation to our generator\ntrain_datagen = ImageDataGenerator(rotation_range=360,\n                                   horizontal_flip=True,\n                                   vertical_flip=True,\n                                   validation_split=0.15,\n                                   preprocessing_function=preprocess_image, \n                                   rescale=1 / 128.)\n\n# Use the dataframe to define train and validation generators\ntrain_generator = train_datagen.flow_from_dataframe(train_df, \n                                                    x_col='id_code', \n                                                    y_col='diagnosis',\n                                                    directory = TRAIN_IMG_PATH,\n                                                    target_size=(IMG_WIDTH, IMG_HEIGHT),\n                                                    batch_size=BATCH_SIZE,\n                                                    class_mode='categorical', \n                                                    subset='training')\n\nval_generator = train_datagen.flow_from_dataframe(train_df, \n                                                  x_col='id_code', \n                                                  y_col='diagnosis',\n                                                  directory = TRAIN_IMG_PATH,\n                                                  target_size=(IMG_WIDTH, IMG_HEIGHT),\n                                                  batch_size=BATCH_SIZE,\n                                                  class_mode='categorical',\n                                                  subset='validation')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:05:39.348731Z","iopub.execute_input":"2024-11-02T18:05:39.349069Z","iopub.status.idle":"2024-11-02T18:05:47.954651Z","shell.execute_reply.started":"2024-11-02T18:05:39.349022Z","shell.execute_reply":"2024-11-02T18:05:47.953722Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Code Source: https://github.com/CyberZHG/keras-radam/blob/master/keras_radam/optimizers.py\nclass RAdam(keras.optimizers.Optimizer):\n    \"\"\"RAdam optimizer.\n    # Arguments\n        lr: float >= 0. Learning rate.\n        beta_1: float, 0 < beta < 1. Generally close to 1.\n        beta_2: float, 0 < beta < 1. Generally close to 1.\n        epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`.\n        decay: float >= 0. Learning rate decay over each update.\n        weight_decay: float >= 0. Weight decay for each param.\n        amsgrad: boolean. Whether to apply the AMSGrad variant of this\n            algorithm from the paper \"On the Convergence of Adam and\n            Beyond\".\n        total_steps: int >= 0. Total number of training steps. Enable warmup by setting a positive value.\n        warmup_proportion: 0 < warmup_proportion < 1. The proportion of increasing steps.\n        min_lr: float >= 0. Minimum learning rate after warmup.\n    # References\n        - [Adam - A Method for Stochastic Optimization](https://arxiv.org/abs/1412.6980v8)\n        - [On the Convergence of Adam and Beyond](https://openreview.net/forum?id=ryQu7f-RZ)\n        - [On The Variance Of The Adaptive Learning Rate And Beyond](https://arxiv.org/pdf/1908.03265v1.pdf)\n    \"\"\"\n\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., weight_decay=0., amsgrad=False,\n                 total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):\n        super(RAdam, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n            self.weight_decay = K.variable(weight_decay, name='weight_decay')\n            self.total_steps = K.variable(total_steps, name='total_steps')\n            self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')\n            self.min_lr = K.variable(lr, name='min_lr')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.initial_weight_decay = weight_decay\n        self.initial_total_steps = total_steps\n        self.amsgrad = amsgrad\n\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n\n        lr = self.lr\n\n        if self.initial_decay > 0:\n            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay))))\n\n        t = K.cast(self.iterations, K.floatx()) + 1\n\n        if self.initial_total_steps > 0:\n            warmup_steps = self.total_steps * self.warmup_proportion\n            decay_steps = self.total_steps - warmup_steps\n            lr = K.switch(\n                t <= warmup_steps,\n                lr * (t / warmup_steps),\n                lr * (1.0 - K.minimum(t, decay_steps) / decay_steps),\n            )\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for (i, p) in enumerate(params)]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for (i, p) in enumerate(params)]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for (i, p) in enumerate(params)]\n        else:\n            vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params))]\n\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        beta_1_t = K.pow(self.beta_1, t)\n        beta_2_t = K.pow(self.beta_2, t)\n\n        sma_inf = 2.0 / (1.0 - self.beta_2) - 1.0\n        sma_t = sma_inf - 2.0 * t * beta_2_t / (1.0 - beta_2_t)\n\n        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n\n            m_corr_t = m_t / (1.0 - beta_1_t)\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                v_corr_t = K.sqrt(vhat_t / (1.0 - beta_2_t) + self.epsilon)\n                self.updates.append(K.update(vhat, vhat_t))\n            else:\n                v_corr_t = K.sqrt(v_t / (1.0 - beta_2_t) + self.epsilon)\n\n            r_t = K.sqrt((sma_t - 4.0) / (sma_inf - 4.0) *\n                         (sma_t - 2.0) / (sma_inf - 2.0) *\n                         sma_inf / sma_t)\n\n            p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)\n\n            if self.initial_weight_decay > 0:\n                p_t += self.weight_decay * p\n\n            p_t = p - lr * p_t\n\n            self.updates.append(K.update(m, m_t))\n            self.updates.append(K.update(v, v_t))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {\n            'lr': float(K.get_value(self.lr)),\n            'beta_1': float(K.get_value(self.beta_1)),\n            'beta_2': float(K.get_value(self.beta_2)),\n            'decay': float(K.get_value(self.decay)),\n            'weight_decay': float(K.get_value(self.weight_decay)),\n            'epsilon': self.epsilon,\n            'amsgrad': self.amsgrad,\n            'total_steps': float(K.get_value(self.total_steps)),\n            'warmup_proportion': float(K.get_value(self.warmup_proportion)),\n            'min_lr': float(K.get_value(self.min_lr)),\n        }\n        base_config = super(RAdam, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:05:47.956808Z","iopub.execute_input":"2024-11-02T18:05:47.957267Z","iopub.status.idle":"2024-11-02T18:05:47.999450Z","shell.execute_reply.started":"2024-11-02T18:05:47.957086Z","shell.execute_reply":"2024-11-02T18:05:47.998596Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class GroupNormalization(Layer):\n    \"\"\"Group normalization layer\n    Group Normalization divides the channels into groups and computes within each group\n    the mean and variance for normalization. GN's computation is independent of batch sizes,\n    and its accuracy is stable in a wide range of batch sizes\n    # Arguments\n        groups: Integer, the number of groups for Group Normalization.\n        axis: Integer, the axis that should be normalized\n            (typically the features axis).\n            For instance, after a `Conv2D` layer with\n            `data_format=\"channels_first\"`,\n            set `axis=1` in `BatchNormalization`.\n        epsilon: Small float added to variance to avoid dividing by zero.\n        center: If True, add offset of `beta` to normalized tensor.\n            If False, `beta` is ignored.\n        scale: If True, multiply by `gamma`.\n            If False, `gamma` is not used.\n            When the next layer is linear (also e.g. `nn.relu`),\n            this can be disabled since the scaling\n            will be done by the next layer.\n        beta_initializer: Initializer for the beta weight.\n        gamma_initializer: Initializer for the gamma weight.\n        beta_regularizer: Optional regularizer for the beta weight.\n        gamma_regularizer: Optional regularizer for the gamma weight.\n        beta_constraint: Optional constraint for the beta weight.\n        gamma_constraint: Optional constraint for the gamma weight.\n    # Input shape\n        Arbitrary. Use the keyword argument `input_shape`\n        (tuple of integers, does not include the samples axis)\n        when using this layer as the first layer in a model.\n    # Output shape\n        Same shape as input.\n    # References\n        - [Group Normalization](https://arxiv.org/abs/1803.08494)\n    \"\"\"\n\n    def __init__(self,\n                 groups=32,\n                 axis=-1,\n                 epsilon=1e-5,\n                 center=True,\n                 scale=True,\n                 beta_initializer='zeros',\n                 gamma_initializer='ones',\n                 beta_regularizer=None,\n                 gamma_regularizer=None,\n                 beta_constraint=None,\n                 gamma_constraint=None,\n                 **kwargs):\n        super(GroupNormalization, self).__init__(**kwargs)\n        self.supports_masking = True\n        self.groups = groups\n        self.axis = axis\n        self.epsilon = epsilon\n        self.center = center\n        self.scale = scale\n        self.beta_initializer = initializers.get(beta_initializer)\n        self.gamma_initializer = initializers.get(gamma_initializer)\n        self.beta_regularizer = regularizers.get(beta_regularizer)\n        self.gamma_regularizer = regularizers.get(gamma_regularizer)\n        self.beta_constraint = constraints.get(beta_constraint)\n        self.gamma_constraint = constraints.get(gamma_constraint)\n\n    def build(self, input_shape):\n        dim = input_shape[self.axis]\n\n        if dim is None:\n            raise ValueError('Axis ' + str(self.axis) + ' of '\n                             'input tensor should have a defined dimension '\n                             'but the layer received an input with shape ' +\n                             str(input_shape) + '.')\n\n        if dim < self.groups:\n            raise ValueError('Number of groups (' + str(self.groups) + ') cannot be '\n                             'more than the number of channels (' +\n                             str(dim) + ').')\n\n        if dim % self.groups != 0:\n            raise ValueError('Number of groups (' + str(self.groups) + ') must be a '\n                             'multiple of the number of channels (' +\n                             str(dim) + ').')\n\n        self.input_spec = InputSpec(ndim=len(input_shape),\n                                    axes={self.axis: dim})\n        shape = (dim,)\n\n        if self.scale:\n            self.gamma = self.add_weight(shape=shape,\n                                         name='gamma',\n                                         initializer=self.gamma_initializer,\n                                         regularizer=self.gamma_regularizer,\n                                         constraint=self.gamma_constraint)\n        else:\n            self.gamma = None\n        if self.center:\n            self.beta = self.add_weight(shape=shape,\n                                        name='beta',\n                                        initializer=self.beta_initializer,\n                                        regularizer=self.beta_regularizer,\n                                        constraint=self.beta_constraint)\n        else:\n            self.beta = None\n        self.built = True\n\n    def call(self, inputs, **kwargs):\n        input_shape = K.int_shape(inputs)\n        tensor_input_shape = K.shape(inputs)\n\n        # Prepare broadcasting shape.\n        reduction_axes = list(range(len(input_shape)))\n        del reduction_axes[self.axis]\n        broadcast_shape = [1] * len(input_shape)\n        broadcast_shape[self.axis] = input_shape[self.axis] // self.groups\n        broadcast_shape.insert(1, self.groups)\n\n        reshape_group_shape = K.shape(inputs)\n        group_axes = [reshape_group_shape[i] for i in range(len(input_shape))]\n        group_axes[self.axis] = input_shape[self.axis] // self.groups\n        group_axes.insert(1, self.groups)\n\n        # reshape inputs to new group shape\n        group_shape = [group_axes[0], self.groups] + group_axes[2:]\n        group_shape = K.stack(group_shape)\n        inputs = K.reshape(inputs, group_shape)\n\n        group_reduction_axes = list(range(len(group_axes)))\n        group_reduction_axes = group_reduction_axes[2:]\n\n        mean = K.mean(inputs, axis=group_reduction_axes, keepdims=True)\n        variance = K.var(inputs, axis=group_reduction_axes, keepdims=True)\n\n        inputs = (inputs - mean) / (K.sqrt(variance + self.epsilon))\n\n        # prepare broadcast shape\n        inputs = K.reshape(inputs, group_shape)\n        outputs = inputs\n\n        # In this case we must explicitly broadcast all parameters.\n        if self.scale:\n            broadcast_gamma = K.reshape(self.gamma, broadcast_shape)\n            outputs = outputs * broadcast_gamma\n\n        if self.center:\n            broadcast_beta = K.reshape(self.beta, broadcast_shape)\n            outputs = outputs + broadcast_beta\n\n        outputs = K.reshape(outputs, tensor_input_shape)\n\n        return outputs\n\n    def get_config(self):\n        config = {\n            'groups': self.groups,\n            'axis': self.axis,\n            'epsilon': self.epsilon,\n            'center': self.center,\n            'scale': self.scale,\n            'beta_initializer': initializers.serialize(self.beta_initializer),\n            'gamma_initializer': initializers.serialize(self.gamma_initializer),\n            'beta_regularizer': regularizers.serialize(self.beta_regularizer),\n            'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),\n            'beta_constraint': constraints.serialize(self.beta_constraint),\n            'gamma_constraint': constraints.serialize(self.gamma_constraint)\n        }\n        base_config = super(GroupNormalization, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n\n    def compute_output_shape(self, input_shape):\n        return input_shape","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:05:48.000920Z","iopub.execute_input":"2024-11-02T18:05:48.001229Z","iopub.status.idle":"2024-11-02T18:05:48.038416Z","shell.execute_reply.started":"2024-11-02T18:05:48.001177Z","shell.execute_reply":"2024-11-02T18:05:48.037704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras.applications import DenseNet121\ndensenet = DenseNet121(\n    weights='../input/densenet-keras/DenseNet-BC-121-32-no-top.h5',\n    include_top=False,\n    input_shape=(224,224,3)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:07:00.420626Z","iopub.execute_input":"2024-11-02T18:07:00.420934Z","iopub.status.idle":"2024-11-02T18:07:30.547793Z","shell.execute_reply.started":"2024-11-02T18:07:00.420888Z","shell.execute_reply":"2024-11-02T18:07:30.547080Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Replace all Batch Normalization layers by Group Normalization layers\nfor i, layer in enumerate(densenet.layers):\n    if \"batch_normalization\" in layer.name:\n        densenet.layers[i] = GroupNormalization(groups=32, axis=-1, epsilon=0.00001)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:07:30.549755Z","iopub.execute_input":"2024-11-02T18:07:30.550111Z","iopub.status.idle":"2024-11-02T18:07:30.555800Z","shell.execute_reply.started":"2024-11-02T18:07:30.550051Z","shell.execute_reply":"2024-11-02T18:07:30.555047Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def build_model():\n\n\n# Sequential-style model construction with added layers\n    model = Sequential()\n    model.add(densenet)  # Add DenseNet as the base\n    model.add(GlobalAveragePooling2D())  # Reduces the dimensions of the feature map to (None, 1024)\n    model.add(Dropout(0.5))  # Dropout layer with rate 0.5 to prevent overfitting\n    model.add(Dense(5, activation='softmax'))  # Output layer with softmax for 5-class classification\n    model.compile(loss='mse',\n                  optimizer=RAdam(lr=0.00005), \n                  metrics=['mse', 'acc'])\n    print(model.summary())\n    return model\n\n# Initialize model\nmodel = build_model()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T18:06:21.023976Z","iopub.execute_input":"2024-11-02T18:06:21.024240Z","iopub.status.idle":"2024-11-02T18:06:31.891606Z","shell.execute_reply.started":"2024-11-02T18:06:21.024189Z","shell.execute_reply":"2024-11-02T18:06:31.890274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# For tracking Quadratic Weighted Kappa score\nkappa_metrics = Metrics()\n# Monitor MSE to avoid overfitting and save best model\nes = EarlyStopping(monitor='val_loss', mode='auto', verbose=1, patience=12)\nrlr = ReduceLROnPlateau(monitor='val_loss', \n                        factor=0.5, \n                        patience=4, \n                        verbose=1, \n                        mode='auto', \n                        epsilon=0.0001)\n\n# Begin training\nmodel.fit_generator(train_generator,\n                    steps_per_epoch=train_generator.samples // BATCH_SIZE,\n                    epochs=15,\n                    validation_data=val_generator,\n                    validation_steps = val_generator.samples // BATCH_SIZE,\n                    callbacks=[kappa_metrics, es, rlr])","metadata":{"_kg_hide-output":true,"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T14:24:37.629781Z","iopub.execute_input":"2024-11-02T14:24:37.630119Z","iopub.status.idle":"2024-11-02T16:10:11.626956Z","shell.execute_reply.started":"2024-11-02T14:24:37.630058Z","shell.execute_reply":"2024-11-02T16:10:11.626192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize mse\nhistory_df = pd.DataFrame(model.history.history)\nhistory_df[['loss', 'val_loss']].plot(figsize=(12,5))\nplt.title(\"Loss (MSE)\", fontsize=16, weight='bold')\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss (MSE)\")\nhistory_df[['acc', 'val_acc']].plot(figsize=(12,5))\nplt.title(\"Accuracy\", fontsize=16, weight='bold')\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"% Accuracy\");","metadata":{"_kg_hide-input":true,"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T16:10:12.362685Z","iopub.execute_input":"2024-11-02T16:10:12.363091Z","iopub.status.idle":"2024-11-02T16:10:13.103209Z","shell.execute_reply.started":"2024-11-02T16:10:12.363032Z","shell.execute_reply":"2024-11-02T16:10:13.101792Z"}},"outputs":[],"execution_count":null}]}