{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":9900,"sourceType":"datasetVersion","datasetId":6209},{"sourceId":10341599,"sourceType":"datasetVersion","datasetId":6391555}],"dockerImageVersionId":27938,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"* Update: changing the batch_size from 32 to 128 when warm up the model, adding Kappa loss from this kernel: https://www.kaggle.com/christofhenkel/weighted-kappa-loss-for-keras-tensorflow","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:47:59.818536Z","iopub.execute_input":"2024-12-31T11:47:59.818806Z","iopub.status.idle":"2024-12-31T11:47:59.825234Z","shell.execute_reply.started":"2024-12-31T11:47:59.818765Z","shell.execute_reply":"2024-12-31T11:47:59.824403Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport skimage.io\nfrom skimage.transform import resize\nfrom imgaug import augmenters as iaa\nfrom tqdm import tqdm\nimport PIL\nfrom PIL import Image, ImageOps\nimport cv2\nfrom sklearn.utils import class_weight, shuffle\nfrom keras.losses import binary_crossentropy, categorical_crossentropy\nfrom keras.applications.resnet50 import preprocess_input\nimport keras.backend as K\nimport tensorflow as tf\nfrom sklearn.metrics import f1_score, fbeta_score, cohen_kappa_score, accuracy_score\nfrom keras.utils import Sequence\nfrom keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split\n\nWORKERS = 2\nCHANNEL = 3\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nSIZE = 224\nNUM_CLASSES = 4","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:47:59.828970Z","iopub.execute_input":"2024-12-31T11:47:59.829319Z","iopub.status.idle":"2024-12-31T11:48:02.221823Z","shell.execute_reply.started":"2024-12-31T11:47:59.829242Z","shell.execute_reply":"2024-12-31T11:48:02.220873Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Reading CSV","metadata":{}},{"cell_type":"code","source":"import csv\n\ndef csv_label_modifier(csv_path):\n    # Input and output CSV file paths\n    input_csv = csv_path\n    output_csv = f\"/kaggle/working/{csv_path.split('/')[-1]}\"\n    \n    # Read the input CSV and modify the labels\n    data = []\n    with open(input_csv, mode=\"r\") as file:\n        reader = csv.DictReader(file)\n        for row in reader:\n            # Convert labels: 1 -> 0, 2 -> 1, 3 -> 2, 4 -> 3\n            row[\"label\"] = str(int(row[\"label\"]) - 1)\n            data.append(row)\n    \n    # Write the modified data to a new CSV file\n    with open(output_csv, mode=\"w\", newline=\"\") as file:\n        writer = csv.DictWriter(file, fieldnames=[\"img_name\", \"label\"])\n        writer.writeheader()\n        writer.writerows(data)\n    \n    print(f\"Modified labels have been saved to '{output_csv}'.\")\n\n    return output_csv\n    \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:48:02.224246Z","iopub.execute_input":"2024-12-31T11:48:02.224647Z","iopub.status.idle":"2024-12-31T11:48:02.232117Z","shell.execute_reply.started":"2024-12-31T11:48:02.224578Z","shell.execute_reply":"2024-12-31T11:48:02.231292Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = pd.read_csv(csv_label_modifier('/kaggle/input/aptos19-clahe/Aptos19_Clahe_onlyDR_aug/Aptos19_Clahe_onlyDR_aug/All/train_labels.csv'))\ndf_test = pd.read_csv(csv_label_modifier('/kaggle/input/aptos19-clahe/Aptos19_Clahe_onlyDR_aug/Aptos19_Clahe_onlyDR_aug/All/test_labels.csv'))\n\nprint(df_train.head())\nprint(df_test.head())\n\nx = df_train['img_name']\ny = df_train['label']\n\nx, y = shuffle(x, y, random_state=8)\ny.hist()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:48:02.233663Z","iopub.execute_input":"2024-12-31T11:48:02.233918Z","iopub.status.idle":"2024-12-31T11:48:02.622654Z","shell.execute_reply.started":"2024-12-31T11:48:02.233868Z","shell.execute_reply":"2024-12-31T11:48:02.621307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def to_multi_label(target):\n#     multi_label = np.zeros((len(target), NUM_CLASSES))\n#     for i in range(len(target)):\n#         j = target[i] + 1\n#         multi_label[i][:j] = 1\n#     return np.array(multi_label)\n# multi_y = to_multi_label(y)\n# for j in range(5):\n#     print('original: ', y[j])\n#     print('multi-label: ', multi_y[j])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:48:02.624804Z","iopub.execute_input":"2024-12-31T11:48:02.625294Z","iopub.status.idle":"2024-12-31T11:48:02.631046Z","shell.execute_reply.started":"2024-12-31T11:48:02.625208Z","shell.execute_reply":"2024-12-31T11:48:02.629223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras.legacy import interfaces\nfrom keras.optimizers import Optimizer\nfrom keras import backend as K\n\n\nclass AdamAccumulate_v1(Optimizer):\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., amsgrad=False, accum_iters=20, **kwargs):\n        super(AdamAccumulate, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.effective_iterations = K.variable(0, dtype='int64', name='effective_iterations')\n\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.amsgrad = amsgrad\n        self.accum_iters = K.variable(accum_iters, dtype='int64')\n\n    @interfaces.legacy_get_updates_support\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n\n        self.updates = [K.update(self.iterations, self.iterations + 1)]\n\n        flag = K.equal(self.iterations % self.accum_iters, self.accum_iters - 1)\n        flag = K.cast(flag, K.floatx())\n\n        self.updates.append(K.update(self.effective_iterations,\n                                     self.effective_iterations + K.cast(flag, 'int64')))\n\n        lr = self.lr\n        if self.initial_decay > 0:\n            lr = lr * (1. / (1. + self.decay * K.cast(self.effective_iterations,\n                                                      K.dtype(self.decay))))\n\n        t = K.cast(self.effective_iterations, K.floatx()) + 1\n\n        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /\n                     (1. - K.pow(self.beta_1, t)))\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        else:\n            vhats = [K.zeros(1) for _ in params]\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        for p, g, m, v, vhat, gg in zip(params, grads, ms, vs, vhats, gs):\n\n            gg_t = (1 - flag) * (gg + g)\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * (gg + flag * g) / K.cast(self.accum_iters, K.floatx())\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(\n                (gg + flag * g) / K.cast(self.accum_iters, K.floatx()))\n\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                p_t = p - flag * lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)\n                self.updates.append(K.update(vhat, vhat_t))\n            else:\n                p_t = p - flag * lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n            self.updates.append((m, flag * m_t + (1 - flag) * m))\n            self.updates.append((v, flag * v_t + (1 - flag) * v))\n            self.updates.append((gg, gg_t))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {'lr': float(K.get_value(self.lr)),\n                  'beta_1': float(K.get_value(self.beta_1)),\n                  'beta_2': float(K.get_value(self.beta_2)),\n                  'decay': float(K.get_value(self.decay)),\n                  'epsilon': self.epsilon,\n                  'amsgrad': self.amsgrad}\n        base_config = super(AdamAccumulate, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n\n\nclass AdamAccumulate(Optimizer):\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., amsgrad=False, accum_iters=2, **kwargs):\n        if accum_iters < 1:\n            raise ValueError('accum_iters must be >= 1')\n        super(AdamAccumulate, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.amsgrad = amsgrad\n        self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))\n        self.accum_iters_float = K.cast(self.accum_iters, K.floatx())\n\n    @interfaces.legacy_get_updates_support\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n\n        lr = self.lr\n\n        completed_updates = K.cast(K.tf.floor(self.iterations / self.accum_iters), K.floatx())\n\n        if self.initial_decay > 0:\n            lr = lr * (1. / (1. + self.decay * completed_updates))\n\n        t = completed_updates + 1\n\n        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))\n\n        # self.iterations incremented after processing a batch\n        # batch:              1 2 3 4 5 6 7 8 9\n        # self.iterations:    0 1 2 3 4 5 6 7 8\n        # update_switch = 1:        x       x    (if accum_iters=4)\n        update_switch = K.equal((self.iterations + 1) % self.accum_iters, 0)\n        update_switch = K.cast(update_switch, K.floatx())\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n        else:\n            vhats = [K.zeros(1) for _ in params]\n\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):\n\n            sum_grad = tg + g\n            avg_grad = sum_grad / self.accum_iters_float\n\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * avg_grad\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(avg_grad)\n\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)\n                self.updates.append(K.update(vhat, (1 - update_switch) * vhat + update_switch * vhat_t))\n            else:\n                p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n            self.updates.append(K.update(m, (1 - update_switch) * m + update_switch * m_t))\n            self.updates.append(K.update(v, (1 - update_switch) * v + update_switch * v_t))\n            self.updates.append(K.update(tg, (1 - update_switch) * sum_grad))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, (1 - update_switch) * p + update_switch * new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {'lr': float(K.get_value(self.lr)),\n                  'beta_1': float(K.get_value(self.beta_1)),\n                  'beta_2': float(K.get_value(self.beta_2)),\n                  'decay': float(K.get_value(self.decay)),\n                  'epsilon': self.epsilon,\n                  'amsgrad': self.amsgrad}\n        base_config = super(AdamAccumulate, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:48:02.635964Z","iopub.execute_input":"2024-12-31T11:48:02.636671Z","iopub.status.idle":"2024-12-31T11:48:02.724347Z","shell.execute_reply.started":"2024-12-31T11:48:02.636571Z","shell.execute_reply":"2024-12-31T11:48:02.723164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split\n\n# Ensure NUM_CLASSES matches the number of unique labels\nNUM_CLASSES = len(np.unique(y))\n\n# Debug: Print unique values in y\nprint(\"Unique labels in y:\", np.unique(y))\n\n# Convert labels to categorical\ny = to_categorical(y, num_classes=NUM_CLASSES)\n\n# Stratified split into training and validation sets\ntrain_x, valid_x, train_y, valid_y = train_test_split(x, y, test_size=0.15,\n                                                      stratify=y.argmax(axis=1), random_state=8)\n\n# Print shapes of the splits\nprint(\"train_x.shape:\", train_x.shape)\nprint(\"train_y.shape:\", train_y.shape)\nprint(\"valid_x.shape:\", valid_x.shape)\nprint(\"valid_y.shape:\", valid_y.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:48:02.726056Z","iopub.execute_input":"2024-12-31T11:48:02.726377Z","iopub.status.idle":"2024-12-31T11:48:02.747437Z","shell.execute_reply.started":"2024-12-31T11:48:02.726315Z","shell.execute_reply":"2024-12-31T11:48:02.746585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# https://github.com/aleju/imgaug\nsometimes = lambda aug: iaa.Sometimes(0.5, aug)\nseq = iaa.Sequential([\n    sometimes(\n        iaa.OneOf([\n            iaa.Add((-10, 10), per_channel=0.5),\n            iaa.Multiply((0.9, 1.1), per_channel=0.5),\n            iaa.ContrastNormalization((0.9, 1.1), per_channel=0.5)\n        ])\n    ),\n    iaa.Fliplr(0.5),\n    iaa.Crop(percent=(0, 0.1)),\n    iaa.Flipud(0.5)\n],random_order=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:48:02.749137Z","iopub.execute_input":"2024-12-31T11:48:02.749618Z","iopub.status.idle":"2024-12-31T11:48:02.756750Z","shell.execute_reply.started":"2024-12-31T11:48:02.749556Z","shell.execute_reply":"2024-12-31T11:48:02.755814Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class My_Generator(Sequence):\n\n    def __init__(self, image_filenames, labels,\n                 batch_size, is_train=False,\n                 mix=False, augment=False):\n        self.image_filenames, self.labels = image_filenames, labels\n        self.batch_size = batch_size\n        self.is_train = is_train\n        self.is_augment = augment\n        if(self.is_train):\n            self.on_epoch_end()\n        self.is_mix = mix\n\n    def __len__(self):\n        return int(np.ceil(len(self.image_filenames) / float(self.batch_size)))\n\n    def __getitem__(self, idx):\n        batch_x = self.image_filenames[idx * self.batch_size:(idx + 1) * self.batch_size]\n        batch_y = self.labels[idx * self.batch_size:(idx + 1) * self.batch_size]\n\n        if(self.is_train):\n            return self.train_generate(batch_x, batch_y)\n        return self.valid_generate(batch_x, batch_y)\n\n    def on_epoch_end(self):\n        if(self.is_train):\n            self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)\n    \n    def mix_up(self, x, y):\n        lam = np.random.beta(0.2, 0.4)\n        ori_index = np.arange(int(len(x)))\n        index_array = np.arange(int(len(x)))\n        np.random.shuffle(index_array)        \n        \n        mixed_x = lam * x[ori_index] + (1 - lam) * x[index_array]\n        mixed_y = lam * y[ori_index] + (1 - lam) * y[index_array]\n        \n        return mixed_x, mixed_y\n\n    def train_generate(self, batch_x, batch_y):\n        batch_images = []\n        for (sample, label) in zip(batch_x, batch_y):\n            img = cv2.imread('/kaggle/input/aptos19-clahe/Aptos19_Clahe_onlyDR_aug/Aptos19_Clahe_onlyDR_aug/All/train/'+sample)\n            img = cv2.resize(img, (SIZE, SIZE))\n            if(self.is_augment):\n                img = seq.augment_image(img)\n            batch_images.append(img)\n        batch_images = np.array(batch_images, np.float32) / 255\n        batch_y = np.array(batch_y, np.float32)\n        if(self.is_mix):\n            batch_images, batch_y = self.mix_up(batch_images, batch_y)\n        return batch_images, batch_y\n\n    def valid_generate(self, batch_x, batch_y):\n        batch_images = []\n        for (sample, label) in zip(batch_x, batch_y):\n            img = cv2.imread('/kaggle/input/aptos19-clahe/Aptos19_Clahe_onlyDR_aug/Aptos19_Clahe_onlyDR_aug/All/train/'+sample)\n            img = cv2.resize(img, (SIZE, SIZE))\n            batch_images.append(img)\n        batch_images = np.array(batch_images, np.float32) / 255\n        batch_y = np.array(batch_y, np.float32)\n        return batch_images, batch_y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:53:00.647613Z","iopub.execute_input":"2024-12-31T11:53:00.647897Z","iopub.status.idle":"2024-12-31T11:53:00.663396Z","shell.execute_reply.started":"2024-12-31T11:53:00.647853Z","shell.execute_reply":"2024-12-31T11:53:00.662263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential, load_model\nfrom keras.layers import (Activation, Dropout, Flatten, Dense, GlobalMaxPooling2D,\n                          BatchNormalization, Input, Conv2D, GlobalAveragePooling2D)\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.callbacks import ModelCheckpoint\nfrom keras import metrics\nfrom keras.optimizers import Adam \nfrom keras import backend as K\nimport keras\nfrom keras.models import Model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:53:04.515003Z","iopub.execute_input":"2024-12-31T11:53:04.515318Z","iopub.status.idle":"2024-12-31T11:53:04.520842Z","shell.execute_reply.started":"2024-12-31T11:53:04.515238Z","shell.execute_reply":"2024-12-31T11:53:04.519842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"function = \"softmax\"\ndef create_model(input_shape, n_out):\n    input_tensor = Input(shape=input_shape)\n    base_model = ResNet50(include_top=False,\n                   weights=None,\n                   input_tensor=input_tensor)\n    base_model.load_weights('../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')\n    x = GlobalAveragePooling2D()(base_model.output)\n    x = Dropout(0.5)(x)\n    x = Dense(1024, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    final_output = Dense(n_out, activation=function, name='final_output')(x)\n    model = Model(input_tensor, final_output)\n    \n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:53:06.790620Z","iopub.execute_input":"2024-12-31T11:53:06.790887Z","iopub.status.idle":"2024-12-31T11:53:06.797149Z","shell.execute_reply.started":"2024-12-31T11:53:06.790847Z","shell.execute_reply":"2024-12-31T11:53:06.796173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# create callbacks list\nfrom keras.callbacks import (ModelCheckpoint, LearningRateScheduler,\n                             EarlyStopping, ReduceLROnPlateau,CSVLogger)\n\nepochs = 30; batch_size = 32\ncheckpoint = ModelCheckpoint('../working/Resnet50.h5', monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4, \n                                   verbose=1, mode='min', epsilon=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=9)\n\ncsv_logger = CSVLogger(filename='../working/training_log.csv',\n                       separator=',',\n                       append=True)\n# callbacks_list = [checkpoint, csv_logger, reduceLROnPlat, early]\n\ntrain_generator = My_Generator(train_x, train_y, 128, is_train=True)\ntrain_mixup = My_Generator(train_x, train_y, batch_size, is_train=True, mix=False, augment=True)\nvalid_generator = My_Generator(valid_x, valid_y, batch_size, is_train=False)\n\nmodel = create_model(\n    input_shape=(SIZE,SIZE,3), \n    n_out=NUM_CLASSES)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:53:12.309207Z","iopub.execute_input":"2024-12-31T11:53:12.309524Z","iopub.status.idle":"2024-12-31T11:53:23.428337Z","shell.execute_reply.started":"2024-12-31T11:53:12.309479Z","shell.execute_reply":"2024-12-31T11:53:23.427222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras.callbacks import Callback\nclass QWKEvaluation(Callback):\n    def __init__(self, validation_data=(), batch_size=64, interval=1):\n        super(Callback, self).__init__()\n\n        self.interval = interval\n        self.batch_size = batch_size\n        self.valid_generator, self.y_val = validation_data\n        self.history = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        if epoch % self.interval == 0:\n            y_pred = self.model.predict_generator(generator=self.valid_generator,\n                                                  steps=np.ceil(float(len(self.y_val)) / float(self.batch_size)),\n                                                  workers=1, use_multiprocessing=True,\n                                                  verbose=1)\n            def flatten(y):\n                return np.argmax(y, axis=1).reshape(-1)\n                # return np.sum(y.astype(int), axis=1) - 1\n            \n            score = cohen_kappa_score(flatten(self.y_val),\n                                      flatten(y_pred),\n                                      labels=[0,1,2,3],\n                                      weights='quadratic')\n#             print(flatten(self.y_val)[:5])\n#             print(flatten(y_pred)[:5])\n            print(\"\\n epoch: %d - QWK_score: %.6f \\n\" % (epoch+1, score))\n            self.history.append(score)\n            if score >= max(self.history):\n                print('save checkpoint: ', score)\n                self.model.save('../working/Resnet50_bestqwk.h5')\n\nqwk = QWKEvaluation(validation_data=(valid_generator, valid_y),\n                    batch_size=batch_size, interval=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:53:23.429914Z","iopub.execute_input":"2024-12-31T11:53:23.430151Z","iopub.status.idle":"2024-12-31T11:53:23.439298Z","shell.execute_reply.started":"2024-12-31T11:53:23.430103Z","shell.execute_reply":"2024-12-31T11:53:23.438598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# warm up model\nfor layer in model.layers:\n    layer.trainable = False\n\nfor i in range(-5,0):\n    model.layers[i].trainable = True\n\nmodel.compile(\n    loss='categorical_crossentropy',\n    # loss='binary_crossentropy',\n    optimizer=Adam(1e-3))\n\nmodel.fit_generator(\n    train_generator,\n    steps_per_epoch=np.ceil(float(len(train_y)) / float(128)),\n    epochs=2,\n    workers=WORKERS, use_multiprocessing=True,\n    verbose=1,\n    callbacks=[qwk])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:53:23.440397Z","iopub.execute_input":"2024-12-31T11:53:23.440595Z","iopub.status.idle":"2024-12-31T11:54:20.599839Z","shell.execute_reply.started":"2024-12-31T11:53:23.440561Z","shell.execute_reply":"2024-12-31T11:54:20.598189Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train all layers\nfor layer in model.layers:\n    layer.trainable = True\n\ncallbacks_list = [checkpoint, csv_logger, reduceLROnPlat, early, qwk]\nmodel.compile(loss='categorical_crossentropy',\n            # loss=kappa_loss,\n            # loss='binary_crossentropy',\n            # optimizer=Adam(lr=1e-4),\n            optimizer=AdamAccumulate(lr=1e-4, accum_iters=2),\n            metrics=['accuracy'])\n\nmodel.fit_generator(\n    train_mixup,\n    steps_per_epoch=np.ceil(float(len(train_x)) / float(batch_size)),\n    validation_data=valid_generator,\n    validation_steps=np.ceil(float(len(valid_x)) / float(batch_size)),\n    epochs=epochs,\n    verbose=1,\n    workers=1, use_multiprocessing=False,\n    callbacks=callbacks_list)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:54:27.698069Z","iopub.execute_input":"2024-12-31T11:54:27.698413Z","iopub.status.idle":"2024-12-31T12:08:11.728690Z","shell.execute_reply.started":"2024-12-31T11:54:27.698350Z","shell.execute_reply":"2024-12-31T12:08:11.727553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n\n# Load the CSV file\nfile_path = '/kaggle/working/training_log.csv'\ndata = pd.read_csv(file_path)\n\n# Plot training and validation accuracy\nplt.figure(figsize=(12, 6))\n\n# Training and Validation Accuracy\nplt.subplot(1, 2, 1)\nplt.plot(data['epoch'], data['acc'], label='Training Accuracy')\nplt.plot(data['epoch'], data['val_acc'], label='Validation Accuracy')\nplt.title('Training and Validation Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\n\n# Training and Validation Loss\nplt.subplot(1, 2, 2)\nplt.plot(data['epoch'], data['loss'], label='Training Loss')\nplt.plot(data['epoch'], data['val_loss'], label='Validation Loss')\nplt.title('Training and Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T12:31:58.596484Z","iopub.execute_input":"2024-12-31T12:31:58.596757Z","iopub.status.idle":"2024-12-31T12:31:59.145662Z","shell.execute_reply.started":"2024-12-31T12:31:58.596718Z","shell.execute_reply":"2024-12-31T12:31:59.144611Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\n\n# Source and destination paths\nsource_csv = \"/kaggle/working/test_labels.csv\"\ndestination_csv = \"/kaggle/working/test_predicted_labels.csv\"\n\n# Copy the CSV file\nshutil.copy(source_csv, destination_csv)\nprint(f\"CSV file copied from {source_csv} to {destination_csv}.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T12:16:29.844158Z","iopub.execute_input":"2024-12-31T12:16:29.844500Z","iopub.status.idle":"2024-12-31T12:16:29.849988Z","shell.execute_reply.started":"2024-12-31T12:16:29.844435Z","shell.execute_reply":"2024-12-31T12:16:29.849097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit = pd.read_csv('/kaggle/working/test_predicted_labels.csv')\n# model.load_weights('../working/Resnet50.h5')\nmodel.load_weights('../working/Resnet50_bestqwk.h5')\npredicted = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T12:17:11.281233Z","iopub.execute_input":"2024-12-31T12:17:11.281589Z","iopub.status.idle":"2024-12-31T12:17:14.125062Z","shell.execute_reply.started":"2024-12-31T12:17:11.281537Z","shell.execute_reply":"2024-12-31T12:17:14.124438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i, name in tqdm(enumerate(submit['img_name'])):\n    path = os.path.join('/kaggle/input/aptos19-clahe/Aptos19_Clahe_onlyDR_aug/Aptos19_Clahe_onlyDR_aug/All/test/',name)\n    image = cv2.imread(path)\n    image = cv2.resize(image, (SIZE, SIZE))\n    score_predict = model.predict((image[np.newaxis])/255)\n    label_predict = np.argmax(score_predict)\n    # label_predict = score_predict.astype(int).sum() - 1\n    predicted.append(str(label_predict))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T12:17:48.369412Z","iopub.execute_input":"2024-12-31T12:17:48.369679Z","iopub.status.idle":"2024-12-31T12:18:01.446816Z","shell.execute_reply.started":"2024-12-31T12:17:48.369642Z","shell.execute_reply":"2024-12-31T12:18:01.445996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit['diagnosis'] = predicted\nsubmit.to_csv('/kaggle/working/test_predicted_labels.csv', index=False)\nsubmit.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T12:23:50.626356Z","iopub.execute_input":"2024-12-31T12:23:50.626639Z","iopub.status.idle":"2024-12-31T12:23:50.643999Z","shell.execute_reply.started":"2024-12-31T12:23:50.626601Z","shell.execute_reply":"2024-12-31T12:23:50.643344Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.metrics import confusion_matrix\nimport matplotlib.pyplot as plt\n\n# Load the CSV file\ncsv_file = \"/kaggle/working/test_predicted_labels.csv\"  # Replace with your file path\ndata = pd.read_csv(csv_file)\n\n# Extract true labels and predictions\ntrue_labels = data['label']\npredicted_labels = data['diagnosis']\n\n# Define class names\nclass_names = [\"Mild\", \"Moderate\", \"Severe\", \"Proliferative\"]\n\n# Compute the confusion matrix\ncm = confusion_matrix(true_labels, predicted_labels, labels=[0, 1, 2, 3])\n\n# Plot the confusion matrix\nplt.figure(figsize=(8, 6))\nplt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)\nplt.title(\"Confusion Matrix\")\nplt.colorbar()\n\n# Add class labels to axes\ntick_marks = np.arange(len(class_names))\nplt.xticks(tick_marks, class_names, rotation=45)\nplt.yticks(tick_marks, class_names)\n\n# Add values to the confusion matrix\nthresh = cm.max() / 2\nfor i, j in np.ndindex(cm.shape):\n    plt.text(j, i, f\"{cm[i, j]}\", horizontalalignment=\"center\",\n             color=\"white\" if cm[i, j] > thresh else \"black\")\n\n# Add labels and title\nplt.ylabel(\"True Label\")\nplt.xlabel(\"Predicted Label\")\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T12:23:57.953792Z","iopub.execute_input":"2024-12-31T12:23:57.954099Z","iopub.status.idle":"2024-12-31T12:23:58.363776Z","shell.execute_reply.started":"2024-12-31T12:23:57.954043Z","shell.execute_reply":"2024-12-31T12:23:58.362604Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.metrics import accuracy_score\n\n# Load the CSV file\ncsv_file = \"/kaggle/working/test_predicted_labels.csv\"  # Replace with your file path\ndata = pd.read_csv(csv_file)\n\n# Extract true labels and predictions\ntrue_labels = data['label']\npredicted_labels = data['diagnosis']\n\n# Calculate accuracy\naccuracy = accuracy_score(true_labels, predicted_labels)\nprint(f\"Test Accuracy: {accuracy * 100:.2f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T12:25:15.291120Z","iopub.execute_input":"2024-12-31T12:25:15.291394Z","iopub.status.idle":"2024-12-31T12:25:15.299939Z","shell.execute_reply.started":"2024-12-31T12:25:15.291354Z","shell.execute_reply":"2024-12-31T12:25:15.299098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Assuming `history` is the object returned by model.fit or model.fit_generator\n# Example: history = model.fit(...)\n\n# Extract validation accuracy and loss\nval_accuracy = history.history['val_accuracy']  # Replace 'val_accuracy' with 'accuracy' if using older Keras\nval_loss = history.history['val_loss']\n\n# Extract training accuracy and loss for comparison (optional)\ntrain_accuracy = history.history['accuracy']\ntrain_loss = history.history['loss']\n\n# Plot Validation and Training Accuracy\nplt.figure(figsize=(12, 5))\n\nplt.subplot(1, 2, 1)\nplt.plot(train_accuracy, label='Training Accuracy')\nplt.plot(val_accuracy, label='Validation Accuracy')\nplt.title('Training and Validation Accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\n\n# Plot Validation and Training Loss\nplt.subplot(1, 2, 2)\nplt.plot(train_loss, label='Training Loss')\nplt.plot(val_loss, label='Validation Loss')\nplt.title('Training and Validation Loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}