{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":556726,"sourceType":"datasetVersion","datasetId":267272},{"sourceId":8270786,"sourceType":"datasetVersion","datasetId":4910516}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport cv2\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport shutil\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\n\n# Set seeds to make the experiment more reproducible.\ndef seed_everything(seed=0):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    \nseed = 0\nseed_everything(seed)\n\n%matplotlib inline\nsns.set(style=\"whitegrid\")\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-30T09:42:58.454264Z","iopub.execute_input":"2024-04-30T09:42:58.45464Z","iopub.status.idle":"2024-04-30T09:42:58.467742Z","shell.execute_reply.started":"2024-04-30T09:42:58.454611Z","shell.execute_reply":"2024-04-30T09:42:58.466851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold_set = pd.read_csv('/kaggle/input/aptos-data-split/5-fold.csv')\nX_train = fold_set[fold_set['fold_0'] == 'train']\nX_val = fold_set[fold_set['fold_0'] == 'validation']\ntest = pd.read_csv('/kaggle/input/aptos2019-blindness-detection/test.csv')\nprint('Number of train samples: ', X_train.shape[0])\nprint('Number of validation samples: ', X_val.shape[0])\nprint('Number of test samples: ', test.shape[0])\n\n# Preprocecss data\nX_train[\"id_code\"] = X_train[\"id_code\"].apply(lambda x: x + \".png\")\nX_val[\"id_code\"] = X_val[\"id_code\"].apply(lambda x: x + \".png\")\ntest[\"id_code\"] = test[\"id_code\"].apply(lambda x: x + \".png\")\ndisplay(X_train.head())","metadata":{"execution":{"iopub.status.busy":"2024-04-30T09:42:58.46946Z","iopub.execute_input":"2024-04-30T09:42:58.469775Z","iopub.status.idle":"2024-04-30T09:42:58.510863Z","shell.execute_reply.started":"2024-04-30T09:42:58.469751Z","shell.execute_reply":"2024-04-30T09:42:58.510023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model parameters\nFACTOR = 4\nBATCH_SIZE = 8 * FACTOR\nEPOCHS = 20\nWARMUP_EPOCHS = 5\nLEARNING_RATE = 1e-4 * FACTOR\nWARMUP_LEARNING_RATE = 1e-3 * FACTOR\nHEIGHT = 224\nWIDTH = 224\nCHANNELS = 3\nES_PATIENCE = 5\nRLROP_PATIENCE = 3\nDECAY_DROP = 0.5\nLR_WARMUP_EPOCHS_1st = 2\nLR_WARMUP_EPOCHS_2nd = 5\nSTEP_SIZE = len(X_train) // BATCH_SIZE\nTOTAL_STEPS_1st = WARMUP_EPOCHS * STEP_SIZE\nTOTAL_STEPS_2nd = EPOCHS * STEP_SIZE\nWARMUP_STEPS_1st = LR_WARMUP_EPOCHS_1st * STEP_SIZE\nWARMUP_STEPS_2nd = LR_WARMUP_EPOCHS_2nd * STEP_SIZE","metadata":{"execution":{"iopub.status.busy":"2024-04-30T09:42:58.512151Z","iopub.execute_input":"2024-04-30T09:42:58.512532Z","iopub.status.idle":"2024-04-30T09:42:58.519184Z","shell.execute_reply.started":"2024-04-30T09:42:58.512482Z","shell.execute_reply":"2024-04-30T09:42:58.518293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_base_path = '/kaggle/input/aptos2019-blindness-detection/train_images/'\ntest_base_path = '/kaggle/input/aptos2019-blindness-detection/test_images/'\ntrain_dest_path = 'base_dir/train_images/'\nvalidation_dest_path = 'base_dir/validation_images/'\ntest_dest_path =  'base_dir/test_images/'\n\n# Making sure directories don't exist\nif os.path.exists(train_dest_path):\n    shutil.rmtree(train_dest_path)\nif os.path.exists(validation_dest_path):\n    shutil.rmtree(validation_dest_path)\nif os.path.exists(test_dest_path):\n    shutil.rmtree(test_dest_path)\n    \n# Creating train, validation and test directories\nos.makedirs(train_dest_path)\nos.makedirs(validation_dest_path)\nos.makedirs(test_dest_path)\n\ndef crop_image(img, tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([img1,img2,img3],axis=-1)\n            \n        return img\n\ndef circle_crop(img):\n    img = crop_image(img)\n\n    height, width, depth = img.shape\n    largest_side = np.max((height, width))\n    img = cv2.resize(img, (largest_side, largest_side))\n\n    height, width, depth = img.shape\n\n    x = width//2\n    y = height//2\n    r = np.amin((x, y))\n\n    circle_img = np.zeros((height, width), np.uint8)\n    cv2.circle(circle_img, (x, y), int(r), 1, thickness=-1)\n    img = cv2.bitwise_and(img, img, mask=circle_img)\n    img = crop_image(img)\n\n    return img\n    \ndef preprocess_image(base_path, save_path, image_id, HEIGHT, WIDTH, sigmaX=10):\n    image = cv2.imread(base_path + image_id)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = circle_crop(image)\n    image = cv2.resize(image, (HEIGHT, WIDTH))\n    image = cv2.addWeighted(image, 4, cv2.GaussianBlur(image, (0,0), sigmaX), -4 , 128)\n    cv2.imwrite(save_path + image_id, image)\n    \n# Pre-procecss train set\nfor i, image_id in enumerate(X_train['id_code']):\n    preprocess_image(train_base_path, train_dest_path, image_id, HEIGHT, WIDTH)\n    \n# Pre-procecss validation set\nfor i, image_id in enumerate(X_val['id_code']):\n    preprocess_image(train_base_path, validation_dest_path, image_id, HEIGHT, WIDTH)\n    \n# Pre-procecss test set\nfor i, image_id in enumerate(test['id_code']):\n    preprocess_image(test_base_path, test_dest_path, image_id, HEIGHT, WIDTH)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T09:42:58.520889Z","iopub.execute_input":"2024-04-30T09:42:58.52118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen=ImageDataGenerator(rescale=1./255, \n                           rotation_range=360,\n                           horizontal_flip=True,\n                           vertical_flip=True)\n\ntrain_generator=datagen.flow_from_dataframe(\n                        dataframe=X_train,\n                        directory=train_dest_path,\n                        x_col=\"id_code\",\n                        y_col=\"diagnosis\",\n                        class_mode=\"raw\",\n                        batch_size=BATCH_SIZE,\n                        target_size=(HEIGHT, WIDTH),\n                        seed=seed)\n\nvalid_generator=datagen.flow_from_dataframe(\n                        dataframe=X_val,\n                        directory=validation_dest_path,\n                        x_col=\"id_code\",\n                        y_col=\"diagnosis\",\n                        class_mode=\"raw\",\n                        batch_size=BATCH_SIZE,\n                        target_size=(HEIGHT, WIDTH),\n                        seed=seed)\n\ntest_generator=datagen.flow_from_dataframe(  \n                       dataframe=test,\n                       directory=test_dest_path,\n                       x_col=\"id_code\",\n                       batch_size=1,\n                       class_mode=None,\n                       shuffle=False,\n                       target_size=(HEIGHT, WIDTH),\n                       seed=seed)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cosine_decay_with_warmup(global_step,\n                             learning_rate_base,\n                             total_steps,\n                             warmup_learning_rate=0.0,\n                             warmup_steps=0,\n                             hold_base_rate_steps=0):\n    \"\"\"\n    Cosine decay schedule with warm up period.\n    In this schedule, the learning rate grows linearly from warmup_learning_rate\n    to learning_rate_base for warmup_steps, then transitions to a cosine decay\n    schedule.\n    :param global_step {int}: global step.\n    :param learning_rate_base {float}: base learning rate.\n    :param total_steps {int}: total number of training steps.\n    :param warmup_learning_rate {float}: initial learning rate for warm up. (default: {0.0}).\n    :param warmup_steps {int}: number of warmup steps. (default: {0}).\n    :param hold_base_rate_steps {int}: Optional number of steps to hold base learning rate before decaying. (default: {0}).\n    :param global_step {int}: global step.\n    :Returns : a float representing learning rate.\n    :Raises ValueError: if warmup_learning_rate is larger than learning_rate_base, or if warmup_steps is larger than total_steps.\n    \"\"\"\n\n    if total_steps < warmup_steps:\n        raise ValueError('total_steps must be larger or equal to warmup_steps.')\n    learning_rate = 0.5 * learning_rate_base * (1 + np.cos(\n        np.pi *\n        (global_step - warmup_steps - hold_base_rate_steps\n         ) / float(total_steps - warmup_steps - hold_base_rate_steps)))\n    if hold_base_rate_steps > 0:\n        learning_rate = np.where(global_step > warmup_steps + hold_base_rate_steps,\n                                 learning_rate, learning_rate_base)\n    if warmup_steps > 0:\n        if learning_rate_base < warmup_learning_rate:\n            raise ValueError('learning_rate_base must be larger or equal to warmup_learning_rate.')\n        slope = (learning_rate_base - warmup_learning_rate) / warmup_steps\n        warmup_rate = slope * global_step + warmup_learning_rate\n        learning_rate = np.where(global_step < warmup_steps, warmup_rate,\n                                 learning_rate)\n    return np.where(global_step > total_steps, 0.0, learning_rate)\n\n\nclass WarmUpCosineDecayScheduler(Callback):\n    \"\"\"Cosine decay with warmup learning rate scheduler\"\"\"\n\n    def __init__(self,\n                 learning_rate_base,\n                 total_steps,\n                 global_step_init=0,\n                 warmup_learning_rate=0.0,\n                 warmup_steps=0,\n                 hold_base_rate_steps=0,\n                 verbose=0):\n        \"\"\"\n        Constructor for cosine decay with warmup learning rate scheduler.\n        :param learning_rate_base {float}: base learning rate.\n        :param total_steps {int}: total number of training steps.\n        :param global_step_init {int}: initial global step, e.g. from previous checkpoint.\n        :param warmup_learning_rate {float}: initial learning rate for warm up. (default: {0.0}).\n        :param warmup_steps {int}: number of warmup steps. (default: {0}).\n        :param hold_base_rate_steps {int}: Optional number of steps to hold base learning rate before decaying. (default: {0}).\n        :param verbose {int}: quiet, 1: update messages. (default: {0}).\n        \"\"\"\n\n        super(WarmUpCosineDecayScheduler, self).__init__()\n        self.learning_rate_base = learning_rate_base\n        self.total_steps = total_steps\n        self.global_step = global_step_init\n        self.warmup_learning_rate = warmup_learning_rate\n        self.warmup_steps = warmup_steps\n        self.hold_base_rate_steps = hold_base_rate_steps\n        self.verbose = verbose\n        self.learning_rates = []\n\n    def on_batch_end(self, batch, logs=None):\n        self.global_step = self.global_step + 1\n        lr = K.get_value(self.model.optimizer.lr)\n        self.learning_rates.append(lr)\n\n    def on_batch_begin(self, batch, logs=None):\n        lr = cosine_decay_with_warmup(global_step=self.global_step,\n                                      learning_rate_base=self.learning_rate_base,\n                                      total_steps=self.total_steps,\n                                      warmup_learning_rate=self.warmup_learning_rate,\n                                      warmup_steps=self.warmup_steps,\n                                      hold_base_rate_steps=self.hold_base_rate_steps)\n        K.set_value(self.model.optimizer.lr, lr)\n        if self.verbose > 0:\n            print('\\nBatch %02d: setting learning rate to %s.' % (self.global_step + 1, lr))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model(input_shape):\n    input_tensor = Input(shape=input_shape)\n    base_model = EfficientNetB5(weights=None, \n                                include_top=False,\n                                input_tensor=input_tensor)\n    base_model.load_weights('../input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5')\n\n    x = GlobalAveragePooling2D()(base_model.output)\n    final_output = Dense(1, activation='linear', name='final_output')(x)\n    model = Model(input_tensor, final_output)\n    \n    return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_model(input_shape=(HEIGHT, WIDTH, CHANNELS))\n\nfor layer in model.layers:\n    layer.trainable = False\n\nfor i in range(-2, 0):\n    model.layers[i].trainable = True\n\ncosine_lr_1st = WarmUpCosineDecayScheduler(learning_rate_base=WARMUP_LEARNING_RATE,\n                                           total_steps=TOTAL_STEPS_1st,\n                                           warmup_learning_rate=0.0,\n                                           warmup_steps=WARMUP_STEPS_1st,\n                                           hold_base_rate_steps=(2 * STEP_SIZE))\n\nmetric_list = [\"accuracy\"]\ncallback_list = [cosine_lr_1st]\noptimizer = optimizers.Adam(lr=WARMUP_LEARNING_RATE)\nmodel.compile(optimizer=optimizer, loss='mean_squared_error', metrics=metric_list)\nmodel.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(2, 1, sharex='col', figsize=(20, 6))\n\nax1.plot(cosine_lr_1st.learning_rates)\nax1.set_title('Warm up learning rates')\n\nax2.plot(cosine_lr_2nd.learning_rates)\nax2.set_title('Fine-tune learning rates')\n\nplt.xlabel('Steps')\nplt.ylabel('Learning rate')\nsns.despine()\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(2, 1, sharex='col', figsize=(20, 14))\n\nax1.plot(history['loss'], label='Train loss')\nax1.plot(history['val_loss'], label='Validation loss')\nax1.legend(loc='best')\nax1.set_title('Loss')\n\nax2.plot(history['acc'], label='Train accuracy')\nax2.plot(history['val_acc'], label='Validation accuracy')\nax2.legend(loc='best')\nax2.set_title('Accuracy')\n\nplt.xlabel('Epochs')\nsns.despine()\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create empty arays to keep the predictions and labels\ndf_preds = pd.DataFrame(columns=['label', 'pred', 'set'])\ntrain_generator.reset()\nvalid_generator.reset()\n\n# Add train predictions and labels\nfor i in range(STEP_SIZE_TRAIN + 1):\n    im, lbl = next(train_generator)\n    preds = model.predict(im, batch_size=train_generator.batch_size)\n    for index in range(len(preds)):\n        df_preds.loc[len(df_preds)] = [lbl[index], preds[index][0], 'train']\n\n# Add validation predictions and labels\nfor i in range(STEP_SIZE_VALID + 1):\n    im, lbl = next(valid_generator)\n    preds = model.predict(im, batch_size=valid_generator.batch_size)\n    for index in range(len(preds)):\n        df_preds.loc[len(df_preds)] = [lbl[index], preds[index][0], 'validation']\n\ndf_preds['label'] = df_preds['label'].astype('int')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def classify(x):\n    if x < 0.5:\n        return 0\n    elif x < 1.5:\n        return 1\n    elif x < 2.5:\n        return 2\n    elif x < 3.5:\n        return 3\n    return 4\n\n# Classify predictions\ndf_preds['predictions'] = df_preds['pred'].apply(lambda x: classify(x))\n\ntrain_preds = df_preds[df_preds['set'] == 'train']\nvalidation_preds = df_preds[df_preds['set'] == 'validation']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']\ndef plot_confusion_matrix(train, validation, labels=labels):\n    train_labels, train_preds = train\n    validation_labels, validation_preds = validation\n    fig, (ax1, ax2) = plt.subplots(1, 2, sharex='col', figsize=(24, 7))\n    train_cnf_matrix = confusion_matrix(train_labels, train_preds)\n    validation_cnf_matrix = confusion_matrix(validation_labels, validation_preds)\n\n    train_cnf_matrix_norm = train_cnf_matrix.astype('float') / train_cnf_matrix.sum(axis=1)[:, np.newaxis]\n    validation_cnf_matrix_norm = validation_cnf_matrix.astype('float') / validation_cnf_matrix.sum(axis=1)[:, np.newaxis]\n\n    train_df_cm = pd.DataFrame(train_cnf_matrix_norm, index=labels, columns=labels)\n    validation_df_cm = pd.DataFrame(validation_cnf_matrix_norm, index=labels, columns=labels)\n\n    sns.heatmap(train_df_cm, annot=True, fmt='.2f', cmap=\"Blues\",ax=ax1).set_title('Train')\n    sns.heatmap(validation_df_cm, annot=True, fmt='.2f', cmap=sns.cubehelix_palette(8),ax=ax2).set_title('Validation')\n    plt.show()\n\nplot_confusion_matrix((train_preds['label'], train_preds['predictions']), (validation_preds['label'], validation_preds['predictions']))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate_model(train, validation):\n    train_labels, train_preds = train\n    validation_labels, validation_preds = validation\n    print(\"Train        Cohen Kappa score: %.3f\" % cohen_kappa_score(train_preds, train_labels, weights='quadratic'))\n    print(\"Validation   Cohen Kappa score: %.3f\" % cohen_kappa_score(validation_preds, validation_labels, weights='quadratic'))\n    print(\"Complete set Cohen Kappa score: %.3f\" % cohen_kappa_score(np.append(train_preds, validation_preds), np.append(train_labels, validation_labels), weights='quadratic'))\n    \nevaluate_model((train_preds['label'], train_preds['predictions']), (validation_preds['label'], validation_preds['predictions']))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apply_tta(model, generator, steps=10):\n    step_size = generator.n//generator.batch_size\n    preds_tta = []\n    for i in range(steps):\n        generator.reset()\n        preds = model.predict_generator(generator, steps=step_size)\n        preds_tta.append(preds)\n\n    return np.mean(preds_tta, axis=0)\n\npreds = apply_tta(model, test_generator)\npredictions = [classify(x) for x in preds]\n\nresults = pd.DataFrame({'id_code':test['id_code'], 'diagnosis':predictions})\nresults['id_code'] = results['id_code'].map(lambda x: str(x)[:-4])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cleaning created directories\nif os.path.exists(train_dest_path):\n    shutil.rmtree(train_dest_path)\nif os.path.exists(validation_dest_path):\n    shutil.rmtree(validation_dest_path)\nif os.path.exists(test_dest_path):\n    shutil.rmtree(test_dest_path)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.subplots(sharex='col', figsize=(24, 8.7))\nsns.countplot(x=\"diagnosis\", data=results, palette=\"GnBu_d\").set_title('Test')\nsns.despine()\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results.to_csv('submission.csv', index=False)\ndisplay(results.head())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_weights('../working/effNetB5_bs32_img224_fold1.h5')","metadata":{},"execution_count":null,"outputs":[]}]}