{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import models, layers\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.callbacks import Callback\nimport os, cv2, json,sys\nfrom PIL import Image\n# from keras.models import load_model\n\nfrom sklearn.model_selection import train_test_split\nimport tensorflow.keras.applications.efficientnet as efn\n\nfrom tensorflow.keras import backend as K\n# ignoring warnings\nimport warnings\nwarnings.simplefilter(\"ignore\")\n\nseed = 2021\n\ntf.random.set_seed(seed)\nnp.random.seed(seed)\nos.environ['PYTHONSEED'] =str(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For easy acces to files\nWORK_DIR = \"../input/cassava-leaf-disease-classification/\"\nos.listdir(WORK_DIR)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('../input/cassava-leaf-disease-classification/label_num_to_disease_map.json', 'r') as file:\n    labels = json.load(file)\n    \nlabels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSES = ['0', '1', '2', '3', '4']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv(WORK_DIR + \"train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#change for the ImageDatagen and flow_from_dataframe\ndata.label = data.label.astype(\"str\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 32\nIMG_SIZE = 256\nCHANNELS = 3\nimg_size = (IMG_SIZE, IMG_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,12))\ndata_sample = data.sample(9).reset_index(drop=True)\n\nfor i in range(8):\n    plt.subplot(2,4,i+1)\n    \n    img = cv2.imread(WORK_DIR + \"train_images/\" + data_sample.image_id[i])\n    img = cv2.resize(img, (IMG_SIZE, IMG_SIZE))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    plt.axis(\"off\")\n    plt.imshow(img)\n    plt.title(labels.get(data_sample.label[i]))\n    \nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Cassava Bacterial Blight (CBB)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels.get(\"0\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,12))\ndata_sample = data[data.label==\"0\"].sample(4).reset_index(drop=True)\nfor i in range(4):\n    plt.subplot(1,4,i+1)\n    \n    img = cv2.imread(WORK_DIR + \"train_images/\" + data_sample.image_id[i])\n    img = cv2.resize(img, (IMG_SIZE, IMG_SIZE))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    plt.axis(\"off\")\n    plt.imshow(img)\n    plt.title(labels.get(data_sample.label[i]))\n    \nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Cassava Brown Streak Disease (CBSD)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels.get(\"1\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,12))\ndata_sample = data[data.label==\"1\"].sample(4).reset_index(drop=True)\nfor i in range(4):\n    plt.subplot(1,4,i+1)\n    \n    img = cv2.imread(WORK_DIR + \"train_images/\" + data_sample.image_id[i])\n    img = cv2.resize(img, (IMG_SIZE, IMG_SIZE))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    plt.axis(\"off\")\n    plt.imshow(img)\n    plt.title(labels.get(data_sample.label[i]))\n    \nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Cassava Green Mottle (CGM)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels.get(\"2\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,12))\ndata_sample = data[data.label==\"2\"].sample(4).reset_index(drop=True)\nfor i in range(4):\n    plt.subplot(1,4,i+1)\n    \n    img = cv2.imread(WORK_DIR + \"train_images/\" + data_sample.image_id[i])\n    img = cv2.resize(img, (IMG_SIZE, IMG_SIZE))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    plt.axis(\"off\")\n    plt.imshow(img)\n    plt.title(labels.get(data_sample.label[i]))\n    \nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Cassava Mosaic Disease (CMD)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels.get(\"3\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,12))\ndata_sample = data[data.label==\"3\"].sample(4).reset_index(drop=True)\nfor i in range(4):\n    plt.subplot(1,4,i+1)\n    \n    img = cv2.imread(WORK_DIR + \"train_images/\" + data_sample.image_id[i])\n    img = cv2.resize(img, (IMG_SIZE, IMG_SIZE))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    plt.axis(\"off\")\n    plt.imshow(img)\n    plt.title(labels.get(data_sample.label[i]))\n    \nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Healthy**"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels.get(\"4\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,12))\ndata_sample = data[data.label==\"4\"].sample(4).reset_index(drop=True)\nfor i in range(4):\n    plt.subplot(1,4,i+1)\n    \n    img = cv2.imread(WORK_DIR + \"train_images/\" + data_sample.image_id[i])\n    img = cv2.resize(img, (IMG_SIZE, IMG_SIZE))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    plt.axis(\"off\")\n    plt.imshow(img)\n    plt.title(labels.get(data_sample.label[i]))\n    \nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Split data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train, valid = train_test_split(data, test_size = 0.1, \\\n                              random_state = seed, stratify = data['label'])\n\ntrain = train.reset_index(drop= True)\nvalid = valid.reset_index(drop= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape, valid.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator = ImageDataGenerator(\n                                    preprocessing_function = tf.keras.applications.efficientnet.preprocess_input,\n                                    rotation_range=40,\n                                    width_shift_range=0.2,\n                                    height_shift_range=0.2,\n                                    brightness_range=[0.1,0.9],\n                                    shear_range=25,\n                                    zoom_range=0.2,\n                                    horizontal_flip=True,\n                                    vertical_flip=True,\n                                    fill_mode= 'nearest'\n) \\\n        .flow_from_dataframe(\n                            train,\n                            directory = WORK_DIR + \"train_images\",\n                            x_col = \"image_id\",\n                            y_col = \"label\",\n                            target_size = img_size,\n                            class_mode = \"categorical\",\n                            interpolation = \"nearest\",\n                            batch_size = BATCH_SIZE,\n                            shuffle = True,\n                            seed = seed,    \n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_generator = ImageDataGenerator(preprocessing_function = tf.keras.applications.efficientnet.preprocess_input\n) \\\n        .flow_from_dataframe(\n                            valid,\n                            directory = WORK_DIR + \"train_images\",\n                            x_col = \"image_id\",\n                            y_col = \"label\",\n                            target_size = (260,260),\n                            class_mode = \"categorical\",\n                            batch_size = BATCH_SIZE,\n                            interpolation = \"nearest\",\n                            shuffle = False,\n                            )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_generator.class_indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass CosineAnnealingScheduler(Callback):\n    \"\"\"Cosine annealing scheduler.\n    \"\"\"\n\n    def __init__(self, T_max, eta_max, eta_min=0, verbose=0):\n        super(CosineAnnealingScheduler, self).__init__()\n        self.T_max = T_max\n        self.eta_max = eta_max\n        self.eta_min = eta_min\n        self.verbose = verbose\n\n    def on_epoch_begin(self, epoch, logs=None):\n        if not hasattr(self.model.optimizer, 'lr'):\n            raise ValueError('Optimizer must have a \"lr\" attribute.')\n        lr = self.eta_min + (self.eta_max - self.eta_min) * (1 + math.cos(math.pi * epoch / self.T_max)) / 2\n        K.set_value(self.model.optimizer.lr, lr)\n        if self.verbose > 0:\n            print('\\nEpoch %05d: CosineAnnealingScheduler setting learning '\n                  'rate to %s.' % (epoch + 1, lr))\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        logs['lr'] = K.get_value(self.model.optimizer.lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_schedule = keras.optimizers.schedules.ExponentialDecay(\n    initial_learning_rate=3e-5, \n    decay_steps=10000, \n    decay_rate=0.9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weights_path = '../input/keras-efficientnetb3-noisy-student/noisy_student_efficientnet_b4.h5'\n\ndef build_model():\n    \n    input_img = layers.Input(shape=(IMG_SIZE,IMG_SIZE,CHANNELS), name='input_image')\n    \n#     img_bn = layers.BatchNormalization()(input_img)\n    \n    x = efn.EfficientNetB4(include_top = False,\n                        weights = weights_path,\n                        drop_connect_rate=0.5\n                        )(input_img)\n    \n    avg_pool = layers.GlobalAveragePooling2D()(x)\n    \n    \n    h1 = layers.Dense(1792, activation='relu', name='h1', \\\n                     bias_regularizer = tf.keras.regularizers.L1L2(l1 = 0.01, l2 = 0.001))(avg_pool)\n      \n    h2 = layers.add([avg_pool, h1])\n    h3 = layers.add([h2, layers.Dense(1792, activation='relu',name='h3')(h2)])\n\n    drop = layers.Dropout(rate= 0.25, name ='drop')(h3)\n\n    predictions = layers.Dense(len(CLASSES), activation='softmax',kernel_regularizer= \\\n                               tf.keras.regularizers.L2(0.01))(drop)\n    \n    model = models.Model(inputs = input_img , outputs = predictions)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(build_model().summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def symmetric_cross_entropy(alpha, beta):\n    def loss(y_true, y_pred):\n        y_true_1 = y_true\n        y_pred_1 = y_pred\n\n        y_true_2 = y_true\n        y_pred_2 = y_pred\n\n        y_pred_1 = tf.clip_by_value(y_pred_1, 1e-7, 1.0)\n        y_true_2 = tf.clip_by_value(y_true_2, 1e-4, 1.0)\n\n        return alpha*tf.reduce_mean(-tf.reduce_sum(y_true_1 * tf.math.log(y_pred_1), axis = -1)) + beta*tf.reduce_mean(-tf.reduce_sum(y_pred_2 * tf.math.log(y_true_2), axis = -1))\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_fn = symmetric_cross_entropy(alpha=1.0,beta=1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# with strategy.scope():\nmodel = build_model()\n    \nmodel.compile(\n    optimizer=tf.keras.optimizers.Adam(learning_rate= lr_schedule, epsilon=0.001),\n\nloss= tf.keras.losses.CategoricalCrossentropy(\n        from_logits = False,\n        label_smoothing=0.0001,\n        name='categorical_crossentropy'\n    ),  \nmetrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import utils\n\nutils.plot_model(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_check = ModelCheckpoint(\n                            \"./best_model.h5\",\n                            monitor = \"val_loss\",\n                            verbose = 1,\n                            save_best_only = True,\n                            save_weights_only = True,\n                            mode = \"min\")\n\nearly_stop= EarlyStopping(\n                        monitor = \"val_loss\",\n                        min_delta=0.001,\n                        patience=5,\n                        verbose=1,\n                        mode=\"min\",\n                        restore_best_weights=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 20\nSTEP_SIZE_TRAIN = train_generator.n // train_generator.batch_size\nSTEP_SIZE_VAL = valid_generator.n // valid_generator.batch_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(train_generator,\n                                epochs = EPOCHS,\n                                validation_data = valid_generator,\n                                steps_per_epoch = STEP_SIZE_TRAIN,\n                                validation_steps = STEP_SIZE_VAL,\n                                callbacks = [model_check,early_stop])\n\n# model.save('leaf_disease_model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15, 5))\nplt.plot(history.history['accuracy'], 'b*-', label=\"train_acc\")\nplt.plot(history.history['val_accuracy'], 'r*-', label=\"val_acc\")\nplt.grid()\nplt.title(\"train_acc vs val_acc\")\nplt.ylabel(\"Accuracy\")\nplt.xlabel(\"Epochs\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15, 5))\nplt.plot(history.history['loss'], 'b*-', label=\"train_loss\")\nplt.plot(history.history['val_loss'], 'r*-', label=\"val_loss\")\nplt.grid()\nplt.title(\"train_loss - val_loss\")\nplt.ylabel(\"Loss\")\nplt.xlabel(\"Epochs\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = load_model('./best_model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_DIR = '../input/cassava-leaf-disease-classification/test_images/'\ntest_images = os.listdir(TEST_DIR)\npredictions = []\n\nfor image in test_images:\n    img = Image.open(TEST_DIR + image)\n    img = img.resize(img_size)\n    img = np.expand_dims(img, axis=0)\n    predictions.extend(model.predict(img).argmax(axis = 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.save('model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.save('model_weights.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.DataFrame({'image_id': test_images, 'label': predictions})\ndisplay(sub)\nsub.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}