{"cells":[{"metadata":{},"cell_type":"markdown","source":"# EfficientNet+Augmentation+Loss Function for Noisy Labels for Cassava Disease Classification using TF/Keras"},{"metadata":{},"cell_type":"markdown","source":"This notebook is copied from Francois Lemarchand Efficientnet notebook"},{"metadata":{},"cell_type":"markdown","source":"This notebook presents a full pipeline to load the data, apply advanced data augmentation, train an EfficientNet and use the model to predict over the test images. To make it possible to run within the allocated time for notebooks, this notebook will only present a single fold with a split of 85% for training and 15% for validation."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nfrom sklearn.utils import shuffle\nfrom sklearn.utils import class_weight\nfrom sklearn.preprocessing import minmax_scale\nfrom sklearn.model_selection import train_test_split, KFold\nimport random\nimport cv2\nfrom imgaug import augmenters as iaa\nimport warnings\nwarnings.filterwarnings('ignore')\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.layers import Dense, Dropout, Activation, Input, BatchNormalization,Lambda, GlobalAveragePooling2D, Concatenate, Lambda\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\nfrom tensorflow.keras.experimental import CosineDecay\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.applications import EfficientNetB5, EfficientNetB3, EfficientNetB4\nfrom tensorflow.keras.applications.efficientnet import preprocess_input\nfrom tensorflow.keras.layers.experimental.preprocessing import RandomCrop,CenterCrop, RandomRotation\nfrom tensorflow.keras.preprocessing.image import load_img\nfrom keras.preprocessing import image\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_folder = '../input/cassava-leaf-disease-classification/train_images/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = Image.open(\"../input/cassava-leaf-disease-classification/train_images/1277648239.jpg\")\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare the training and validation data generators"},{"metadata":{"trusted":true},"cell_type":"code","source":"samples_df = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")\nsamples_df = shuffle(samples_df, random_state=42)\nsamples_df[\"filepath\"] = training_folder+samples_df[\"image_id\"]\nsamples_df[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"samples_df.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As it is a multi-class classification problem (5 classes), we will one-hot encode the target variable."},{"metadata":{"trusted":true},"cell_type":"code","source":"# One_hot Encoding\ny=samples_df['label'].values\ny = to_categorical(y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 8\nimage_size = 512\ninput_shape = (image_size, image_size, 3)\ndropout_rate = 0.4\nclasses_to_predict = sorted(samples_df.label.unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(samples_df, y, random_state=42, test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_data = tf.data.Dataset.from_tensor_slices((X_train.filepath.values, y_train))\nvalidation_data = tf.data.Dataset.from_tensor_slices((X_test.filepath.values, y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image_and_label_from_path(image_path, label):\n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    return img,label\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n\ntraining_data = training_data.map(load_image_and_label_from_path, num_parallel_calls=AUTOTUNE)\nvalidation_data = validation_data.map(load_image_and_label_from_path, num_parallel_calls=AUTOTUNE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Converting into tensorflow data batches\ntraining_data_batches = training_data.shuffle(buffer_size=1000).batch(batch_size).prefetch(buffer_size=AUTOTUNE)\nvalidation_data_batches = validation_data.shuffle(buffer_size=1000).batch(batch_size).prefetch(buffer_size=AUTOTUNE)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I also prepare a special dataset that will be fed to the Normalization layer. The EfficientnetB4 provided by tf.keras includes an out-of-the-box Normalization layer fit onto the imagenet dataset. Therefore, we can pull that layer and use the adapt function to refit it to the Cassava Disease dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"adapt_data = tf.data.Dataset.from_tensor_slices(X_train.filepath.values)\ndef adapt_mode(image_path):\n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = layers.experimental.preprocessing.Rescaling(1.0 / 255)(img)\n    return img\n\nadapt_data = adapt_data.map(adapt_mode, num_parallel_calls=AUTOTUNE)\nadapt_data_batches = adapt_data.shuffle(buffer_size=1000).batch(batch_size).prefetch(buffer_size=AUTOTUNE)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Applying Data Augementation Layers"},{"metadata":{},"cell_type":"markdown","source":"The data augmentation preprocessing layers below will be used when training the model but disabled in inference mode.\n\n\n\n\n\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def brighten_img(x, max_delta=0.1):\n    x = tf.image.random_brightness(x, max_delta)\n    return x\ndef brighten(max_delta=0.1):\n    return layers.Lambda(lambda x: brighten_img(x, max_delta))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def saturate_img(x, lower=0.6, upper=1.3):\n    x = tf.image.random_saturation(x, lower, upper)\n    return x\ndef saturate(lower=0.6, upper=1.3):\n    return layers.Lambda(lambda x: saturate_img(x, lower, upper))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_augmentation_layers = tf.keras.Sequential(\n    [\n        \n        layers.experimental.preprocessing.RandomCrop(height=image_size, width=image_size),\n        layers.experimental.preprocessing.RandomFlip(\"horizontal_and_vertical\"),\n        layers.experimental.preprocessing.RandomRotation(0.25),\n        layers.experimental.preprocessing.RandomZoom((-0.2, 0)),\n        layers.experimental.preprocessing.RandomContrast((0.2,0.2)),\n        brighten(),\n        saturate(),\n        \n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualizing what our augmentation will look like on images"},{"metadata":{},"cell_type":"markdown","source":"Simply reusing some of the code from this tutorial to show what our augmentations look like. I add the image previously opened to a batch and pass it through the data augmentation layers."},{"metadata":{"trusted":true},"cell_type":"code","source":"image = Image.open(\"../input/cassava-leaf-disease-classification/train_images/1481899695.jpg\")\nplt.imshow(image)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image = tf.expand_dims(np.array(image), 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 12))\nfor i in range(16):\n  augmented_image = data_augmentation_layers(image)\n  ax = plt.subplot(4, 4, i + 1)\n  plt.imshow(augmented_image[0])\n  plt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build the model"},{"metadata":{},"cell_type":"markdown","source":"I am using an EfficientNetB4 on top of which I add some outputs layers to predict our 5 disease classes. I decided to load the imagenet pretrained weights locally to keep the internet off (part of the requirements to submit a kernel to this competition)."},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model():\n    efficientnet= EfficientNetB4(weights=\"../input/tfkeras-efficientnet-weights/efficientnetb4_notop.h5\", \n                                   include_top=False, \n                                   input_shape=input_shape, \n                                   )\n\n\n    input_layer = Input(shape = input_shape)\n    augmented = data_augmentation_layers(input_layer)\n    efficientnet = efficientnet(augmented)\n    pooling = layers.GlobalAveragePooling2D()(efficientnet)\n    dropout = layers.Dropout(dropout_rate)(pooling)\n    outputs = Dense(len(classes_to_predict), activation=\"softmax\")(dropout)\n    model = Model(inputs=input_layer, outputs=outputs)\n\n    return model\n\nmodel = create_model() # define your model normally\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The 3rd layer of the Efficientnet is the Normalization layer, which can be tuned to our new dataset instead of imagenet. Be patient on this one, it does take a bit of time as we're going through the entire training set."},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodel.get_layer('efficientnetb4').get_layer('normalization').adapt(adapt_data_batches)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Implementation of Bi Tempered Logistic Loss Function\nAs their is noisy labelling in the data,I am going to use bi tempered logistic loss.\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Tempered Softmax Activation\n\ndef log_t(u, t):\n    epsilon = 1e-7\n    \"\"\"Compute log_t for `u`.\"\"\"\n    if t == 1.0:\n        return tf.math.log(u + epsilon)\n    else:\n        return (u**(1.0 - t) - 1.0) / (1.0 - t)\n# Bi Tempered Logistic Loss\ndef bi_tempered_logistic_loss(y_pred, y_true, t1, label_smoothing=0.0):\n    \"\"\"Bi-Tempered Logistic Loss with custom gradient.\n    Args:\n    y_pred: A multi-dimensional probability tensor with last dimension `num_classes`.\n    y_true: A tensor with shape and dtype as y_pred.\n    t1: Temperature 1 (< 1.0 for boundedness).\n    label_smoothing: A float in [0, 1] for label smoothing.\n    Returns:\n    A loss tensor.\n    \"\"\"\n    y_pred = tf.cast(y_pred, tf.float32)\n    y_true = tf.cast(y_true, tf.float32)\n\n    if label_smoothing > 0.0:\n        num_classes = tf.cast(tf.shape(y_true)[-1], tf.float32)\n        y_true = (1 - num_classes /(num_classes - 1) * label_smoothing) * y_true + label_smoothing / (num_classes - 1)\n\n    temp1 = (log_t(y_true + 1e-7, t1) - log_t(y_pred, t1)) * y_true\n    temp2 = (1 / (2 - t1)) * (tf.math.pow(y_true, 2 - t1) - tf.math.pow(y_pred, 2 - t1))\n    loss_values = temp1 - temp2\n\n    return tf.math.reduce_sum(loss_values, -1)\n\nclass BiTemperedLogisticLoss(tf.keras.losses.Loss):\n    def __init__(self, t1=0.8, label_smoothing=0.2):\n        super(BiTemperedLogisticLoss, self).__init__()\n        self.t1 = t1\n        self.label_smoothing = label_smoothing\n\n    def call(self, y_true, y_pred):\n        return bi_tempered_logistic_loss(y_pred, y_true, self.t1, self.label_smoothing)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 8\ndecay_steps = int(round(len(X_train)/batch_size))*epochs\ncosine_decay = CosineDecay(initial_learning_rate=1e-5, decay_steps=decay_steps, alpha=0.3)\ncallbacks = [ModelCheckpoint(filepath='best_model.h5', monitor='val_loss', save_best_only=True)]\nloss = BiTemperedLogisticLoss()\nmodel.compile(loss=loss, optimizer=tf.keras.optimizers.Adam(cosine_decay), metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(training_data_batches,\n                  epochs = epochs, \n                  validation_data = validation_data_batches,\n                  callbacks = callbacks)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"First, we will check that we perform on similar level on both the training and validation. The training curve will also tell us if we stopped training too early or may have overfitted in comparison to the validation data."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('Accuracy over epochs')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Validation'], loc='best')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We load the best weights that were kept from the training phase. Just to check how our model is performing, we will attempt predictions over the validation set. This can help to highlight any classes that will be consistently miscategorised."},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights(\"best_model.h5\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction on test images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def scan_over_image(img_path, crop_size=512):\n    '''\n    Will extract 512x512 images covering the whole original image\n    with some overlap between images\n    '''\n    \n    img = Image.open(img_path)\n    img_height, img_width = img.size\n    img = np.array(img)\n    \n    y = random.randint(0,img_height-crop_size)\n    x = random.randint(0,img_width-crop_size)\n\n    x_img_origins = [0,img_width-crop_size]\n    y_img_origins = [0,img_height-crop_size]\n    img_list = []\n    for x in x_img_origins:\n        for y in y_img_origins:\n            img_list.append(img[x:x+crop_size , y:y+crop_size,:])\n  \n    return np.array(img_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_samples(img_path):\n    '''\n    Display all 512x512 images extracted from original images\n    '''\n    \n    img_list = scan_over_image(img_path)\n    sample_number = len(img_list)\n    fig = plt.figure(figsize = (8,sample_number))\n    for i in range(0,sample_number):\n        ax = fig.add_subplot(2, 4, i+1)\n        ax.imshow(img_list[i])\n        ax.set_title(str(i))\n    plt.tight_layout()\n    plt.show()\n\ndisplay_samples(\"../input/cassava-leaf-disease-classification/train_images/3412658650.jpg\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Apply Test time augmentation on the local images extracted from the folder"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_time_augmentation_layers = tf.keras.Sequential(\n    [\n        layers.experimental.preprocessing.RandomCrop(height=image_size, width=image_size),\n        layers.experimental.preprocessing.RandomFlip(\"horizontal_and_vertical\"),\n        layers.experimental.preprocessing.RandomZoom((-0.2, 0)),\n        layers.experimental.preprocessing.RandomContrast((0.2,0.2)),\n    ]\n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_and_vote(image_filename, folder, TTA_runs=4):\n    '''\n    Run the model over 4 local areas of the given image,\n    before making a decision depending on the most predicted\n    disease.\n    '''\n    \n    #apply TTA to each of the 4 images and sum all predictions for each local image\n    localised_predictions = []\n    local_image_list = scan_over_image(folder+image_filename)\n    for local_image in local_image_list:\n        local_image = tf.expand_dims(local_image,0)\n        augmented_images = [test_time_augmentation_layers(local_image) for i in range(TTA_runs)]\n        predictions = model.predict(np.array(augmented_images[0]))\n        localised_predictions.append(np.sum(predictions, axis=0))\n    \n    #sum all predictions from all 4 images and retrieve the index of the highest value\n    global_predictions = np.sum(np.array(localised_predictions),axis=0)\n    final_prediction = np.argmax(global_predictions)\n    \n    return final_prediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_predictions_over_image_list(image_list, folder):\n    predictions = []\n    with tqdm(total=len(image_list)) as pbar:\n        for image_filename in image_list:\n            pbar.update(1)\n            predictions.append(predict_and_vote(image_filename, folder))\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"First, I test my entire prediction pipeline on the validation set as we have little visibility over the test set."},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test[\"results\"] = run_predictions_over_image_list(X_test[\"image_id\"], training_folder)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cat ../input/cassava-leaf-disease-classification/label_num_to_disease_map.json","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test[:30]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"true_positives = 0\nprediction_distribution_per_class = {\"0\":{\"0\": 0, \"1\": 0, \"2\":0, \"3\":0, \"4\":0},\n                                     \"1\":{\"0\": 0, \"1\": 0, \"2\":0, \"3\":0, \"4\":0},\n                                     \"2\":{\"0\": 0, \"1\": 0, \"2\":0, \"3\":0, \"4\":0},\n                                     \"3\":{\"0\": 0, \"1\": 0, \"2\":0, \"3\":0, \"4\":0},\n                                     \"4\":{\"0\": 0, \"1\": 0, \"2\":0, \"3\":0, \"4\":0}}\nnumber_of_images = len(X_test)\nfor idx, pred in X_test.iterrows():\n    if int(pred[\"label\"]) == pred.results:\n        true_positives+=1\n    prediction_distribution_per_class[str(pred[\"label\"])][str(pred.results)] += 1\nprint(\"accuracy: {}%\".format(true_positives/number_of_images*100))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_distribution_per_class","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"heatmap_df = pd.DataFrame(columns={\"groundtruth\",\"prediction\",\"value\"})\nfor key in prediction_distribution_per_class.keys():\n    for pred_key in prediction_distribution_per_class[key].keys():\n        value = prediction_distribution_per_class[key][pred_key]/X_test.query(\"label==@key\").count()[0]\n        heatmap_df = heatmap_df.append({\"groundtruth\":key,\"prediction\":pred_key,\"value\":value}, ignore_index=True)   \n\nheatmap = heatmap_df.pivot(index='groundtruth', columns='prediction', values='value')\nsns.heatmap(heatmap,cmap=\"Blues\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_folder = '../input/cassava-leaf-disease-classification/test_images/'\nsubmission_df = pd.DataFrame(columns={\"image_id\",\"label\"})\nsubmission_df[\"image_id\"] =  os.listdir(test_folder)\nsubmission_df[\"label\"] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df[\"label\"] = run_predictions_over_image_list(submission_df[\"image_id\"], test_folder)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}