{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.039382,"end_time":"2020-11-19T21:45:23.042097","exception":false,"start_time":"2020-11-19T21:45:23.002715","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Introduction\n**This notebook is based on  [Jesse Mostipak’s Tutorial](https://www.kaggle.com/jessemostipak/getting-started-tpus-cassava-leaf-disease)**  \nIn this notebook use fine tuning on efficientnet7.\n\nParameters optimisation are given in [this notebook](https://www.kaggle.com/pierreballesta/cassava-leaf-disease-efficientnet5-optimisation)."},{"metadata":{"papermill":{"duration":0.037375,"end_time":"2020-11-19T21:45:23.192515","exception":false,"start_time":"2020-11-19T21:45:23.15514","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Set up environment"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install '../input/tensorflow230/tensorflow_estimator-2.3.0-py2.py3-none-any.whl'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install '../input/tensorflow230/tensorflow-2.3.0rc2-cp37-cp37m-manylinux2010_x86_64.whl'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.core.display import HTML\nHTML(\"<script>Jupyter.notebook.kernel.restart()</script>\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random, re, math, os, json, gc\nimport numpy as np, pandas as pd, seaborn as sn\nimport cv2\nimport matplotlib.pyplot as plt\nimport tensorflow as tf, tensorflow.keras.backend as K\nfrom tensorflow.keras.utils import plot_model,to_categorical\nfrom tensorflow.keras.models import load_model\nfrom functools import partial\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport IPython\nprint('Tensorflow version ' + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import Counter\nimport itertools\nfrom tensorflow.keras import backend as K\n#from tensorflow.keras.applications import EfficientNetB7\nfrom IPython.display import FileLink","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we will see below, the classes are not equally represented. Therefore, we need more than just accuracy to determine the validity of our model. Hence, we use the f1 score as our metric."},{"metadata":{"trusted":true},"cell_type":"code","source":"def recall_m(y_true, y_pred):\n    true_positives = K.sum(y_true * y_pred,axis=0)\n    possible_positives = K.sum(y_true,axis=0)\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\ndef precision_m(y_true, y_pred):\n    true_positives = K.sum(y_true * y_pred,axis=0)\n    predicted_positives = K.sum(y_pred,axis=0)\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\ndef f1_m(y_true, y_pred):\n    y_pred = tf.one_hot(tf.argmax(y_pred,axis=-1),len(CLASSES))\n    precision = precision_m(y_true, y_pred)\n    recall = recall_m(y_true, y_pred)\n    return 2*K.mean((precision*recall)/(precision+recall+K.epsilon()))\n","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.038122,"end_time":"2020-11-19T21:45:34.458722","exception":false,"start_time":"2020-11-19T21:45:34.4206","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Set up variables\nWe'll set up some of our variables for our notebook here. "},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:45:34.555293Z","iopub.status.busy":"2020-11-19T21:45:34.541822Z","iopub.status.idle":"2020-11-19T21:47:59.71579Z","shell.execute_reply":"2020-11-19T21:47:59.714961Z"},"papermill":{"duration":145.219568,"end_time":"2020-11-19T21:47:59.715925","exception":false,"start_time":"2020-11-19T21:45:34.496357","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"L_BATCH = 1\nBASE_DIR = '../input/cassava-leaf-disease-classification/'\n#GCS_PATH = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification')\nBATCH_SIZE = 64\nIMAGE_EN = {'B0':224,'B1':240,'B2':260,'B3':300,'B4':380,'B5':456,'B6':528,'B7':600}\nCLASSES = ['0', '1', '2', '3', '4']\nEPOCHS = 10\nPROBA_CONTRAST=1.\nos.chdir(r'/kaggle/working')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EfficientNet architecture.  \nWe load a pretrained EfficientNet7 network."},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = [IMAGE_EN['B7'],IMAGE_EN['B7']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = load_model('../input/trained-model/EffNet7_2.h5',compile=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Three fully connected layers are added.\nplot_model(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.037843,"end_time":"2020-11-19T21:47:59.792061","exception":false,"start_time":"2020-11-19T21:47:59.754218","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Load the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(\n    tf.io.gfile.glob('../input/cassava-leaf-disease-classification/train_tfrecords/ld_train*.tfrec'),\n    test_size=0.125, random_state=5)\n\nTEST_FILENAMES = tf.io.gfile.glob('../input/cassava-leaf-disease-classification/test_tfrecords/ld_test*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(os.path.join(BASE_DIR, \"label_num_to_disease_map.json\")) as file:\n    map_classes = json.loads(file.read())\n    map_classes = {int(k) : v for k, v in map_classes.items()}\n    \nprint(json.dumps(map_classes, indent=4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_files = os.listdir(os.path.join(BASE_DIR, \"train_images\"))\nprint(f\"Number of train images: {len(input_files)}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(BASE_DIR, \"train.csv\"))\ndf_train[\"class_name\"] = df_train[\"label\"].map(map_classes)\nplt.figure(figsize=(8, 4))\nsn.countplot(y=\"class_name\", data=df_train);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Since the classes are not evenly distributed, we need to use weights to prevent metastable results."},{"metadata":{"trusted":true},"cell_type":"code","source":"Counter(df_train[\"label\"])","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.038439,"end_time":"2020-11-19T21:47:59.869037","exception":false,"start_time":"2020-11-19T21:47:59.830598","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Decode the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:47:59.952622Z","iopub.status.busy":"2020-11-19T21:47:59.951868Z","iopub.status.idle":"2020-11-19T21:47:59.954997Z","shell.execute_reply":"2020-11-19T21:47:59.955558Z"},"papermill":{"duration":0.04859,"end_time":"2020-11-19T21:47:59.955731","exception":false,"start_time":"2020-11-19T21:47:59.907141","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def decode_image(image,image_size=IMAGE_SIZE):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32)\n    image = tf.image.resize(image, image_size)\n    return image","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:48:00.123967Z","iopub.status.busy":"2020-11-19T21:48:00.123143Z","iopub.status.idle":"2020-11-19T21:48:00.126902Z","shell.execute_reply":"2020-11-19T21:48:00.126284Z"},"papermill":{"duration":0.052475,"end_time":"2020-11-19T21:48:00.127039","exception":false,"start_time":"2020-11-19T21:48:00.074564","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def read_tfrecord(example, labeled,image_size):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'],image_size=image_size)\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:48:00.325942Z","iopub.status.busy":"2020-11-19T21:48:00.324875Z","iopub.status.idle":"2020-11-19T21:48:00.327502Z","shell.execute_reply":"2020-11-19T21:48:00.328493Z"},"papermill":{"duration":0.073623,"end_time":"2020-11-19T21:48:00.328703","exception":false,"start_time":"2020-11-19T21:48:00.25508","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False,image_size=IMAGE_SIZE):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=L_BATCH) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled,image_size=image_size), num_parallel_calls=L_BATCH)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.038372,"end_time":"2020-11-19T21:48:00.765394","exception":false,"start_time":"2020-11-19T21:48:00.727022","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Adding in augmentations "},{"metadata":{"trusted":true},"cell_type":"code","source":"# One hot label and float images.\ndef data_treat(image,label):\n    label = tf.one_hot(label,len(CLASSES))\n    image = tf.cast(image, tf.float32)\n    return image,label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Values going from 0 to 255.\ndef data_treat_test(image,label):\n    image = tf.cast(image, tf.float32)\n    image = image-tf.math.reduce_min(image)\n    image = image/tf.math.reduce_max(image)\n    image = image*255\n    return image,label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Values going from 0 to 255.\ndef data_standard(image,label):\n    image = image-tf.math.reduce_min(image)\n    image = image/tf.math.reduce_max(image)\n    image = image*255\n    return image,label","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.038742,"end_time":"2020-11-19T21:48:00.930185","exception":false,"start_time":"2020-11-19T21:48:00.891443","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Define data loading methods\nThe following functions will be used to load our `training`, `validation`, and `test` datasets, as well as print out the number of images in each dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset(ordered=False,image_size=IMAGE_SIZE):\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=ordered,image_size=image_size)  \n    dataset = dataset.map(data_treat, num_parallel_calls=L_BATCH)  \n    dataset = dataset.map(data_standard,num_parallel_calls=L_BATCH)  \n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(L_BATCH)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_dataset(ordered=False,image_size=IMAGE_SIZE):\n    dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=ordered,image_size=image_size) \n    dataset = dataset.map(data_treat, num_parallel_calls=L_BATCH)  \n    dataset = dataset.map(data_standard,num_parallel_calls=L_BATCH)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(L_BATCH)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(ordered=False,image_size=IMAGE_SIZE):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered,image_size=image_size)\n    dataset = dataset.map(data_treat_test, num_parallel_calls=L_BATCH)  \n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(L_BATCH)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALID_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\n\nprint('Dataset: {} training images, {} validation images, {} (unlabeled) test images'.format(\n    NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CallBacks. "},{"metadata":{"trusted":true},"cell_type":"code","source":"# We create a callback to stop training if the validity loss does not deacrease for 4 epochs.\nearly_stop=tf.keras.callbacks.EarlyStopping(monitor='val_f1_m', min_delta=0, patience=5, verbose=0,\n    mode='max', baseline=None, restore_best_weights=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Clean every two epoch\nclass GarbCol(tf.keras.callbacks.Callback):\n    def on_epoch_end(self, epochs, logs={}):\n        if epochs%2 == 0:\n            gc.collect()\n            \ncallb = GarbCol()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def loop_schedule(l):\n    def scheduler(epoch,lr):\n      if epoch>0:\n        return lr/1.01\n      else :\n        return l\n    schedule = tf.keras.callbacks.LearningRateScheduler(scheduler)\n    return schedule","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE + 1\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE + 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Results."},{"metadata":{},"cell_type":"markdown","source":"### Distribution. "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_dataset(ordered=True)\nvalid_dataset = get_validation_dataset(ordered=True)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T22:05:05.184942Z","iopub.status.busy":"2020-11-19T22:05:05.183725Z","iopub.status.idle":"2020-11-19T22:05:05.18757Z","shell.execute_reply":"2020-11-19T22:05:05.186823Z"},"papermill":{"duration":1.270192,"end_time":"2020-11-19T22:05:05.187694","exception":false,"start_time":"2020-11-19T22:05:03.917502","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# this code will convert our test image data to a float32 \ndef to_float32(image, label):\n    return tf.cast(image, tf.float32), label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_ds = valid_dataset.map(to_float32)\nfit_valid_label = model.predict(valid_ds, steps=VALID_STEPS,batch_size=BATCH_SIZE,verbose=1)\nfit_valid_proba = np.max(fit_valid_label,axis=1)\nfit_valid_label = np.argmax(fit_valid_label,axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Confusion matrix."},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=True)\n#dataset = tf.data.TFRecordDataset(TRAINING_FILENAMES[0], num_parallel_reads=AUTOTUNE)\nvalid_labels = []\nfor images, labels in dataset.take(-1):  # only take first element of dataset\n    valid_labels.append(labels.numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                        normalize=False,\n                        title='Confusion matrix',\n                        cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n\n    print(cm)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, np.round(1000*cm[i, j])/1000,\n            horizontalalignment=\"center\",\n            color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm = confusion_matrix(valid_labels[:len(fit_valid_label)],fit_valid_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_confusion_matrix(cm, [0,1,2,3,4],normalize=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prediction of the test dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True) \ntest_ds = test_ds.map(to_float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":1.271799,"end_time":"2020-11-19T22:05:24.759257","exception":false,"start_time":"2020-11-19T22:05:23.487458","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Creating a submission file\nNow that we've trained a model and made predictions we're ready to submit to the competition! You can run the following code below to get your submission file."},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T22:05:27.316025Z","iopub.status.busy":"2020-11-19T22:05:27.315202Z","iopub.status.idle":"2020-11-19T22:05:28.241598Z","shell.execute_reply":"2020-11-19T22:05:28.24078Z"},"papermill":{"duration":2.185537,"end_time":"2020-11-19T22:05:28.241723","exception":false,"start_time":"2020-11-19T22:05:26.056186","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\n!head submission.csv","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}