{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.039382,"end_time":"2020-11-19T21:45:23.042097","exception":false,"start_time":"2020-11-19T21:45:23.002715","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Introduction\n**This notebook is based on  [Jesse Mostipak’s Tutorial](https://www.kaggle.com/jessemostipak/getting-started-tpus-cassava-leaf-disease)**  \nIn this notebook we check the importance of colours in the classification process.  \nWe study a baseline model."},{"metadata":{"papermill":{"duration":0.037375,"end_time":"2020-11-19T21:45:23.192515","exception":false,"start_time":"2020-11-19T21:45:23.15514","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Set up environment"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:45:23.274004Z","iopub.status.busy":"2020-11-19T21:45:23.273154Z","iopub.status.idle":"2020-11-19T21:45:30.119096Z","shell.execute_reply":"2020-11-19T21:45:30.119725Z"},"papermill":{"duration":6.890298,"end_time":"2020-11-19T21:45:30.119979","exception":false,"start_time":"2020-11-19T21:45:23.229681","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import math, re, os, json\nimport seaborn as sn\nimport cv2\nimport itertools\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow import keras\nimport tensorflow as tf, tensorflow.keras.backend as K\nfrom tensorflow.keras.utils import plot_model,to_categorical\nfrom tensorflow.keras.models import load_model\nfrom functools import partial\nfrom sklearn.model_selection import train_test_split\nfrom collections import Counter\nfrom sklearn.metrics import confusion_matrix\nprint(\"Tensorflow version \" + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.037444,"end_time":"2020-11-19T21:45:30.195328","exception":false,"start_time":"2020-11-19T21:45:30.157884","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# TPU\n"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:45:30.373218Z","iopub.status.busy":"2020-11-19T21:45:30.372403Z","iopub.status.idle":"2020-11-19T21:45:34.382672Z","shell.execute_reply":"2020-11-19T21:45:34.382035Z"},"papermill":{"duration":4.150374,"end_time":"2020-11-19T21:45:34.382816","exception":false,"start_time":"2020-11-19T21:45:30.232442","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.038122,"end_time":"2020-11-19T21:45:34.458722","exception":false,"start_time":"2020-11-19T21:45:34.4206","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Set up variables\nWe'll set up some of our variables for our notebook here. "},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:45:34.555293Z","iopub.status.busy":"2020-11-19T21:45:34.541822Z","iopub.status.idle":"2020-11-19T21:47:59.71579Z","shell.execute_reply":"2020-11-19T21:47:59.714961Z"},"papermill":{"duration":145.219568,"end_time":"2020-11-19T21:47:59.715925","exception":false,"start_time":"2020-11-19T21:45:34.496357","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_PATH = KaggleDatasets().get_gcs_path()\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nBASE_DIR = '../input/cassava-leaf-disease-classification/'\nIMAGE_SIZE = [512, 512]\nCLASSES = ['0', '1', '2', '3', '4']\nEPOCHS = 50\nPROBA_CONTRAST=1.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### F1 score."},{"metadata":{"trusted":true},"cell_type":"code","source":"def recall_m(y_true, y_pred):\n    true_positives = K.sum(y_true * y_pred,axis=0)\n    possible_positives = K.sum(y_true,axis=0)\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\ndef precision_m(y_true, y_pred):\n    true_positives = K.sum(y_true * y_pred,axis=0)\n    predicted_positives = K.sum(y_pred,axis=0)\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\ndef f1_m(y_true, y_pred):\n    y_pred = tf.one_hot(tf.argmax(y_pred,axis=-1),len(CLASSES))\n    precision = precision_m(y_true, y_pred)\n    recall = recall_m(y_true, y_pred)\n    return 2*K.mean((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Confusion matrix."},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                        normalize=False,\n                        title='Confusion matrix',\n                        cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n\n    print(cm)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, np.round(1000*cm[i, j])/1000,\n            horizontalalignment=\"center\",\n            color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.037843,"end_time":"2020-11-19T21:47:59.792061","exception":false,"start_time":"2020-11-19T21:47:59.754218","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Load the data"},{"metadata":{"papermill":{"duration":0.038439,"end_time":"2020-11-19T21:47:59.869037","exception":false,"start_time":"2020-11-19T21:47:59.830598","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Decode the data"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:47:59.952622Z","iopub.status.busy":"2020-11-19T21:47:59.951868Z","iopub.status.idle":"2020-11-19T21:47:59.954997Z","shell.execute_reply":"2020-11-19T21:47:59.955558Z"},"papermill":{"duration":0.04859,"end_time":"2020-11-19T21:47:59.955731","exception":false,"start_time":"2020-11-19T21:47:59.907141","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32)\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:48:00.123967Z","iopub.status.busy":"2020-11-19T21:48:00.123143Z","iopub.status.idle":"2020-11-19T21:48:00.126902Z","shell.execute_reply":"2020-11-19T21:48:00.126284Z"},"papermill":{"duration":0.052475,"end_time":"2020-11-19T21:48:00.127039","exception":false,"start_time":"2020-11-19T21:48:00.074564","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:48:00.325942Z","iopub.status.busy":"2020-11-19T21:48:00.324875Z","iopub.status.idle":"2020-11-19T21:48:00.327502Z","shell.execute_reply":"2020-11-19T21:48:00.328493Z"},"papermill":{"duration":0.073623,"end_time":"2020-11-19T21:48:00.328703","exception":false,"start_time":"2020-11-19T21:48:00.25508","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:48:00.683596Z","iopub.status.busy":"2020-11-19T21:48:00.607588Z","iopub.status.idle":"2020-11-19T21:48:00.687244Z","shell.execute_reply":"2020-11-19T21:48:00.686445Z"},"papermill":{"duration":0.225941,"end_time":"2020-11-19T21:48:00.687385","exception":false,"start_time":"2020-11-19T21:48:00.461444","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(\n    tf.io.gfile.glob(GCS_PATH + '/train_tfrecords/ld_train*.tfrec'),\n    test_size=0.35, random_state=5\n)\n\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test_tfrecords/ld_test*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.038372,"end_time":"2020-11-19T21:48:00.765394","exception":false,"start_time":"2020-11-19T21:48:00.727022","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Adding in augmentations "},{"metadata":{"trusted":true},"cell_type":"code","source":"# One hot label and float images.\ndef data_treat(image,label):\n    label = tf.one_hot(label,len(CLASSES))\n    image = tf.cast(image, tf.float32)\n    return image,label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Values going from 0 to 255.\ndef data_treat_test(image,label):\n    image = tf.cast(image, tf.float32)\n    image = image-tf.math.reduce_min(image)\n    image = image/tf.math.reduce_max(image)\n    image = image*255\n    return image,label","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:48:00.849827Z","iopub.status.busy":"2020-11-19T21:48:00.848867Z","iopub.status.idle":"2020-11-19T21:48:00.85179Z","shell.execute_reply":"2020-11-19T21:48:00.85115Z"},"papermill":{"duration":0.047715,"end_time":"2020-11-19T21:48:00.851918","exception":false,"start_time":"2020-11-19T21:48:00.804203","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    # Thanks to the dataset.prefetch(AUTO) statement in the following function this happens essentially for free on TPU. \n    # Data pipeline code is executed on the \"CPU\" part of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def func_standard(p=PROBA_CONTRAST):\n    def data_standard(image, label):\n        if tf.random.uniform(shape=(), minval=0, maxval=1)<p:\n            image = image-tf.math.reduce_min(image)\n            image = image/tf.math.reduce_max(image)\n            image = tf.math.round(image*255)\n        return image, label\n    return data_standard","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.038742,"end_time":"2020-11-19T21:48:00.930185","exception":false,"start_time":"2020-11-19T21:48:00.891443","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Define data loading methods\nThe following functions will be used to load our `training`, `validation`, and `test` datasets, as well as print out the number of images in each dataset."},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:48:01.301631Z","iopub.status.busy":"2020-11-19T21:48:01.30084Z","iopub.status.idle":"2020-11-19T21:48:01.304479Z","shell.execute_reply":"2020-11-19T21:48:01.303807Z"},"papermill":{"duration":0.05422,"end_time":"2020-11-19T21:48:01.304611","exception":false,"start_time":"2020-11-19T21:48:01.250391","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:48:01.39253Z","iopub.status.busy":"2020-11-19T21:48:01.391743Z","iopub.status.idle":"2020-11-19T21:48:01.395039Z","shell.execute_reply":"2020-11-19T21:48:01.395972Z"},"papermill":{"duration":0.051209,"end_time":"2020-11-19T21:48:01.396198","exception":false,"start_time":"2020-11-19T21:48:01.344989","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALID_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\n\nprint('Dataset: {} training images, {} validation images, {} (unlabeled) test images'.format(\n    NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.041086,"end_time":"2020-11-19T21:48:01.478205","exception":false,"start_time":"2020-11-19T21:48:01.437119","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Brief exploratory data analysis (EDA)\nFirst we'll print out the shapes and labels for a sample of each of our three datasets:"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(os.path.join(BASE_DIR, \"label_num_to_disease_map.json\")) as file:\n    map_classes = json.loads(file.read())\n    map_classes = {int(k) : v for k, v in map_classes.items()}\n    \nprint(json.dumps(map_classes, indent=4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_files = os.listdir(os.path.join(BASE_DIR, \"train_images\"))\nprint(f\"Number of train images: {len(input_files)}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(BASE_DIR, \"train.csv\"))\ndf_train[\"class_name\"] = df_train[\"label\"].map(map_classes)\nplt.figure(figsize=(8, 4))\nsn.countplot(y=\"class_name\", data=df_train);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This function returns the labels weights, compounded by a coefficient n.\ndef c_weights(labels,n=3/4):\n    c_labels = Counter(labels)\n    A=len(c_labels)/np.sum([x**-n for x in c_labels.values()])\n    cw = {i:A*c_labels[i]**-n for i in range(5)}\n    return cw","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cw = c_weights(df_train[\"label\"],n=3/4)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.044101,"end_time":"2020-11-19T21:48:18.6862","exception":false,"start_time":"2020-11-19T21:48:18.642099","status":"completed"},"tags":[]},"cell_type":"markdown","source":"The following code chunk sets up a series of functions that will print out a grid of images. The grid of images will contain images and their corresponding labels."},{"metadata":{"papermill":{"duration":0.230199,"end_time":"2020-11-19T21:48:28.353794","exception":false,"start_time":"2020-11-19T21:48:28.123595","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Building the model\n## Learning rate schedule\nWe learned about learning rates in the **[Intro to Deep Learning: Stochastic Gradient Descent](https://www.kaggle.com/ryanholbrook/stochastic-gradient-descent)** lesson, and here I've created a learning rate schedule mostly using the defaults in the **[Keras Exponential Decay Learning Rate Scheduler](https://keras.io/api/optimizers/learning_rate_schedules/exponential_decay/)** documentation (I did change the `initial_learning_rate`. You can adjust the learning rate scheduler below, and read more about the other types of schedulers available to you in the **[Keras learning rate schedules API](https://keras.io/api/optimizers/learning_rate_schedules/)**."},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:48:28.8307Z","iopub.status.busy":"2020-11-19T21:48:28.829632Z","iopub.status.idle":"2020-11-19T21:48:28.833152Z","shell.execute_reply":"2020-11-19T21:48:28.832481Z"},"papermill":{"duration":0.248904,"end_time":"2020-11-19T21:48:28.83328","exception":false,"start_time":"2020-11-19T21:48:28.584376","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"lr_scheduler = keras.optimizers.schedules.ExponentialDecay(\n    initial_learning_rate=1e-4, \n    decay_steps=1000, \n    decay_rate=0.9)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Simple color model.\nWe want to check the importance of colors to the illness detection. We create a color only model."},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_histogram(x,l):\n    Z=tf.split(x, 3, axis=-1)\n    Z=[tf.histogram_fixed_width(z,[0., 255.], nbins=32) for z in Z]\n    Z=tf.concat(Z, -1)\n    return Z,l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_histogram(ordered=False):\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=ordered)  \n    dataset = dataset.map(data_treat, num_parallel_calls=AUTOTUNE)  \n    dataset = dataset.map(func_standard(p=1),num_parallel_calls=AUTOTUNE)  \n    dataset = dataset.map(create_histogram, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_histogram(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.map(data_treat_test, num_parallel_calls=AUTOTUNE)  \n    dataset = dataset.map(create_histogram, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.repeat()\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_histogram(ordered=False):\n    dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=ordered) \n    dataset = dataset.map(data_treat, num_parallel_calls=AUTOTUNE)  \n    dataset = dataset.map(func_standard(p=1),num_parallel_calls=AUTOTUNE)  \n    dataset = dataset.map(create_histogram, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Counting the labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"p = 0.1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Model based on the colours distribution."},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model():\n    A2=tf.keras.layers.Input(shape=(32*3,))\n    X=tf.keras.layers.Dense(2048,activation='relu')(A2)\n    X=tf.keras.layers.Dropout(p)(X)\n    X=tf.keras.layers.Dense(128,activation='relu')(X)\n    X=tf.keras.layers.Dropout(p)(X)\n    X=tf.keras.layers.Dense(len(CLASSES),activation='softmax')(X)\n    model = tf.keras.Model(inputs=A2, outputs=X)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():       \n    model = create_model()\n    if os.path.exists('pierre_color.h5'):\n        print('loading')\n        modelp = tf.keras.models.load_model('pierre_color.h5',compile=False)\n        model.set_weights(modelp.get_weights())\n    else:\n        print('creating')\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=lr_scheduler),\n        loss='categorical_crossentropy',  metrics=['categorical_accuracy',f1_m],\n        )","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T22:04:51.98529Z","iopub.status.busy":"2020-11-19T22:04:51.984414Z","iopub.status.idle":"2020-11-19T22:04:51.988629Z","shell.execute_reply":"2020-11-19T22:04:51.987853Z"},"papermill":{"duration":1.344562,"end_time":"2020-11-19T22:04:51.988755","exception":false,"start_time":"2020-11-19T22:04:50.644193","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load data\ntrain_dataset = get_training_histogram()\nvalid_dataset = get_validation_histogram()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We first train without class weights."},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE\n\nhistory = model.fit(train_dataset, \n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    epochs=EPOCHS,#class_weight=c_weigths,\n                    validation_data=valid_dataset,\n                    validation_steps=VALID_STEPS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('pierre_color.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Checking the results."},{"metadata":{"trusted":true},"cell_type":"code","source":"# this code will convert our test image data to a float32 \ndef to_float32(image, label):\n    return tf.cast(image, tf.float32), label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_histogram(ordered=True)\nvalid_dataset = get_validation_histogram(ordered=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=True)\n#dataset = tf.data.TFRecordDataset(TRAINING_FILENAMES[0], num_parallel_reads=AUTOTUNE)\nvalid_labels = []\nfor images, labels in dataset.take(-1):  # only take first element of dataset\n    valid_labels.append(labels.numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = train_dataset.map(to_float32)\nfit_train_label = model.predict(train_ds,steps=STEPS_PER_EPOCH)\nfit_train_label = np.argmax(fit_train_label,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_ds = valid_dataset.map(to_float32)\nfit_valid_label = model.predict(valid_ds)\nfit_valid_label = np.argmax(fit_valid_label,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm = confusion_matrix(valid_labels[:len(fit_valid_label)],fit_valid_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_confusion_matrix(cm, [0,1,2,3,4],normalize=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This model simply return the most common class (here 3). We will compare with the same model with weights added."},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():       \n    model = create_model()\n    if os.path.exists('pierre_color_W.h5'):\n        print('loading')\n        modelp = tf.keras.models.load_model('pierre_color_W.h5',compile=False)\n        model.set_weights(modelp.get_weights())\n    else:\n        print('creating')\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=lr_scheduler, epsilon=0.001),\n        loss='categorical_crossentropy',  metrics=['categorical_accuracy',f1_m],\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_histogram(ordered=False)\nvalid_dataset = get_validation_histogram(ordered=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# First weight : same weights per class.\ncw = c_weights(df_train[\"label\"],n=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE\n\nhistory = model.fit(train_dataset, \n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    epochs=EPOCHS,class_weight=cw,\n                    validation_data=valid_dataset,\n                    validation_steps=VALID_STEPS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('pierre_color_W.h5')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":1.245239,"end_time":"2020-11-19T22:04:54.493139","exception":false,"start_time":"2020-11-19T22:04:53.2479","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Visualizing the new results.\nLet's see if the distribution has changed."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_histogram(ordered=True)\nvalid_dataset = get_validation_histogram(ordered=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = train_dataset.map(to_float32)\nfit_train_label = model.predict(train_ds,steps=STEPS_PER_EPOCH)\nfit_train_label = np.argmax(fit_train_label,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_ds = valid_dataset.map(to_float32)\nfit_valid_label = model.predict(valid_ds)\nfit_valid_label = np.argmax(fit_valid_label,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm = confusion_matrix(valid_labels[:len(fit_valid_label)],fit_valid_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_confusion_matrix(cm, [0,1,2,3,4],normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":1.26977,"end_time":"2020-11-19T22:04:49.3763","exception":false,"start_time":"2020-11-19T22:04:48.10653","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Now, all entries goes to the smaller class.  \nChecking the litterature, a power law of 3/4 is often prefered. We try it as our last model."},{"metadata":{},"cell_type":"markdown","source":"## Last model n=3/4. "},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T22:04:57.050462Z","iopub.status.busy":"2020-11-19T22:04:57.0494Z","iopub.status.idle":"2020-11-19T22:04:57.053166Z","shell.execute_reply":"2020-11-19T22:04:57.053855Z"},"papermill":{"duration":1.31245,"end_time":"2020-11-19T22:04:57.054025","exception":false,"start_time":"2020-11-19T22:04:55.741575","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"with strategy.scope():       \n    model = create_model()\n    if os.path.exists('pierre_color_34.h5'):\n        print('loading')\n        modelp = tf.keras.models.load_model('pierre_color_34.h5',compile=False)\n        model.set_weights(modelp.get_weights())\n    else:\n        print('creating')\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=lr_scheduler, epsilon=0.001),\n        loss='categorical_crossentropy',  metrics=['categorical_accuracy',f1_m],\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_histogram(ordered=False)\nvalid_dataset = get_validation_histogram(ordered=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Second weight : n=3/4.\ncw = c_weights(df_train[\"label\"],n=3/4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE\n\nhistory = model.fit(train_dataset, \n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    epochs=EPOCHS,class_weight=cw,\n                    validation_data=valid_dataset,\n                    validation_steps=VALID_STEPS)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T22:04:59.5746Z","iopub.status.busy":"2020-11-19T22:04:59.573814Z","iopub.status.idle":"2020-11-19T22:04:59.983142Z","shell.execute_reply":"2020-11-19T22:04:59.982506Z"},"papermill":{"duration":1.671861,"end_time":"2020-11-19T22:04:59.983272","exception":false,"start_time":"2020-11-19T22:04:58.311411","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"model.save('pierre_color_34.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualizing the last results.\nLet's see if the distribution has changed."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_histogram(ordered=True)\nvalid_dataset = get_validation_histogram(ordered=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = train_dataset.map(to_float32)\nfit_train_label = model.predict(train_ds,steps=STEPS_PER_EPOCH)\nfit_train_label = np.argmax(fit_train_label,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_ds = valid_dataset.map(to_float32)\nfit_valid_label = model.predict(valid_ds)\nfit_valid_label = np.argmax(fit_valid_label,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm = confusion_matrix(valid_labels[:len(fit_valid_label)],fit_valid_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_confusion_matrix(cm, [0,1,2,3,4],normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":1.326243,"end_time":"2020-11-19T22:05:02.628032","exception":false,"start_time":"2020-11-19T22:05:01.301789","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Once again, only one class is found. We can conclude that colour distribution is not a good parameter to determine the illness of cassava leaf. A model based on correlation seems needed."},{"metadata":{"papermill":{"duration":1.271799,"end_time":"2020-11-19T22:05:24.759257","exception":false,"start_time":"2020-11-19T22:05:23.487458","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Creating a submission file\nNow that we've trained a model and made predictions we're ready to submit to the competition! You can run the following code below to get your submission file."},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T22:05:27.316025Z","iopub.status.busy":"2020-11-19T22:05:27.315202Z","iopub.status.idle":"2020-11-19T22:05:28.241598Z","shell.execute_reply":"2020-11-19T22:05:28.24078Z"},"papermill":{"duration":2.185537,"end_time":"2020-11-19T22:05:28.241723","exception":false,"start_time":"2020-11-19T22:05:26.056186","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"#print('Generating submission.csv file...')\n#test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n#test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\n#np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\n#!head submission.csv","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":1.255302,"end_time":"2020-11-19T22:05:30.746339","exception":false,"start_time":"2020-11-19T22:05:29.491037","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Be aware that because this is a code competition with a hidden test set, internet and TPUs cannot be enabled on your submission notebook. Therefore TPUs will only be available for training models. For a walk-through on how to train on TPUs and run inference/submit on GPUs, see our [TPU Docs](https://www.kaggle.com/docs/tpu#tpu6)."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}