{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.039382,"end_time":"2020-11-19T21:45:23.042097","exception":false,"start_time":"2020-11-19T21:45:23.002715","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Introduction\n**This notebook is based on  [Jesse Mostipak’s Tutorial](https://www.kaggle.com/jessemostipak/getting-started-tpus-cassava-leaf-disease)**  \nIn this notebook we check the importance of colours in the classification process."},{"metadata":{"papermill":{"duration":0.037375,"end_time":"2020-11-19T21:45:23.192515","exception":false,"start_time":"2020-11-19T21:45:23.15514","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Set up environment"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q tensorflow==2.3.2 # Use 2.3.0 for built-in EfficientNet\n\n!pip install -q git+https://github.com/keras-team/keras-tuner@master # Use github head for newly added TPU support\n!pip install -q cloud-tpu-client # Needed for sync TPU version\n!pip install -U tensorflow-gcs-config==2.3.0 # Needed for using private dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random, re, math, os, json\nimport numpy as np, pandas as pd, seaborn as sn\nimport cv2\nimport matplotlib.pyplot as plt\nimport tensorflow as tf, tensorflow.keras.backend as K\nfrom tensorflow.keras.utils import plot_model,to_categorical\nfrom tensorflow.keras.models import load_model\nfrom functools import partial\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nimport albumentations as A\nimport kerastuner as kt\nimport IPython\nfrom IPython.display import FileLink\nprint('Tensorflow version ' + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # Sync TPU version\n    from cloud_tpu_client import Client\n    c = Client()\n    c.configure_tpu_version(tf.__version__, restart_type='ifNeeded')\n    \n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n    \n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\nprint(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import Counter\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nimport itertools\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.applications import EfficientNetB5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we will see below, the classes are not equally represented. Therefore, we need more than just accuracy to determine the validity of our model. Hence, we define the f1 metric."},{"metadata":{"trusted":true},"cell_type":"code","source":"def recall_m(y_true, y_pred):\n    true_positives = K.sum(y_true * y_pred, axis=0)\n    possible_positives = K.sum(y_true, axis=0)\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\ndef precision_m(y_true, y_pred):\n    true_positives = K.sum(y_true * y_pred, axis=0)\n    predicted_positives = K.sum(y_pred, axis=0)\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\ndef f1_m(y_true, y_pred):\n    y_pred = tf.one_hot(tf.argmax(y_pred,axis=-1),len(CLASSES))\n    precision = precision_m(y_true, y_pred)\n    recall = recall_m(y_true, y_pred)\n    return 2*K.mean((precision*recall)/(precision+recall+K.epsilon()))\n","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.038122,"end_time":"2020-11-19T21:45:34.458722","exception":false,"start_time":"2020-11-19T21:45:34.4206","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Set up variables\nWe'll set up some of our variables for our notebook here. "},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:45:34.555293Z","iopub.status.busy":"2020-11-19T21:45:34.541822Z","iopub.status.idle":"2020-11-19T21:47:59.71579Z","shell.execute_reply":"2020-11-19T21:47:59.714961Z"},"papermill":{"duration":145.219568,"end_time":"2020-11-19T21:47:59.715925","exception":false,"start_time":"2020-11-19T21:45:34.496357","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nBASE_DIR = '../input/cassava-leaf-disease-classification/'\nGCS_PATH = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification')\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n#BATCH_SIZE = 512\nIMAGE_EN = {'B0':224,'B1':240,'B2':260,'B3':300,'B4':380,'B5':456,'B6':528,'B7':600}\nCLASSES = ['0', '1', '2', '3', '4']\nEPOCHS = 10\nPROBA_CONTRAST=1.\nos.chdir(r'/kaggle/working')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EfficientNet architecture.  \nWe look at the best places to conduct fine tuning."},{"metadata":{"trusted":true},"cell_type":"code","source":"base_model = EfficientNetB5(weights='imagenet', include_top=False)\nIMAGE_SIZE = [IMAGE_EN['B5'],IMAGE_EN['B5']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_model(base_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_Add = [i for i in range(len(base_model.layers)) if isinstance(base_model.layers[i],tf.keras.layers.Add)]\nprint(n_Add)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.037843,"end_time":"2020-11-19T21:47:59.792061","exception":false,"start_time":"2020-11-19T21:47:59.754218","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Load the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(\n    tf.io.gfile.glob(GCS_PATH + '/train_tfrecords/ld_train*.tfrec'),\n    test_size=0.125, random_state=5)\n\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test_tfrecords/ld_test*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(os.path.join(BASE_DIR, \"label_num_to_disease_map.json\")) as file:\n    map_classes = json.loads(file.read())\n    map_classes = {int(k) : v for k, v in map_classes.items()}\n    \nprint(json.dumps(map_classes, indent=4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_files = os.listdir(os.path.join(BASE_DIR, \"train_images\"))\nprint(f\"Number of train images: {len(input_files)}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(BASE_DIR, \"train.csv\"))\ndf_train[\"class_name\"] = df_train[\"label\"].map(map_classes)\nplt.figure(figsize=(8, 4))\nsn.countplot(y=\"class_name\", data=df_train);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Since the classes are not evenly distributed, we need to use weights to prevent metastable results."},{"metadata":{"trusted":true},"cell_type":"code","source":"# This function returns the labels weights, compounded by a coefficient n.\ndef c_weights(labels,n=3/4):\n    c_labels = Counter(labels)\n    A=len(c_labels)/np.sum([x**-n for x in c_labels.values()])\n    cw = {i:A*c_labels[i]**-n for i in range(5)}\n    return cw","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Counter(df_train[\"label\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Shows examples"},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize_batch(image_ids, labels):\n    plt.figure(figsize=(20, 15))\n    \n    for ind, (image_id, label) in enumerate(zip(image_ids, labels)):\n        plt.subplot(5, 4, ind + 1)\n        image = cv2.imread(os.path.join(BASE_DIR, \"train_images\", image_id))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        plt.imshow(image)\n        plt.title(f\"Class: {label}\", fontsize=12)\n        plt.axis(\"off\")\n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_index = []\nfor i in range(5):\n    tmp_index += list(df_train.loc[df_train['label']==i].sample(4).index)\n    \nimage_ids = df_train[\"image_id\"].loc[tmp_index].values\nlabels = df_train[\"label\"].loc[tmp_index].values\n\nvisualize_batch(image_ids, labels)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.038439,"end_time":"2020-11-19T21:47:59.869037","exception":false,"start_time":"2020-11-19T21:47:59.830598","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Decode the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:47:59.952622Z","iopub.status.busy":"2020-11-19T21:47:59.951868Z","iopub.status.idle":"2020-11-19T21:47:59.954997Z","shell.execute_reply":"2020-11-19T21:47:59.955558Z"},"papermill":{"duration":0.04859,"end_time":"2020-11-19T21:47:59.955731","exception":false,"start_time":"2020-11-19T21:47:59.907141","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32)\n    image = tf.image.resize(image, IMAGE_SIZE)\n    return image","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:48:00.123967Z","iopub.status.busy":"2020-11-19T21:48:00.123143Z","iopub.status.idle":"2020-11-19T21:48:00.126902Z","shell.execute_reply":"2020-11-19T21:48:00.126284Z"},"papermill":{"duration":0.052475,"end_time":"2020-11-19T21:48:00.127039","exception":false,"start_time":"2020-11-19T21:48:00.074564","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:48:00.325942Z","iopub.status.busy":"2020-11-19T21:48:00.324875Z","iopub.status.idle":"2020-11-19T21:48:00.327502Z","shell.execute_reply":"2020-11-19T21:48:00.328493Z"},"papermill":{"duration":0.073623,"end_time":"2020-11-19T21:48:00.328703","exception":false,"start_time":"2020-11-19T21:48:00.25508","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train-Validation class repartition"},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#dataset = tf.data.TFRecordDataset(TRAINING_FILENAMES[0], num_parallel_reads=AUTOTUNE)\ntrain_labels = []\nfor _, labels in dataset.take(-1):  # only take first element of dataset\n    train_labels.append(labels.numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=True)\n#dataset = tf.data.TFRecordDataset(TRAINING_FILENAMES[0], num_parallel_reads=AUTOTUNE)\nvalid_labels = []\nfor images, labels in dataset.take(-1):  # only take first element of dataset\n    valid_labels.append(labels.numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c_train = Counter(train_labels)\nx_train = [0,1,2,3,4]\ny_train = [c_train[i] for i in x_train]\n\nc_valid = Counter(valid_labels)\nx_valid = [0,1,2,3,4]\ny_valid = [c_valid[i] for i in x_valid]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We verify that the train and valid distributions are similar."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.barh(x_train,y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.barh(x_valid,y_valid)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.038372,"end_time":"2020-11-19T21:48:00.765394","exception":false,"start_time":"2020-11-19T21:48:00.727022","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Adding in augmentations "},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_treat(image,label):\n    label = tf.one_hot(label,len(CLASSES))\n    image = tf.cast(image, tf.float32)\n    return image,label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_treat_test(image,label):\n    image = tf.cast(image, tf.float32)\n    image = image-tf.math.reduce_min(image)\n    image = image/tf.math.reduce_max(image)\n    image = image*255\n    return image,label","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:48:00.849827Z","iopub.status.busy":"2020-11-19T21:48:00.848867Z","iopub.status.idle":"2020-11-19T21:48:00.85179Z","shell.execute_reply":"2020-11-19T21:48:00.85115Z"},"papermill":{"duration":0.047715,"end_time":"2020-11-19T21:48:00.851918","exception":false,"start_time":"2020-11-19T21:48:00.804203","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def data_augment(image,label):\n    # Thanks to the dataset.prefetch(AUTO) statement in the following function this happens essentially for free on TPU. \n    # Data pipeline code is executed on the \"CPU\" part of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    return image,label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This function returns a function with variable probability of applying standardisation.\ndef func_standard(p=1):\n    def data_standard(image,label):\n        if tf.random.uniform(shape=(), minval=0, maxval=1)<p:\n            image = image-tf.math.reduce_min(image)\n            image = image/tf.math.reduce_max(image)\n            image = image*255\n        return image,label\n    return data_standard","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def func_BAW(p=PROBA_CONTRAST):\n# random black and white\n    def data_BAW(image,label):\n        if tf.random.uniform(shape=(), minval=0, maxval=1)<p:\n            image = tf.image.rgb_to_grayscale(image)\n            image = tf.math.round(image)\n            image = tf.image.grayscale_to_rgb(image)\n        return image,label\n    return data_BAW","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.038742,"end_time":"2020-11-19T21:48:00.930185","exception":false,"start_time":"2020-11-19T21:48:00.891443","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Define data loading methods\nThe following functions will be used to load our `training`, `validation`, and `test` datasets, as well as print out the number of images in each dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset(ordered=False):\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=ordered)  \n    dataset = dataset.map(data_treat, num_parallel_calls=AUTOTUNE)  \n    dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE)  \n    dataset = dataset.map(func_standard(p=1),num_parallel_calls=AUTOTUNE)  \n    dataset = dataset.map(func_BAW(p=1./5.),num_parallel_calls=AUTOTUNE)  \n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(1024)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Black and white transformation is not performed on the validation dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=ordered) \n    dataset = dataset.map(data_treat, num_parallel_calls=AUTOTUNE)  \n    dataset = dataset.map(func_standard(p=1),num_parallel_calls=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.map(data_treat_test, num_parallel_calls=AUTOTUNE)  \n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALID_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\n\nprint('Dataset: {} training images, {} validation images, {} (unlabeled) test images'.format(\n    NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.230199,"end_time":"2020-11-19T21:48:28.353794","exception":false,"start_time":"2020-11-19T21:48:28.123595","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Learning rate schedule.  "},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:48:28.8307Z","iopub.status.busy":"2020-11-19T21:48:28.829632Z","iopub.status.idle":"2020-11-19T21:48:28.833152Z","shell.execute_reply":"2020-11-19T21:48:28.832481Z"},"papermill":{"duration":0.248904,"end_time":"2020-11-19T21:48:28.83328","exception":false,"start_time":"2020-11-19T21:48:28.584376","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def loop_schedule(l):\n    def scheduler(epoch,lr):\n      if epoch>0:\n        return lr/1.01\n      else :\n        return l\n    schedule = tf.keras.callbacks.LearningRateScheduler(scheduler)\n    return schedule","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Early stopping. "},{"metadata":{"trusted":true},"cell_type":"code","source":"# We create a callback to stop training if the validity loss does not deacrease for 4 epochs.\nearly_stop=tf.keras.callbacks.EarlyStopping(monitor='val_f1_m', min_delta=0, patience=4, verbose=0,\n    mode='max', baseline=None, restore_best_weights=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Optimization"},{"metadata":{"trusted":true},"cell_type":"code","source":"# load data\ntrain_dataset = get_training_dataset()\nvalid_dataset = get_validation_dataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We create a function to instanciate our model.\ndef createModel(dropout_rate=0.5,\n                unit_1 = 997,\n                unit_2 = len(CLASSES),\n                trainable_loops=0):\n    base_model = EfficientNetB5(weights='imagenet', include_top=False,input_shape=[*IMAGE_SIZE,3])\n    # Last trainable layer.\n    last_trainable = n_Add[-trainable_loops-1]\n    for layer in base_model.layers: layer.trainable = False\n    for i in range(last_trainable,len(base_model.layers)):\n            base_model.layers[i].trainable=True\n            \n    model = tf.keras.Sequential([\n            base_model,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dropout(dropout_rate),\n            tf.keras.layers.Dense(unit_1, activation='relu'),\n            tf.keras.layers.Dropout(dropout_rate),\n            tf.keras.layers.Dense(unit_2, activation='relu'),\n            tf.keras.layers.Dropout(dropout_rate),\n            tf.keras.layers.Dense(len(CLASSES), activation='softmax')  \n        ])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def random_search(num):\n    # No loop variation.\n    # Tune the number of units in the two dense layer.\n    values_mem = {}\n    results = {}\n    for i in range(num):\n        # initial learning rate.\n        lr = random.random()*np.log(0.01/0.000001)+np.log(0.000001)\n        lr = np.exp(lr)\n        # dropout rate.\n        dp = random.random()*np.log(0.5/0.01)+np.log(0.01)\n        dp = np.exp(dp)\n        # number of nodes first layer.\n        units_1 = random.random()*np.log(4096/4)+np.log(4)\n        units_1 = int(np.round(np.exp(units_1)))\n        # number of nodes second layer.\n        units_2 = random.random()*np.log(4096/4)+np.log(4)\n        units_2 = int(np.round(np.exp(units_2)))\n        # exponent\n        wt = random.random()\n        \n        values_mem[i] = (dp,units_1,units_2,lr,wt) \n    \n        with strategy.scope():  \n            model = createModel(dropout_rate=dp,unit_1 = units_1,unit_2 = units_2)\n\n            model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=0.0002),\n                    loss = tf.keras.losses.CategoricalCrossentropy(), metrics = ['categorical_accuracy',f1_m])\n        \n        model.fit(train_dataset, \n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    epochs=15,\n                    class_weight=c_weights(df_train[\"label\"],n=wt),\n                    callbacks=[early_stop,loop_schedule(lr)])\n        \n        results[i] = model.evaluate(valid_dataset,steps=VALID_STEPS)\n    results = [list(values_mem[i])+results[i] for i in range(len(results))]\n    results = pd.DataFrame({'Dropout_rate':[r[0] for r in results],\n                       'units_1':[r[1] for r in results],\n                       'units_2':[r[2] for r in results],\n                       'learning_rate':[r[3] for r in results],\n                       'weight_exponant':[r[4] for r in results],\n                       'loss':[r[5] for r in results],\n                       'categorical_accuracy':[r[6] for r in results],\n                       'f1_m':[r[7] for r in results],})\n    \n    return results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for memory reason it may be needed to run this search many times, while restarting the kernel.\nresults = random_search(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Easy to use download button.\nresults.to_pickle(r'opti.pkl')\nFileLink(r'opti.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.semilogx(results['Dropout_rate'],results['f1_m'],'.')\nplt.xlabel('Dropout_rate')\nplt.ylabel('f1_m')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.semilogx(results['units_1'],results['f1_m'],'.')\nplt.xlabel('units_1')\nplt.ylabel('f1_m')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.22538,"end_time":"2020-11-19T21:48:29.285377","exception":false,"start_time":"2020-11-19T21:48:29.059997","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Building the final model\nIn order to ensure that our model is trained on the TPU, we build it using `with strategy.scope()`.    \n\nThis model was built using transfer learning, meaning that we have a _pre-trained model_ (ResNet50) as our base model and then the customizable model built using `tf.keras.Sequential`.\n\nNote that we're using `sparse_categorical_crossentropy` as our loss function, because we did _not_ one-hot encode our labels."},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():       \n    model = createModel(dropout_rate=0.35,unit_1 = 256,\n                unit_2 = 16,trainable_loops=0)\n    \n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=0.0002),\n        loss='categorical_crossentropy',  \n        metrics=['categorical_accuracy',f1_m])","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":1.26977,"end_time":"2020-11-19T22:04:49.3763","exception":false,"start_time":"2020-11-19T22:04:48.10653","status":"completed"},"tags":[]},"cell_type":"markdown","source":"We check the resulting network to be sure that the added layers are trainable, while the efficientNet5 is not trained."},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T22:04:51.98529Z","iopub.status.busy":"2020-11-19T22:04:51.984414Z","iopub.status.idle":"2020-11-19T22:04:51.988629Z","shell.execute_reply":"2020-11-19T22:04:51.987853Z"},"papermill":{"duration":1.344562,"end_time":"2020-11-19T22:04:51.988755","exception":false,"start_time":"2020-11-19T22:04:50.644193","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.174404,"end_time":"2020-11-19T21:48:49.513099","exception":false,"start_time":"2020-11-19T21:48:49.338695","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Train the model on pictures\nAfter checking the importance of colours, we train our model with B&W pictures, in this case we duplicate 3 times the same black and white image to create a false RGB image. This image is then fed into resnet50.  \nThe B&W transformation is added as an image augmentation, as latter on we will want to train models on colored images with some B&W images."},{"metadata":{"trusted":true},"cell_type":"code","source":"# load data\ntrain_dataset = get_training_dataset()\nvalid_dataset = get_validation_dataset()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T21:48:50.304573Z","iopub.status.busy":"2020-11-19T21:48:50.303472Z","iopub.status.idle":"2020-11-19T22:04:46.794408Z","shell.execute_reply":"2020-11-19T22:04:46.795473Z"},"papermill":{"duration":956.681695,"end_time":"2020-11-19T22:04:46.795744","exception":false,"start_time":"2020-11-19T21:48:50.114049","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE\n\nhistory = model.fit(train_dataset, \n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    epochs=20,\n                    class_weight=c_weights(df_train[\"label\"],n=3/4),\n                    validation_data=valid_dataset,\n                    validation_steps=VALID_STEPS,\n                    callbacks=[early_stop,loop_schedule(0.00025)])\nmodel.save('EffNet5_0.h5')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":1.245239,"end_time":"2020-11-19T22:04:54.493139","exception":false,"start_time":"2020-11-19T22:04:53.2479","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Visualizing training\nWe visualize evolution of loss and accuracy over epochs. "},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T22:04:57.050462Z","iopub.status.busy":"2020-11-19T22:04:57.0494Z","iopub.status.idle":"2020-11-19T22:04:57.053166Z","shell.execute_reply":"2020-11-19T22:04:57.053855Z"},"papermill":{"duration":1.31245,"end_time":"2020-11-19T22:04:57.054025","exception":false,"start_time":"2020-11-19T22:04:55.741575","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# print out variables available to us\nprint(history.history.keys())","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T22:04:59.5746Z","iopub.status.busy":"2020-11-19T22:04:59.573814Z","iopub.status.idle":"2020-11-19T22:04:59.983142Z","shell.execute_reply":"2020-11-19T22:04:59.982506Z"},"papermill":{"duration":1.671861,"end_time":"2020-11-19T22:04:59.983272","exception":false,"start_time":"2020-11-19T22:04:58.311411","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# create learning curves to evaluate model performance\nhistory_frame = pd.DataFrame(history.history)\nhistory_frame.loc[:, ['f1_m', 'val_f1_m']].plot()\nhistory_frame.loc[:, ['categorical_accuracy', 'val_categorical_accuracy']].plot();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Result distribution."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_dataset(ordered=True)\nvalid_dataset = get_validation_dataset(ordered=True)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T22:05:05.184942Z","iopub.status.busy":"2020-11-19T22:05:05.183725Z","iopub.status.idle":"2020-11-19T22:05:05.18757Z","shell.execute_reply":"2020-11-19T22:05:05.186823Z"},"papermill":{"duration":1.270192,"end_time":"2020-11-19T22:05:05.187694","exception":false,"start_time":"2020-11-19T22:05:03.917502","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# this code will convert our test image data to a float32 \ndef to_float32(image, label):\n    return tf.cast(image, tf.float32), label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_ds = valid_dataset.map(to_float32)\nfit_valid_label = model.predict(valid_ds)\nfit_valid_label = np.argmax(fit_valid_label,axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Confusion matrix."},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                        normalize=False,\n                        title='Confusion matrix',\n                        cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if normalize:\n        nm = np.sqrt(np.array([[sum(cm[i,:])*sum(cm[j,:]) for i in range(5)] for j in range(5)]))\n        #cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        cm = cm.astype('float') / nm\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n\n    print(cm)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, np.round(1000*cm[i, j])/1000,\n            horizontalalignment=\"center\",\n            color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm = confusion_matrix(valid_labels[:len(fit_valid_label)],fit_valid_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_confusion_matrix(cm, [0,1,2,3,4],normalize=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fine tuning. \nWe will now fine tune the model. \nWe will compare fine tuning of the last block, and off the last two blocks."},{"metadata":{"trusted":true},"cell_type":"code","source":"# 1 trainable block.\nwith strategy.scope():       \n    model = createModel(dropout_rate=0.35,unit_1 = 256,\n                unit_2 = 16,trainable_loops=1)\n    \n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=0.00002),\n        loss='categorical_crossentropy',  \n        metrics=['categorical_accuracy',f1_m])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_dataset()\nvalid_dataset = get_validation_dataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE\n\nhistory = model.fit(train_dataset, \n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    epochs=30,\n                    class_weight=c_weights(df_train[\"label\"],n=3/4),\n                    validation_data=valid_dataset,\n                    validation_steps=VALID_STEPS,\n                    callbacks=[early_stop,loop_schedule(0.00015)])\nmodel.save('EffNet5_1.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2 loops"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_dataset()\nvalid_dataset = get_validation_dataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 2 trainable block.\nwith strategy.scope():       \n    model = model = createModel(dropout_rate=0.35,unit_1 = 256,\n                unit_2 = 16,trainable_loops=2)\n    \n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=0.00002),\n        loss='categorical_crossentropy',  \n        metrics=['categorical_accuracy',f1_m])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_dataset, \n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    epochs=15,\n                    class_weight=c_weights(df_train[\"label\"],n=3/4),\n                    validation_data=valid_dataset,\n                    validation_steps=VALID_STEPS,\n                    callbacks=[early_stop,loop_schedule(0.0001)]\n                   )\nmodel.save('EffNet5_2.h5')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":1.326243,"end_time":"2020-11-19T22:05:02.628032","exception":false,"start_time":"2020-11-19T22:05:01.301789","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Making predictions\nNow that we've trained our model we can use it to make predictions! "},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True) ","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T22:05:07.746039Z","iopub.status.busy":"2020-11-19T22:05:07.744935Z","iopub.status.idle":"2020-11-19T22:05:22.234912Z","shell.execute_reply":"2020-11-19T22:05:22.235492Z"},"papermill":{"duration":15.776858,"end_time":"2020-11-19T22:05:22.235661","exception":false,"start_time":"2020-11-19T22:05:06.458803","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True) \ntest_ds = test_ds.map(to_float32)\n\nprint('Computing predictions...')\ntest_images_ds = test_ds\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":1.271799,"end_time":"2020-11-19T22:05:24.759257","exception":false,"start_time":"2020-11-19T22:05:23.487458","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Creating a submission file\nNow that we've trained a model and made predictions we're ready to submit to the competition! You can run the following code below to get your submission file."},{"metadata":{"execution":{"iopub.execute_input":"2020-11-19T22:05:27.316025Z","iopub.status.busy":"2020-11-19T22:05:27.315202Z","iopub.status.idle":"2020-11-19T22:05:28.241598Z","shell.execute_reply":"2020-11-19T22:05:28.24078Z"},"papermill":{"duration":2.185537,"end_time":"2020-11-19T22:05:28.241723","exception":false,"start_time":"2020-11-19T22:05:26.056186","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\n!head submission.csv","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":1.255302,"end_time":"2020-11-19T22:05:30.746339","exception":false,"start_time":"2020-11-19T22:05:29.491037","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Be aware that because this is a code competition with a hidden test set, internet and TPUs cannot be enabled on your submission notebook. Therefore TPUs will only be available for training models. For a walk-through on how to train on TPUs and run inference/submit on GPUs, see our [TPU Docs](https://www.kaggle.com/docs/tpu#tpu6)."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}