{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Importing the libraries"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#import cv2\nimport re, os, math\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport numpy as np # linear algebra\nimport pandas as pd \nimport tensorflow.keras.backend as K\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import models, layers\nfrom tensorflow.keras.layers import Input, Conv2D,ReLU, Flatten, Dense, MaxPool2D, concatenate, BatchNormalization, AveragePooling2D\nfrom tensorflow.keras.layers import Activation\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import CategoricalCrossentropy\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, LearningRateScheduler\nfrom tensorflow.keras.applications import ResNet50, DenseNet121, InceptionResNetV2\n#from tensorflow.keras.applications import EfficientNetB3\nfrom kaggle_datasets import KaggleDatasets\nfrom functools import partial","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Checking if TPU is connected"},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setting up the variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nWORK_DIR = '/kaggle/input/cassava-leaf-disease-classification/'\nGCS_PATH = KaggleDatasets().get_gcs_path()\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [512, 512]\nI_SIZE = 512\nCLASSES = ['0', '1', '2', '3', '4']\nEPOCHS = 25\nHEIGHT = 512\nWIDTH = 512\nCHANNELS = 3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Decoding the image file to tensor, normalising it and reshaping it."},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels = 3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image,[*IMAGE_SIZE,3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(\n    tf.io.gfile.glob(GCS_PATH + '/train_tfrecords/ld_train*.tfrec'),\n    test_size=0.35, random_state=5\n)\n\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test_tfrecords/ld_test*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    # Shear\n    if p_shear > .2:\n        if p_shear > .6:\n            image = transform_shear(image, HEIGHT, shear=20.)\n        else:\n            image = transform_shear(image, HEIGHT, shear=-20.)\n            \n    # Rotation\n    if p_rotation > .2:\n        if p_rotation > .6:\n            image = transform_rotation(image, HEIGHT, rotation=45.)\n        else:\n            image = transform_rotation(image, HEIGHT, rotation=-45.)\n            \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > .75:\n        image = tf.image.transpose(image)\n        \n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n        \n    # Pixel-level transforms\n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.5, upper=0.9)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.5, upper=0.9)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n        \n    # Crops\n    if p_crop > .7:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.6)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.8)\n    elif p_crop > .4:\n        crop_size = tf.random.uniform([], int(HEIGHT*.6), HEIGHT, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n            \n    image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loading the dataset to TPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=ordered) \n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model building."},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_scheduler = tf.keras.optimizers.schedules.ExponentialDecay(\n    initial_learning_rate=1e-5, \n    decay_steps=10000, \n    decay_rate=0.9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=7, verbose=1,\n                              mode='min', restore_best_weights=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_check = ModelCheckpoint(\n                            \"./firstTry.h5\",\n                            monitor = \"val_loss\",\n                            verbose = 1,\n                            save_best_only = True,\n                            save_weights_only = False,\n                            mode = \"min\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():       \n    img_adjust_layer = tf.keras.layers.Lambda(tf.keras.applications.resnet50.preprocess_input, input_shape=[*IMAGE_SIZE, 3])\n    \n    base_model = tf.keras.applications.ResNet50(weights='imagenet', include_top=False)\n    base_model.trainable = True\n    \n    model = tf.keras.Sequential([\n        tf.keras.layers.BatchNormalization(renorm=True),\n        img_adjust_layer,\n        base_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(1024, activation='relu'),\n        tf.keras.layers.BatchNormalization(renorm=True),\n        tf.keras.layers.Dense(1024, activation='relu'),\n        #tf.keras.layers.Dropout(0.5)(x)\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax')  \n    ])\n    \n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=lr_scheduler, epsilon=0.001),\n        loss='sparse_categorical_crossentropy',  \n        metrics=['sparse_categorical_accuracy'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get the dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_dataset()\nvalid_dataset = get_validation_dataset()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Resnet50 model"},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_dataset, \n                    steps_per_epoch=104, \n                    epochs=50,\n                    validation_data=valid_dataset,\n                    validation_steps=62, callbacks=[early_stopping])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_frame = pd.DataFrame(history.history)\nhistory_frame.loc[:, ['loss', 'val_loss']].plot()\nhistory_frame.loc[:, ['sparse_categorical_accuracy', 'val_sparse_categorical_accuracy']].plot();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get predictions from resnet50"},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_float32(image, label):\n    return tf.cast(image, tf.float32), label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True) \ntest_ds = test_ds.map(to_float32)\ntest_images_ds = test_ds\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\npredictions_Resnet = np.argmax(probabilities, axis=-1)\nprint(predictions_Resnet)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Building a DenseNet Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():       \n    img_adjust_layer = tf.keras.layers.Lambda(tf.keras.applications.densenet.preprocess_input, input_shape=[*IMAGE_SIZE, 3])\n    \n    base_model = tf.keras.applications.DenseNet169(weights='imagenet', include_top=False)\n    base_model.trainable = True\n    \n    model_Dense = tf.keras.Sequential([\n        tf.keras.layers.BatchNormalization(renorm=True),\n        img_adjust_layer,\n        base_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(1024, activation='relu'),\n        #tf.keras.layers.BatchNormalization(renorm=True),\n        tf.keras.layers.Dense(1024, activation='relu'),\n        #tf.keras.layers.Dropout(0.5)(x)\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax')  \n    ])\n    \n    model_Dense.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=lr_scheduler, epsilon=0.001),\n        loss='sparse_categorical_crossentropy',  \n        metrics=['sparse_categorical_accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_Dense.fit(train_dataset, \n                    steps_per_epoch=104, \n                    epochs=50,\n                    validation_data=valid_dataset,\n                    validation_steps=62)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_frame = pd.DataFrame(history.history)\nhistory_frame.loc[:, ['loss', 'val_loss']].plot()\nhistory_frame.loc[:, ['sparse_categorical_accuracy', 'val_sparse_categorical_accuracy']].plot();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction from Densenet model"},{"metadata":{"trusted":true},"cell_type":"code","source":"probabilities = model_Dense.predict(test_images_ds)\npredictions_Densenet = np.argmax(probabilities, axis=-1)\nprint(predictions_Densenet)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Building a InceptionResV2 model"},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():       \n    img_adjust_layer = tf.keras.layers.Lambda(tf.keras.applications.inception_resnet_v2.preprocess_input, input_shape=[*IMAGE_SIZE, 3])\n    \n    base_model = tf.keras.applications.InceptionResNetV2(weights='imagenet', include_top=False)\n    base_model.trainable = True\n    \n    model_incept = tf.keras.Sequential([\n        tf.keras.layers.BatchNormalization(renorm=True),\n        img_adjust_layer,\n        base_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(1024, activation='relu'),\n        tf.keras.layers.BatchNormalization(renorm=True),\n        tf.keras.layers.Dense(1024, activation='relu'),\n        #tf.keras.layers.Dropout(0.5)(x)\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax')  \n    ])\n    \n    model_incept.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=lr_scheduler, epsilon=0.001),\n        loss='sparse_categorical_crossentropy',  \n        metrics=['sparse_categorical_accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_incept.fit(train_dataset, \n                    steps_per_epoch=104, \n                    epochs=50,\n                    validation_data=valid_dataset,\n                    validation_steps=62)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_frame = pd.DataFrame(history.history)\nhistory_frame.loc[:, ['loss', 'val_loss']].plot()\nhistory_frame.loc[:, ['sparse_categorical_accuracy', 'val_sparse_categorical_accuracy']].plot();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction from InceptionResV2"},{"metadata":{"trusted":true},"cell_type":"code","source":"probabilities = model_Dense.predict(test_images_ds)\npredictions_incept = np.argmax(probabilities, axis=-1)\nprint(predictions_incept)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list_pred = [predictions_Resnet[0],predictions_incept[0],predictions_Densenet[0]]\nfinal_pred = max(set(list_pred), key=list_pred.count)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}