{"cells":[{"metadata":{},"cell_type":"markdown","source":"## About this notebook\n\nIn this notebook, I followed the tutorial notebook to build a machine learning model to classify 4 types of cassaval leaf disease and 1 healthy type based on their images.\n\nI am going to train the model on a Tensor Processing Unit (TPU).\n"},{"metadata":{},"cell_type":"markdown","source":"# Setups #"},{"metadata":{"trusted":true},"cell_type":"code","source":"## Basic packages\nimport math, re, os, random, warnings, glob, cv2, gc\nwarnings.simplefilter(action = 'ignore', category = FutureWarning)\n\n## For data handling\nimport pandas as pd\nimport numpy as np\nfrom functools import partial\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split\n## For plotting\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n## Tensorflow packages\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Activation, Dense, BatchNormalization, Conv2D, MaxPool2D, Flatten\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.metrics import categorical_crossentropy\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.applications import imagenet_utils\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\nimport tensorflow.keras.backend as K\n\nprint(\"Tensorflow version \" + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nSEED = 414\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect TPU, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() \n\nREPLICAS = strategy.num_replicas_in_sync\n\nprint(\"REPLICAS: \", REPLICAS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading Data #"},{"metadata":{"trusted":true},"cell_type":"code","source":"#from kaggle_datasets import KaggleDatasets\n\n#GCS_DS_PATH = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification')\nGCS_DS_PATH = '../input/cassava-leaf-disease-classification'\nprint(GCS_DS_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16 * REPLICAS\nWARMUP_EPOCHS = 3\nWARMUP_LEARNING_RATE = 1e-4 * REPLICAS\nEPOCHS = 20\nLEARNING_RATE = 5e-5 * REPLICAS\nES_PATIENCE = 5\n\nCHANNELS = 3\nN_CLASSES = 5\nDIM = 512\nHEIGHT = 512\nWIDTH = 512\nCLASSES = ['0', '1', '2', '3', '4']\n\n#model_path = f'model_efn.h5'\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROT_ = 180.0\nSHR_ = 2.0\nHZOOM_ = 8.0\nWZOOM_ = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform(image, DIM=512):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'target'                       : tf.io.FixedLenFeature([], tf.int64)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['target']\n\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['image_name'] if return_image_name else 0\n\n \ndef prepare_image(img, augment=True, dim=512):    \n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.cast(img, tf.float32) / 255.0\n    \n    if augment:\n        img = transform(img,DIM=dim)\n        img = tf.image.random_flip_left_right(img)\n        #img = tf.image.random_hue(img, 0.01)\n        #img = tf.image.random_saturation(img, 0.7, 1.3)\n        #img = tf.image.random_contrast(img, 0.8, 1.2)\n        img = tf.image.random_brightness(img, 0.1)\n                      \n    img = tf.reshape(img, [dim,dim, 3])\n            \n    return img\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(files, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_names=True, batch_size=BATCH_SIZE, dim=512):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n                    num_parallel_calls=AUTO)      \n    \n    ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment, dim=dim), \n                                               imgname_or_label), \n                num_parallel_calls=AUTO)\n    \n    ds = ds.batch(batch_size * REPLICAS)\n    ds = ds.prefetch(AUTO)\n    return ds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train test split"},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + '/test_tfrecords/*.tfrec')\n#TEST_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + '/train_tfrecords/ld_train08-1338.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(TEST_FILENAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nprint('Dataset: {} unlabeled test images'.format(NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle')\n! pip install /kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import DenseNet201, Xception, InceptionV3, and InceptionResNetV2\nimport efficientnet.keras as efn\n#from tensorflow.keras.applications import DenseNet201, Xception","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model_efnB6():\n    base_model = efn.EfficientNetB6(weights=None,\n                                          include_top=False,\n                                          input_shape=[HEIGHT, WIDTH, 3])\n    #base_model.trainable = False # Freeze layers\n    model = tf.keras.Sequential([\n        base_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n    ])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n     model_efnB6 = create_model_efnB6()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TTA = 1\nprint('Predicting Test with TTA...')\ntest_ds = get_dataset(TEST_FILENAMES,labeled=False,return_image_names=False,augment=False,\n                      repeat=False,shuffle=False)\ntest_ct = count_data_items(TEST_FILENAMES); \nSTEPS = TTA * test_ct/BATCH_SIZE/REPLICAS\nif STEPS < 1:\n    STEPS = 1\n\ntest_df = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nprobabilities = np.zeros((test_df.shape[0],5))\n\nfor f in range(5):\n    model_efnB6.load_weights(\"../input/cassava-leaf-disease-classification-kfold/model_efnB6_train_fold_%i.h5\"%f, by_name=True)\n    #prob = model_efnB4.predict(test_ds,steps=STEPS,verbose=2)[:TTA*test_ct,]\n    prob = model_efnB6.predict(test_ds, verbose=2)\n    probabilities += prob/5\n\n#model_efnB6.load_weights(\"../input/cassava-leaf-disease-classification-kfold/model_efnB6_train_fold_0.h5\", by_name=True)\n#probabilities = model_efnB6.predict(test_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\nprint(probabilities)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Generating submission.csv file...')\nds = get_dataset(TEST_FILENAMES,labeled=False,return_image_names=True,augment=False,\n                      repeat=False,shuffle=False)\n# Get image ids from test set and convert to unicode\ntest_ids = np.array([img_name.numpy().decode(\"utf-8\") \n                        for img, img_name in iter(ds.unbatch())])\n\n# Write the submission file\nnp.savetxt(\n    'submission.csv',\n    np.rec.fromarrays([test_ids, predictions]),\n    fmt=['%s', '%d'],\n    delimiter=',',\n    header='image_id,label',\n    comments='',\n)\n\n# Look at the first few predictions\n!head submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}