{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\nGCS_PATH = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-50-tfrecords-center-{HEIGHT}x{WIDTH}') # Center croped and resized (50 TFRecord)\nGCS_PATH_EXT = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-50-tfrecords-external-{HEIGHT}x{WIDTH}') # Center croped and resized (50 TFRecord) (External)\nGCS_PATH_CLASSES = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-50-tfrecords-classes-{HEIGHT}x{WIDTH}') # Center croped and resized (50 TFRecord) by classes\nGCS_PATH_EXT_CLASSES = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-ext-50-tfrec-classes-{HEIGHT}x{WIDTH}')\n\nprint ('database_base_path=',KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification'))\nprint ('GCS_PATH=',GCS_PATH)\nprint ('GCS_PATH_EXT=',GCS_PATH_EXT)\nprint ('GCS_PATH_CLASSES=',GCS_PATH_CLASSES)\nprint ('GCS_PATH_EXT_CLASSES=',GCS_PATH_EXT_CLASSES)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hi to all!!!\n# I have prepared a notebook that works on both COLAB and KAGGLE!!!","metadata":{}},{"cell_type":"markdown","source":"I took this notebook as a basis: https://www.kaggle.com/dimitreoliveira/cassava-leaf-disease-training-with-tpu-v2-pods","metadata":{}},{"cell_type":"markdown","source":"It is important to work with colab you need kaggle.json (https://www.kaggle.com/docs/api)\n\nI also created a folder called Models on my Google Drive and put the kaggle.json file in it.","metadata":{}},{"cell_type":"markdown","source":"Improvements\n - `Custom training loop`: Using a custom training loop greatly improves the training time and resource usage.\n - `Maximize MXU and minimize Idle time`: I have made a few adjustments to the Tensorflow pipeline to improve performance.\n\nExperiments\n - Small improvements using external data (2019 competition).\n - Small improvements from using `CCE label smoothing`.\n - Small improvements from using `CutOut`.\n - Small improvements from `oversmapling` classes `0`, `1`, `2` and `4`.\n - Small improvements from keeping `batch normalization` layers frozen.\n - No relevant improvements from using `class weights`.\n - No relevant improvements from using `MixUp`.\n - No relevant improvements from using different backbones.\n - Worse performance by using different image resolution even the default `EfficientNet` input size.\n - Changing `Sparse CCE` to `CCE` has no impact, as expected.\n - Was not able to make progressive unfreezing work.\n - Changing the `learning rate` batch wise seems more efficent than epoch wise, specially for the warm up phase.","metadata":{"id":"-zWNGAsZCaOu"}},{"cell_type":"code","source":"EPOCHS = 1#10 \nHEIGHT = 512\nWIDTH = 512\nHEIGHT_RS = 512\nWIDTH_RS = 512\nCHANNELS = 3\nN_CLASSES = 5\nN_FOLDS = 5\nFOLDS_USED = 5\nES_PATIENCE = 5","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nCOLAB=False\nmodels_path=''\n\nif not os.path.exists('../input/cassava-leaf-disease-classification'):# Let's check if we use kaggle notebook\n    COLAB=True\n    import gc\n    !pip install fsspec\n    !pip install gcsfs \n    !pip install --upgrade --force-reinstall --no-deps kaggle\n\n    from google.colab import drive\n    drive.mount('/content/drive')# You must grant COLAB access to your Google Drive\n    \n    #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n    \n    database_base_path = 'gs://kds-8a8a0e757020ef17f93b37a540d540ccfa0003dbd9620ed6ef47ea9b/'\n    GCS_PATH = 'gs://kds-fbe3004cbea028fa22b3442390e2fa87a1671221c12f3a625d68ec8b'\n    GCS_PATH_EXT = 'gs://kds-ab7b916763a2655169633af5e6c92ce88670b266c929f78a733ea228'\n    GCS_PATH_CLASSES = 'gs://kds-ebf0f9ac962adcdab49275d8fb50516ecedc4b726a0944f99b4ff60f'\n    GCS_PATH_EXT_CLASSES = 'gs://kds-62aaac1dd1d2587cfc0981ec461352d1527965ab1e895d7d253a8816'\n    \n    \n    # \n    #This is a path to a dataset that changes over time, so you need to constantly update it. To update the path just run the code: \n    #GCS_DS_PATH = KaggleDatasets (). Get_gcs_path ()\n    #print (GCS_PATH)......\n    models_path='/content/drive/MyDrive/Models/Cassava/'# I created a folder called Models/Cassava on my Google Drive\nelse:\n    from kaggle_datasets import KaggleDatasets\n    database_base_path = '/kaggle/input/cassava-leaf-disease-classification/'\n    GCS_PATH = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-50-tfrecords-center-{HEIGHT}x{WIDTH}') # Center croped and resized (50 TFRecord)\n    GCS_PATH_EXT = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-50-tfrecords-external-{HEIGHT}x{WIDTH}') # Center croped and resized (50 TFRecord) (External)\n    GCS_PATH_CLASSES = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-50-tfrecords-classes-{HEIGHT}x{WIDTH}') # Center croped and resized (50 TFRecord) by classes\n    GCS_PATH_EXT_CLASSES = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-ext-50-tfrec-classes-{HEIGHT}x{WIDTH}')\n    print ('database_base_path', KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification'))\n    print ('GCS_PATH',GCS_PATH)\n    print ('GCS_PATH_EXT',GCS_PATH_EXT)\n    print ('GCS_PATH_CLASSES',GCS_PATH_CLASSES)\n    print ('GCS_PATH_EXT_CLASSES',GCS_PATH_EXT_CLASSES)","metadata":{"id":"YZBauHrJCkn7","outputId":"75ff8b73-7ecd-406b-d335-07749a0ebe4f","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if COLAB:# Prepare the kaggle.json file for use \n    from google.colab import files\n    if not os.path.exists('/.kaggle/kaggle.json'):\n        !mkdir ~/.kaggle\n        if not os.path.exists('/content/drive/My Drive/Models/kaggle.json'):\n            files.upload()\n            !cp kaggle.json ~/.kaggle/\n        else:\n            !cp '/content/drive/My Drive/Models/kaggle.json' ~/.kaggle/  \n        !chmod 600 ~/.kaggle/kaggle.json\n","metadata":{"id":"5V15UaoACh-M","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if COLAB:# force TF to 2.2\n    !pip install -q tensorflow~=2.2.0 tensorflow_gcs_config~=2.2.0\n    \n    import requests\n    import os\n    resp = requests.post(\"http://{}:8475/requestversion/{}\".format(os.environ[\"COLAB_TPU_ADDR\"].split(\":\")[0], tf.__version__))\n    if resp.status_code != 200:\n      print(\"Failed to switch the TPU to TF {}\".format(version))","metadata":{"id":"urYjsB4IV-t5","outputId":"eaa751df-e422-4f7e-e9b3-61c79f4b549d","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dependencies","metadata":{"id":"DJLH967uCaOv"}},{"cell_type":"code","source":"!pip install --quiet efficientnet","metadata":{"id":"u8lneueOCaOv","outputId":"158c8e2d-18d1-4b41-ee26-3d6387842baa","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math, os, re, warnings, random, time\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import classification_report, confusion_matrix\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras import optimizers, Sequential, losses, metrics, Model\nfrom tensorflow.keras.callbacks import EarlyStopping\n#import efficientnet.tfkeras as efn\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","metadata":{"id":"-iMoIRz7CaOw","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hardware configuration\n\nNote that we have `32` cores, this is because the `TPU v2 Pod` have more cores than a single `TPU v3` which has `8` cores.","metadata":{"id":"Z-146lSGCaOx"}},{"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n    print ('tpu',tpu)\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"id":"cCFwEwkVCaOy","outputId":"9eccc6c3-2e0d-4938-8e64-80cf9b520759","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model parameters","metadata":{"id":"NK8TAx9JCaOz"}},{"cell_type":"code","source":"BATCH_SIZE = 8 * REPLICAS\nLEARNING_RATE = 1e-5 * REPLICAS","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{"id":"UmwrOwlYCaOz"}},{"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r'-([0-9]*)\\.').search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\n#database_base_path = '/kaggle/input/cassava-leaf-disease-classification/'\n\ntrain = pd.read_csv(f'{database_base_path}train.csv')\nprint(f'Train samples: {len(train)}')\n\n# GCS_PATH = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification') # Original dataset\n#GCS_PATH = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-50-tfrecords-center-{HEIGHT}x{WIDTH}') # Center croped and resized (50 TFRecord)\n#GCS_PATH_EXT = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-50-tfrecords-external-{HEIGHT}x{WIDTH}') # Center croped and resized (50 TFRecord) (External)\n#GCS_PATH_CLASSES = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-50-tfrecords-classes-{HEIGHT}x{WIDTH}') # Center croped and resized (50 TFRecord) by classes\n#GCS_PATH_EXT_CLASSES = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-ext-50-tfrec-classes-{HEIGHT}x{WIDTH}') # Center croped and resized (50 TFRecord) (External) by classes\n\n# FILENAMES_COMP = tf.io.gfile.glob(GCS_PATH + '/train_tfrecords/*.tfrec') # Original TFRecords\nFILENAMES_COMP = tf.io.gfile.glob(GCS_PATH + '/*.tfrec')\nFILENAMES_2019 = tf.io.gfile.glob(GCS_PATH_EXT + '/*.tfrec')\n\nFILENAMES_COMP_CBB = tf.io.gfile.glob(GCS_PATH_CLASSES + '/CBB*.tfrec')\nFILENAMES_COMP_CBSD = tf.io.gfile.glob(GCS_PATH_CLASSES + '/CBSD*.tfrec')\nFILENAMES_COMP_CGM = tf.io.gfile.glob(GCS_PATH_CLASSES + '/CGM*.tfrec')\nFILENAMES_COMP_CMD = tf.io.gfile.glob(GCS_PATH_CLASSES + '/CMD*.tfrec')\nFILENAMES_COMP_Healthy = tf.io.gfile.glob(GCS_PATH_CLASSES + '/Healthy*.tfrec')\n\nFILENAMES_2019_CBB = tf.io.gfile.glob(GCS_PATH_EXT_CLASSES + '/CBB*.tfrec')\nFILENAMES_2019_CBSD = tf.io.gfile.glob(GCS_PATH_EXT_CLASSES + '/CBSD*.tfrec')\nFILENAMES_2019_CGM = tf.io.gfile.glob(GCS_PATH_EXT_CLASSES + '/CGM*.tfrec')\nFILENAMES_2019_CMD = tf.io.gfile.glob(GCS_PATH_EXT_CLASSES + '/CMD*.tfrec')\nFILENAMES_2019_Healthy = tf.io.gfile.glob(GCS_PATH_EXT_CLASSES + '/Healthy*.tfrec')\n\n\nTRAINING_FILENAMES = (FILENAMES_COMP + \n                      FILENAMES_2019 + \n                      (2 * FILENAMES_COMP_CBB) + \n                      (2 * FILENAMES_2019_CBB) + \n                      (2 * FILENAMES_COMP_CBSD) + \n                      (2 * FILENAMES_2019_CBSD) + \n                      (2 * FILENAMES_COMP_CGM) + \n                      (2 * FILENAMES_2019_CGM) + \n                      (2 * FILENAMES_COMP_Healthy) + \n                      (2 * FILENAMES_2019_Healthy))\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\n\nprint(f'GCS: train images: {NUM_TRAINING_IMAGES}')\ndisplay(train.head())\n\nCLASSES = ['Cassava Bacterial Blight', \n           'Cassava Brown Streak Disease', \n           'Cassava Green Mottle', \n           'Cassava Mosaic Disease', \n           'Healthy']","metadata":{"id":"UxM4MGNGCaO0","outputId":"e51cb972-bc4c-4b10-bf00-efd483772c8e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentation","metadata":{"id":"BZIhYsg_CaO1"}},{"cell_type":"code","source":"def data_augment(image, label):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_cutout = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    # Shear\n    if p_shear > .2:\n        if p_shear > .6:\n            image = transform_shear(image, HEIGHT, shear=20.)\n        else:\n            image = transform_shear(image, HEIGHT, shear=-20.)\n            \n    # Rotation\n    if p_rotation > .2:\n        if p_rotation > .6:\n            image = transform_rotation(image, HEIGHT, rotation=45.)\n        else:\n            image = transform_rotation(image, HEIGHT, rotation=-45.)\n            \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > .75:\n        image = tf.image.transpose(image)\n        \n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n        \n    # Pixel-level transforms\n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n        \n    # Crops\n    if p_crop > .6:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.5)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.6)\n        elif p_crop > .7:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.8)\n    elif p_crop > .3:\n        crop_size = tf.random.uniform([], int(HEIGHT*.6), HEIGHT, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n            \n    image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n\n    if p_cutout > .5:\n        image = data_augment_cutout(image)\n        \n    return image, label","metadata":{"id":"Wwh329L6CaO1","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Auxiliary functions","metadata":{"id":"r4Hg1nKgCaO1"}},{"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\n# CutOut\ndef data_augment_cutout(image, min_mask_size=(int(HEIGHT * .1), int(HEIGHT * .1)), \n                        max_mask_size=(int(HEIGHT * .125), int(HEIGHT * .125))):\n    p_cutout = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    if p_cutout > .85: # 10~15 cut outs\n        n_cutout = tf.random.uniform([], 10, 15, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    elif p_cutout > .6: # 5~10 cut outs\n        n_cutout = tf.random.uniform([], 5, 10, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    elif p_cutout > .25: # 2~5 cut outs\n        n_cutout = tf.random.uniform([], 2, 5, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    else: # 1 cut out\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=1)\n\n    return image\n\ndef random_cutout(image, height, width, channels=3, min_mask_size=(10, 10), max_mask_size=(80, 80), k=1):\n    assert height > min_mask_size[0]\n    assert width > min_mask_size[1]\n    assert height > max_mask_size[0]\n    assert width > max_mask_size[1]\n\n    for i in range(k):\n      mask_height = tf.random.uniform(shape=[], minval=min_mask_size[0], maxval=max_mask_size[0], dtype=tf.int32)\n      mask_width = tf.random.uniform(shape=[], minval=min_mask_size[1], maxval=max_mask_size[1], dtype=tf.int32)\n\n      pad_h = height - mask_height\n      pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)\n      pad_bottom = pad_h - pad_top\n\n      pad_w = width - mask_width\n      pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)\n      pad_right = pad_w - pad_left\n\n      cutout_area = tf.zeros(shape=[mask_height, mask_width, channels], dtype=tf.uint8)\n\n      cutout_mask = tf.pad([cutout_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)\n      cutout_mask = tf.squeeze(cutout_mask, axis=0)\n      image = tf.multiply(tf.cast(image, tf.float32), tf.cast(cutout_mask, tf.float32))\n\n    return image","metadata":{"id":"-DueNRgfCaO1","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Datasets utility functions\ndef decode_image(image_data):\n    \"\"\"\n        Decode a JPEG-encoded image to a uint8 tensor.\n    \"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    return image\n\ndef scale_image(image, label):\n    \"\"\"\n        Cast tensor to float and normalizes (range between 0 and 1).\n    \"\"\"\n    image = tf.cast(image, tf.float32)\n    image /= 255.0\n    return image, label\n\ndef prepare_image(image, label):\n    \"\"\"\n        Resize and reshape images to the expected size.\n    \"\"\"\n    image = tf.image.resize(image, [HEIGHT_RS, WIDTH_RS])\n    image = tf.reshape(image, [HEIGHT_RS, WIDTH_RS, 3])\n    return image, label\n\ndef read_tfrecord(example, labeled=True):\n    \"\"\"\n        1. Parse data based on the 'TFREC_FORMAT' map.\n        2. Decode image.\n        3. If 'labeled' returns (image, label) if not (image, name).\n    \"\"\"\n    if labeled:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'target': tf.io.FixedLenFeature([], tf.int64), \n        }\n    else:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'image_name': tf.io.FixedLenFeature([], tf.string), \n        }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    if labeled:\n        label_or_name = tf.cast(example['target'], tf.int32)\n        # One-Hot Encoding needed to use \"categorical_crossentropy\" loss\n        label_or_name = tf.one_hot(tf.cast(label_or_name, tf.int32), N_CLASSES)\n    else:\n        label_or_name = example['image_name']\n    return image, label_or_name\n\ndef get_dataset(FILENAMES, labeled=True, ordered=False, repeated=False, \n                cached=False, augment=False):\n    \"\"\"\n        Return a Tensorflow dataset ready for training or inference.\n    \"\"\"\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n        dataset = tf.data.Dataset.list_files(FILENAMES)\n        dataset = dataset.interleave(tf.data.TFRecordDataset, num_parallel_calls=AUTO)\n    else:\n        dataset = tf.data.TFRecordDataset(FILENAMES, num_parallel_reads=AUTO)\n        \n    dataset = dataset.with_options(ignore_order)\n    \n    dataset = dataset.map(lambda x: read_tfrecord(x, labeled=labeled), num_parallel_calls=AUTO)\n    \n    if augment:\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n        \n    dataset = dataset.map(scale_image, num_parallel_calls=AUTO)\n    dataset = dataset.map(prepare_image, num_parallel_calls=AUTO)\n    \n    if not ordered:\n        dataset = dataset.shuffle(2048)\n    if repeated:\n        dataset = dataset.repeat()\n        \n    dataset = dataset.batch(BATCH_SIZE)\n    \n    if cached:\n        dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef unfreeze_model(model):\n    # Unfreeze layers while leaving BatchNorm layers frozen\n    for layer in model.layers:\n        if not isinstance(layer, L.BatchNormalization):\n            layer.trainable = True\n        else:\n            layer.trainable = False\n                \ndef unfreeze_block(model, block_name=None, n_top=3):\n    # Unfreeze layers while leaving BatchNorm layers frozen\n    for layer in model.layers[:-n_top]:\n        if isinstance(layer, L.BatchNormalization):\n            layer.trainable = False\n        else:\n            if block_name and (block_name in layer.name):\n                layer.trainable = True","metadata":{"id":"CPcg0bWECaO1","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualization utility functions\nnp.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', \n                  fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n\ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    labels = np.argmax(labels, axis=-1)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n    \n# Visualize model predictions\ndef dataset_to_numpy_util(dataset, N):\n    dataset = dataset.unbatch().batch(N)\n    for images, labels in dataset:\n        numpy_images = images.numpy()\n        numpy_labels = labels.numpy()\n        break;  \n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    label = np.argmax(label, axis=-1)\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(label, str(correct), ', shoud be ' if not correct else '',\n                                correct_label if not correct else ''), correct\n\ndef display_one_flower_eval(image, title, subplot, red=False):\n    plt.subplot(subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    plt.title(title, fontsize=14, color='red' if red else 'black')\n    return subplot+1\n\ndef display_9_images_with_predictions(images, predictions, labels):\n    subplot=331\n    plt.figure(figsize=(13,13))\n    for i, image in enumerate(images):\n        title, correct = title_from_label_and_target(predictions[i], labels[i])\n        subplot = display_one_flower_eval(image, title, subplot, not correct)\n        if i >= 8:\n            break;\n              \n    plt.tight_layout()\n    plt.subplots_adjust(wspace=0.1, hspace=0.1)\n    plt.show()\n\n\n# Model evaluation\ndef plot_metrics(history):\n    fig, axes = plt.subplots(2, 1, sharex='col', figsize=(20, 8))\n    axes = axes.flatten()\n    \n    axes[0].plot(history['loss'], label='Train loss')\n    axes[0].plot(history['val_loss'], label='Validation loss')\n    axes[0].legend(loc='best', fontsize=16)\n    axes[0].set_title('Loss')\n    axes[0].axvline(np.argmin(history['loss']), linestyle='dashed')\n    axes[0].axvline(np.argmin(history['val_loss']), linestyle='dashed', color='orange')\n    \n    axes[1].plot(history['accuracy'], label='Train accuracy')\n    axes[1].plot(history['val_accuracy'], label='Validation accuracy')\n    axes[1].legend(loc='best', fontsize=16)\n    axes[1].set_title('Accuracy')\n    axes[1].axvline(np.argmax(history['accuracy']), linestyle='dashed')\n    axes[1].axvline(np.argmax(history['val_accuracy']), linestyle='dashed', color='orange')\n\n    plt.xlabel('Epochs', fontsize=16)\n    sns.despine()\n    plt.show()","metadata":{"id":"gHAxNOInCaO2","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training data samples (with augmentation)","metadata":{"id":"oCtCB3WdCaO2"}},{"cell_type":"code","source":"train_dataset = get_dataset(FILENAMES_COMP, ordered=True, augment=True)\ntrain_iter = iter(train_dataset.unbatch().batch(20))\n\ndisplay_batch_of_images(next(train_iter))\ndisplay_batch_of_images(next(train_iter))","metadata":{"id":"p8IMzXLRCaO2","outputId":"a662d8ae-ccdc-40c5-921e-b4d55f3668f9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Datasets distribution\n\n### Competition data","metadata":{"id":"gfrU-uCiCaO3"}},{"cell_type":"code","source":"ds_comp = get_dataset(FILENAMES_COMP)\nlabels_comp = [target.numpy() for img, target in iter(ds_comp.unbatch())]\nlabels_comp = np.argmax(labels_comp, axis=-1)\n\nfig, ax = plt.subplots(1, 1, figsize=(18, 8))\nax = sns.countplot(y=labels_comp, palette='viridis')\nax.tick_params(labelsize=16)\n\nplt.show()","metadata":{"id":"WJywiXKHCaO4","outputId":"6e5abb28-d75c-4f4b-bd1a-e2b06f0c5756","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2019 competition data","metadata":{"id":"H03votk7CaO4"}},{"cell_type":"code","source":"ds_2019 = get_dataset(FILENAMES_2019)\nlabels_2019 = [target.numpy() for img, target in iter(ds_2019.unbatch())]\nlabels_2019 = np.argmax(labels_2019, axis=-1)\n\nfig, ax = plt.subplots(1, 1, figsize=(18, 8))\nax = sns.countplot(y=labels_2019, palette='viridis')\nax.tick_params(labelsize=16)\n\nplt.show()","metadata":{"id":"SfD2fI73CaO4","outputId":"0c9d1b77-9527-4f4c-f1f7-90d7c1fecfd4","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset oversampled","metadata":{"id":"ZXY3rsY4CaO4"}},{"cell_type":"code","source":"FILENAMES_COMP_OVER = (FILENAMES_COMP + \n                       FILENAMES_2019 + \n                       (2 * FILENAMES_COMP_CBB) + \n                       (2 * FILENAMES_2019_CBB) + \n                       (2 * FILENAMES_COMP_CBSD) + \n                       (2 * FILENAMES_2019_CBSD) + \n                       (2 * FILENAMES_COMP_CGM) + \n                       (2 * FILENAMES_2019_CGM) + \n                       (2 * FILENAMES_COMP_Healthy) + \n                       (2 * FILENAMES_2019_Healthy))\n\nds_comp = get_dataset(FILENAMES_COMP_OVER)\nlabels_comp = [target.numpy() for img, target in iter(ds_comp.unbatch())]\nlabels_comp = np.argmax(labels_comp, axis=-1)\n\nfig, ax = plt.subplots(1, 1, figsize=(18, 8))\nax = sns.countplot(y=labels_comp, palette='viridis')\nax.tick_params(labelsize=16)\n\nplt.show()","metadata":{"id":"PRXTNO7RCaO5","outputId":"b611c8c2-e016-4993-c53e-8895d35c5724","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Learning rate schedule\n\nWe are going to use a `cosine learning rate schedule with a warm-up phase`, this may be a good idea since we are using a pre-trained model, the warm-up phase will be useful to avoid the pre-trained weights degradation resulting in catastrophic forgetting, during the schedule the learning rate will slowly decrease to very low values, this helps the model to land on more stable weights.","metadata":{"id":"HvXRzhofCaO5"}},{"cell_type":"code","source":"lr_start = 1e-8\nlr_min = 1e-8\nlr_max = LEARNING_RATE\nnum_cycles = 1.\nwarmup_epochs = 1\nhold_max_epochs = 0\ntotal_epochs = EPOCHS\nwarmup_steps = warmup_epochs * (NUM_TRAINING_IMAGES//BATCH_SIZE)\ntotal_steps = total_epochs * (NUM_TRAINING_IMAGES//BATCH_SIZE)\n\n@tf.function\ndef lrfn(step):\n    if step < warmup_steps:\n        lr = (lr_max - lr_start) / warmup_steps * step + lr_start\n    else:\n        progress = (step - warmup_steps) / (total_steps - warmup_steps)\n        lr = lr_max * (0.5 * (1.0 + tf.math.cos(np.pi * ((num_cycles * progress) % 1.0))))\n        if lr_min is not None:\n            lr = tf.math.maximum(lr_min, float(lr))\n\n    return lr\n\n\n# rng = [i for i in range(total_epochs)]\nrng = [i for i in range(total_steps)]\ny = [lrfn(tf.cast(x, tf.float32)) for x in rng]\n\nsns.set(style='whitegrid')\nfig, ax = plt.subplots(figsize=(20, 6))\nplt.plot(rng, y)\n\nprint(f'{total_steps} total steps and {NUM_TRAINING_IMAGES//BATCH_SIZE} steps per epoch')\nprint(f'Learning rate schedule: {y[0]:.3g} to {max(y):.3g} to {y[-1]:.3g}')","metadata":{"id":"I9cuJCl1CaO5","outputId":"abf16927-4b79-40b5-bd80-12b04cf3ec04","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"id":"OMIrA5jxCaO5"}},{"cell_type":"code","source":"def model_fn(input_shape, N_CLASSES):\n    inputs = L.Input(shape=input_shape, name='input_image')\n    base_model = efn.EfficientNetB4(input_tensor=inputs, \n                                    include_top=False, \n                                    weights='noisy-student', \n                                    pooling='avg')\n    base_model.trainable = False\n\n    x = L.Dropout(.5)(base_model.output)\n    output = L.Dense(N_CLASSES, activation='softmax', name='output')(x)\n    model = Model(inputs=inputs, outputs=output)\n\n    return model","metadata":{"id":"pRksD6K6CaO5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#help(strategy)","metadata":{"id":"4zl0qy-pcR-A","outputId":"690906ad-084e-4d98-c224-3b41870afad2","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"id":"hMc0tk0VCaO5"}},{"cell_type":"code","source":"skf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)\noof_pred = []; oof_labels = []; history_list = []\n\nfor fold,(idxT, idxV) in enumerate(skf.split(np.arange(50))):\n    if fold >= FOLDS_USED:\n        break\n    if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    K.clear_session()\n    print(f'\\nFOLD: {fold+1}')\n    print(f'TRAIN: {idxT} VALID: {idxV}')\n\n    # Create train and validation sets\n    FILENAMES_COMP = tf.io.gfile.glob([GCS_PATH + '/Id_train%.2i*.tfrec' % x for x in idxT])\n    FILENAMES_2019 = tf.io.gfile.glob([GCS_PATH_EXT + '/Id_train%.2i*.tfrec' % x for x in idxT])\n\n    FILENAMES_COMP_CBB = tf.io.gfile.glob([GCS_PATH_CLASSES + '/CBB%.2i*.tfrec' % x for x in idxT])\n    FILENAMES_COMP_CBSD = tf.io.gfile.glob([GCS_PATH_CLASSES + '/CBSD%.2i*.tfrec' % x for x in idxT])\n    FILENAMES_COMP_CGM = tf.io.gfile.glob([GCS_PATH_CLASSES + '/CGM%.2i*.tfrec' % x for x in idxT])\n    FILENAMES_COMP_Healthy = tf.io.gfile.glob([GCS_PATH_CLASSES + '/Healthy%.2i*.tfrec' % x for x in idxT])\n    \n    FILENAMES_2019_CBB = tf.io.gfile.glob([GCS_PATH_EXT_CLASSES + '/CBB%.2i*.tfrec' % x for x in idxT])\n    FILENAMES_2019_CBSD = tf.io.gfile.glob([GCS_PATH_EXT_CLASSES + '/CBSD%.2i*.tfrec' % x for x in idxT])\n    FILENAMES_2019_CGM = tf.io.gfile.glob([GCS_PATH_EXT_CLASSES + '/CGM%.2i*.tfrec' % x for x in idxT])\n    FILENAMES_2019_Healthy = tf.io.gfile.glob([GCS_PATH_EXT_CLASSES + '/Healthy%.2i*.tfrec' % x for x in idxT])\n\n    TRAIN_FILENAMES = (FILENAMES_COMP + \n                       FILENAMES_2019 + \n                       (2 * FILENAMES_COMP_CBB) + \n                       (2 * FILENAMES_2019_CBB) + \n                       (2 * FILENAMES_COMP_CBSD) + \n                       (2 * FILENAMES_2019_CBSD) + \n                       (2 * FILENAMES_COMP_CGM) + \n                       (2 * FILENAMES_2019_CGM) + \n                       (2 * FILENAMES_COMP_Healthy) + \n                       (2 * FILENAMES_2019_Healthy))\n    \n    VALID_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/Id_train%.2i*.tfrec' % x for x in idxV])\n    np.random.shuffle(TRAIN_FILENAMES)\n    \n    ct_train = count_data_items(TRAIN_FILENAMES)\n    ct_valid = count_data_items(VALID_FILENAMES)\n    \n    step_size = (ct_train // BATCH_SIZE)\n    valid_step_size = (ct_valid // BATCH_SIZE)\n    total_steps=(total_epochs * step_size)\n    warmup_steps=(warmup_epochs * step_size)\n    \n    \n    # Build TF datasets\n    train_ds = strategy.experimental_distribute_dataset(get_dataset(TRAIN_FILENAMES, repeated=True, augment=True))\n    valid_ds = strategy.experimental_distribute_dataset(get_dataset(VALID_FILENAMES, ordered=True, repeated=True, cached=True))\n    train_data_iter = iter(train_ds)\n    valid_data_iter = iter(valid_ds)\n    \n    \n    # Step functions\n    @tf.function\n    def train_step(data_iter):\n        def train_step_fn(x, y):\n            with tf.GradientTape() as tape:\n                probabilities = model(x, training=True)\n                loss = loss_fn(y, probabilities, label_smoothing=.3)\n            gradients = tape.gradient(loss, model.trainable_variables)\n            optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n            # update metrics\n            train_accuracy.update_state(y, probabilities)\n            train_loss.update_state(loss)\n        for _ in tf.range(step_size):\n            if COLAB:\n                #print(COLAB)\n                strategy.experimental_run_v2(train_step_fn, next(data_iter))\n            else:\n                strategy.experimental_run_v2(train_step_fn, next(data_iter))\n\n    @tf.function\n    def valid_step(data_iter):\n        def valid_step_fn(x, y):\n            probabilities = model(x, training=False)\n            loss = loss_fn(y, probabilities)\n            # update metrics\n            valid_accuracy.update_state(y, probabilities)\n            valid_loss.update_state(loss)\n        for _ in tf.range(valid_step_size):\n            if COLAB:\n                strategy.experimental_run_v2(valid_step_fn, next(data_iter))\n            else:\n                strategy.experimental_run_v2(valid_step_fn, next(data_iter))\n    \n    \n    # Model\n    model_path = models_path+f'model_{fold}.h5'\n    with strategy.scope():\n        model = model_fn((None, None, CHANNELS), N_CLASSES)\n        unfreeze_model(model) # unfreeze all layers except \"batch normalization\"\n        \n        optimizer = optimizers.Adam(learning_rate=lambda: lrfn(tf.cast(optimizer.iterations, tf.float32)))\n        loss_fn = losses.categorical_crossentropy\n\n        train_accuracy = metrics.CategoricalAccuracy()\n        valid_accuracy = metrics.CategoricalAccuracy()\n        train_loss = metrics.Sum()\n        valid_loss = metrics.Sum()\n    \n    \n    # Setup training loop\n    step = 0\n    epoch_steps = 0\n    patience_cnt = 0\n    best_val = 0\n    history = {'loss': [], 'val_loss': [], 'accuracy': [], 'val_accuracy': []}\n\n    ### Train model\n    for epoch in range(EPOCHS):\n        epoch_start_time = time.time()\n\n        # Run training step\n        train_step(train_data_iter)\n        epoch_steps += step_size\n        step += step_size\n            \n\n        # Validation run at the end of each epoch\n        if (step // step_size) > epoch:\n            # Validation run\n            valid_epoch_steps = 0\n            valid_step(valid_data_iter)\n            valid_epoch_steps += valid_step_size\n\n            # Compute metrics\n            history['accuracy'].append(train_accuracy.result().numpy())\n            history['loss'].append(train_loss.result().numpy() / (BATCH_SIZE * epoch_steps))\n            history['val_accuracy'].append(valid_accuracy.result().numpy())\n            history['val_loss'].append(valid_loss.result().numpy() / (BATCH_SIZE * valid_epoch_steps))\n\n            # Report metrics\n            epoch_time = time.time() - epoch_start_time\n            print(f'\\nEPOCH {epoch+1}/{EPOCHS}')\n            print(f'time: {epoch_time:0.1f}s',\n                  f\"loss: {history['loss'][-1]:0.4f}\",\n                  f\"accuracy: {history['accuracy'][-1]:0.4f}\",\n                  f\"val_loss: {history['val_loss'][-1]:0.4f}\",\n                  f\"val_accuracy: {history['val_accuracy'][-1]:0.4f}\",\n                  f'lr: {lrfn(tf.cast(optimizer.iterations, tf.int32).numpy()):0.4g}')\n\n            # Early stopping monitor\n            if history['val_accuracy'][-1] >= best_val:\n                best_val = history['val_accuracy'][-1]\n                model.save_weights(model_path)\n                print(f'Saved model weights at \"{model_path}\"')\n                patience_cnt = 1\n            else:\n                patience_cnt += 1\n            if patience_cnt > ES_PATIENCE:\n                print(f'Epoch {epoch:05d}: early stopping')\n                break\n\n                \n            # Set up next epoch\n            epoch = step // step_size\n            epoch_steps = 0\n            train_accuracy.reset_states()\n            train_loss.reset_states()\n            valid_accuracy.reset_states()\n            valid_loss.reset_states()\n    \n    \n    ### RESULTS\n    print(f\"#### FOLD {fold+1} OOF Accuracy = {np.max(history['val_accuracy']):.3f}\")\n    \n    history_list.append(history)\n    # Load best model weights\n    model.load_weights(model_path)\n\n    # OOF predictions\n    ds_valid = get_dataset(VALID_FILENAMES, ordered=True)\n    oof_labels.append([target.numpy() for img, target in iter(ds_valid.unbatch())])\n    x_oof = ds_valid.map(lambda image, target: image)\n    oof_pred.append(np.argmax(model.predict(x_oof), axis=-1))","metadata":{"id":"8HrFGFRNCaO5","outputId":"8b365c69-b71a-4745-c54f-65b1d917c756","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model loss graph","metadata":{"id":"NQyHtXnRCaO5"}},{"cell_type":"code","source":"for fold, history in enumerate(history_list):\n    print(f'\\nFOLD: {fold+1}')\n    plot_metrics(history)","metadata":{"id":"_7pUdch3CaO6","outputId":"78aef160-73cc-401b-9e9c-d75810159dd4","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model evaluation\n\nNow we can evaluate the performance of the model, first, we can evaluate the usual metrics like, `accuracy`, `precision`, `recall`, and `f1-score`, `scikit-learn` provides the perfect function for this `classification_report`.\n\nWe are evaluating the model on the `OOF` predictions, it stands for `Out Of Fold`, since we are training using `K-Fold` our model will see all the data, and the correct way to evaluate each fold is by looking at the predictions that are not from that fold.\n\n## OOF metrics","metadata":{"id":"ImheYrtKCaO7"}},{"cell_type":"markdown","source":"#### I am still having some problems to get the real model `OOF` scores while using `TPU Pods`, so the results here and the confusion matrix are just placeholders.","metadata":{"id":"I_QvO8rPCaO7"}},{"cell_type":"code","source":"y_true = np.concatenate(oof_labels)\ny_true = np.argmax(y_true, axis=-1)\ny_pred = np.concatenate(oof_pred)\n\nprint(classification_report(y_true, y_pred, target_names=CLASSES))","metadata":{"id":"q7TfhkICCaO7","outputId":"a37a40d3-206b-4309-d10a-fe7ae7c83c3a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Confusion matrix\n\nLet's also take a look at the confusion matrix, this will give us an idea about what classes the model is mixing or having a hard time.","metadata":{"id":"pIAJafCnCaO7"}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(20, 12))\ncfn_matrix = confusion_matrix(y_true, y_pred, labels=range(len(CLASSES)))\ncfn_matrix = (cfn_matrix.T / cfn_matrix.sum(axis=1)).T\ndf_cm = pd.DataFrame(cfn_matrix, index=CLASSES, columns=CLASSES)\nax = sns.heatmap(df_cm, cmap='Blues', annot=True, fmt='.2f', linewidths=.5).set_title('Train', fontsize=30)\nplt.show()","metadata":{"id":"dw2F8Wk2CaO7","outputId":"4d732291-93b8-4bbc-83e7-8b15d5517374","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize predictions\n\nFinally, it is a good practice to always inspect some of the model's prediction by looking at the data, this can give an idea if the model is getting some predictions wrong because the data is really hard, of if it is because the model is actually bad.\n\n\n### Class map\n```\n0: Cassava Bacterial Blight (CBB)\n1: Cassava Brown Streak Disease (CBSD)\n2: Cassava Green Mottle (CGM)\n3: Cassava Mosaic Disease (CMD)\n4: Healthy\n```\n\n\n## Train set","metadata":{"id":"AaZVX_ZhCaO7"}},{"cell_type":"code","source":"train_dataset = get_dataset(TRAINING_FILENAMES, ordered=True)\nx_samp, y_samp = dataset_to_numpy_util(train_dataset, 18)\ny_samp = np.argmax(y_samp, axis=-1)\n\nx_samp_1, y_samp_1 = x_samp[:9,:,:,:], y_samp[:9]\nsamp_preds_1 = model.predict(x_samp_1, batch_size=9)\ndisplay_9_images_with_predictions(x_samp_1, samp_preds_1, y_samp_1)\n\nx_samp_2, y_samp_2 = x_samp[9:,:,:,:], y_samp[9:]\nsamp_preds_2 = model.predict(x_samp_2, batch_size=9)\ndisplay_9_images_with_predictions(x_samp_2, samp_preds_2, y_samp_2)","metadata":{"id":"ExEWWtxyCaO8","outputId":"49032a35-80f7-45de-a68e-869fa30a653c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_data=True# You can immediately create a kaggle dataset from your models\nif COLAB and save_data:\n  \n    import json\n\n\n    \n    data = {\"title\": \"Cassava Leaf Disease\", \n        \"id\": \"aikhmelnytskyy/CassavaLeafDisease\", \n        \"licenses\": [\n                     {\n                         \"name\": \"CC0-1.0\"\n                      }\n                     ]}\n    \n\n    # for kaggle api Connection\n    \n    !kaggle datasets init -p /content/drive/MyDrive/Models/Cassava/\n    \n    with open(\"/content/drive/MyDrive/Models/Cassava/dataset-metadata.json\", \"w\", encoding=\"utf-8\") as file:\n        json.dump(data, file)\n    \n    #if new dataset\n    !kaggle datasets create -p /content/drive/MyDrive/Models/Cassava/\n    #If you’d like to upload a new version of an existing dataset\n    #!kaggle datasets version -p /content/drive/MyDrive/Models/Cassava/ -m \"Your message here\"","metadata":{"id":"h9CeTJAYKYQd","outputId":"81703127-3be3-4aa6-dced-23398280a3ee","trusted":true},"execution_count":null,"outputs":[]}]}