{"cells":[{"metadata":{},"cell_type":"markdown","source":"References:\nhttps://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\nhttps://www.kaggle.com/dimitreoliveira/cassava-leaf-disease-tpu-tensorflow-training/notebook#Training"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install --quiet efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math, os, re, warnings, random\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.utils import class_weight\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras import optimizers, applications, Sequential, losses, metrics\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\nimport efficientnet.tfkeras as efn\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = \"TPU\" #or \"GPU\"\n\n# train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\n# test  = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\nsub   = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\n\nFOLDS = 3\n# WHICH IMAGE SIZES TO LOAD EACH FOLD\n# CHOOSE 128, 192, 256, 384, 512, 768\nIMG_SIZES = [256, 384, 512]\n# INCLUDE OLD COMP DATA? YES=1 NO=0\n# INC2019 = [1,0,1,1,0,1,0,1]\n# INC2018 = [0,1,1,0,1,1,1,1]\nBATCH_SIZES = [64,64,64]   #[64,64,64]\nEPOCHS = [20, 20, 20]   #[10, 10, 10]\n# WHICH EFFICIENTNET B? TO USE\nEFF_NETS = [2, 2, 3]\n# WEIGHTS FOR FOLD MODELS WHEN PREDICTING TEST\nWGTS = [1/FOLDS]*FOLDS#[1/FOLDS,1/FOLDS,1/FOLDS]\n# TEST TIME AUGMENTATION STEPS\nTTA = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16 * REPLICAS\nLEARNING_RATE = 3e-5 * REPLICAS\n# EPOCHS = 30\nHEIGHT = 512\nWIDTH = 512\nCHANNELS = 3\nN_CLASSES = 5\nES_PATIENCE = 4\n#N_FOLDS = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    # Shear\n    if p_shear > .2:\n        if p_shear > .6:\n            image = transform_shear(image, HEIGHT, shear=20.)\n        else:\n            image = transform_shear(image, HEIGHT, shear=-20.)\n            \n    # Rotation\n    if p_rotation > .2:\n        if p_rotation > .6:\n            image = transform_rotation(image, HEIGHT, rotation=45.)\n        else:\n            image = transform_rotation(image, HEIGHT, rotation=-45.)\n            \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > .75:\n        image = tf.image.transpose(image)\n        \n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n        \n    # Pixel-level transforms\n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n        \n    # Crops\n    if p_crop > .7:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.6)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.8)\n    elif p_crop > .4:\n        crop_size = tf.random.uniform([], int(HEIGHT*.6), HEIGHT, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n            \n    image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Auxiliary Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Datasets utility functions\ndef decode_image(image_data):\n    \"\"\"\n        1. Decode a JPEG-encoded image to a uint8 tensor.\n        2. Cast tensor to float and normalizes (range between 0 and 1).\n        3. Resize and reshape images to the expected size.\n    \"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n                      \n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_tfrecord(example, labeled=True):\n    \"\"\"\n        1. Parse data based on the 'TFREC_FORMAT' map.\n        2. Decode image.\n        3. If 'labeled' returns (image, label) if not (image, name).\n    \"\"\"\n    if labeled:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'target': tf.io.FixedLenFeature([], tf.int64), \n        }\n    else:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'image_name': tf.io.FixedLenFeature([], tf.string), \n        }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    if labeled:\n        label_or_name = tf.cast(example['target'], tf.int32)\n    else:\n        label_or_name =  example['image_name']\n    return image, label_or_name\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    \"\"\"\n        Create a Tensorflow dataset from TFRecords.\n    \"\"\"\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda x: read_tfrecord(x, labeled=labeled), num_parallel_calls=AUTO)\n    return dataset\n\ndef get_dataset(FILENAMES, labeled=True, ordered=False, repeat=False,\n                augment=False, dim=256, batch_size=16):\n    \"\"\"\n        Return a Tensorflow dataset ready for training or inference.\n    \"\"\"\n    dataset = load_dataset(FILENAMES, labeled=labeled, ordered=ordered)\n    if augment:\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    if repeat:\n        dataset = dataset.repeat()\n    if not ordered:\n        dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualization utility functions\nnp.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', \n                  fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n\ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n    \n# Visualize model predictions\ndef dataset_to_numpy_util(dataset, N):\n    dataset = dataset.unbatch().batch(N)\n    for images, labels in dataset:\n        numpy_images = images.numpy()\n        numpy_labels = labels.numpy()\n        break;  \n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    label = np.argmax(label, axis=-1)\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(label, str(correct), ', shoud be ' if not correct else '',\n                                correct_label if not correct else ''), correct\n\ndef display_one_flower_eval(image, title, subplot, red=False):\n    plt.subplot(subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    plt.title(title, fontsize=14, color='red' if red else 'black')\n    return subplot+1\n\ndef display_9_images_with_predictions(images, predictions, labels):\n    subplot=331\n    plt.figure(figsize=(13,13))\n    for i, image in enumerate(images):\n        title, correct = title_from_label_and_target(predictions[i], labels[i])\n        subplot = display_one_flower_eval(image, title, subplot, not correct)\n        if i >= 8:\n            break;\n              \n    plt.tight_layout()\n    plt.subplots_adjust(wspace=0.1, hspace=0.1)\n    plt.show()\n\n\n# Model evaluation\ndef plot_metrics(history):\n    metric_list = [m for m in list(history.keys()) if m is not 'lr']\n    size = len(metric_list)//2\n    fig, axes = plt.subplots(size, 1, sharex='col', figsize=(20, size * 4))\n    if size > 1:\n        axes = axes.flatten()\n    else:\n        axes = [axes]\n    \n    for index in range(len(metric_list)//2):\n        metric_name = metric_list[index]\n        val_metric_name = metric_list[index+size]\n        axes[index].plot(history[metric_name], label='Train %s' % metric_name)\n        axes[index].plot(history[val_metric_name], label='Validation %s' % metric_name)\n        axes[index].legend(loc='best', fontsize=16)\n        axes[index].set_title(metric_name)\n        if 'loss' in metric_name:\n            axes[index].axvline(np.argmin(history[metric_name]), linestyle='dashed')\n            axes[index].axvline(np.argmin(history[val_metric_name]), linestyle='dashed', color='orange')\n        else:\n            axes[index].axvline(np.argmax(history[metric_name]), linestyle='dashed')\n            axes[index].axvline(np.argmax(history[val_metric_name]), linestyle='dashed', color='orange')\n\n    plt.xlabel('Epochs', fontsize=16)\n    sns.despine()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_dataset = get_dataset(TRAINING_FILENAMES, ordered=True)\n# train_iter = iter(train_dataset.unbatch().batch(2))\n\n# display_batch_of_images(next(train_iter))\n# display_batch_of_images(next(train_iter))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\nLearning rate schedule\n\nWe are going to use a cosine learning rate schedule with a warm-up phase, this may be a good idea since we are using a pre-trained model, the warm-up phase will be useful to avoid the pre-trained weights degradation resulting in catastrophic forgetting, during the schedule the learning rate will slowly decrease to very low values, this helps the model to land on more stable weights.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"LR_START = 1e-8\nLR_MIN = 1e-8\nLR_MAX = LEARNING_RATE\nLR_RAMPUP_EPOCHS = 3\nLR_SUSTAIN_EPOCHS = 0\nN_CYCLES = .5\nEPOCHS2 = 30\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        progress = (epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) / (EPOCHS2 - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)\n        lr = LR_MAX * (0.5 * (1.0 + tf.math.cos(math.pi * N_CYCLES * 2.0 * progress)))\n        if LR_MIN is not None:\n            lr = tf.math.maximum(LR_MIN, lr)\n            \n    return lr\n\nrng = [i for i in range(EPOCHS2)]\ny = [lrfn(x) for x in rng]\n\nsns.set(style='whitegrid')\nfig, ax = plt.subplots(figsize=(20, 6))\nplt.plot(rng, y)\n\n# print(f'{EPOCHS2} total epochs and {NUM_TRAINING_IMAGES//BATCH_SIZE} steps per epoch')\nprint(f'Learning rate schedule: {y[0]:.3g} to {max(y):.3g} to {y[-1]:.3g}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]\n\ndef build_model(dim=128, ef=0):\n    input_image = tf.keras.layers.Input(shape=(dim,dim,3))\n    #base = EFNS[ef](input_shape=(dim,dim,3),weights='imagenet',include_top=False)\n    \n    base_model = EFNS[ef](input_tensor=input_image, \n                                    include_top=False, \n                                    weights='noisy-student', \n                                    pooling='avg')\n    \n    model = Sequential([\n                base_model,\n                L.Dropout(.25),\n                L.Dense(N_CLASSES, activation='softmax', name='output')\n            ])\n    \n    optimizer = optimizers.Adam(lr=LEARNING_RATE)\n    model.compile(optimizer=optimizer, \n                  loss=losses.SparseCategoricalCrossentropy(), \n                  metrics=['sparse_categorical_accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r'-([0-9]*)\\.').search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\ndatabase_base_path = '/kaggle/input/cassava-leaf-disease-classification/'\ntrain = pd.read_csv(f'{database_base_path}train.csv')\nprint(f'Train samples: {len(train)}')\n\n# GCS_PATH = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification') # Original dataset\n# GCS_PATH = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-tfrecords-{HEIGHT}x{WIDTH}') # Only resized\nGCS_PATH = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-tfrecords-{HEIGHT}x{WIDTH}') # Center croped and resized\n\n# TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train_tfrecords/*.tfrec') # Original TFRecords\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/*.tfrec')\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\n\nprint(f'GCS: train images: {NUM_TRAINING_IMAGES}')\ndisplay(train.head())\n\nCLASSES = ['Cassava Bacterial Blight', \n           'Cassava Brown Streak Disease', \n           'Cassava Green Mottle', \n           'Cassava Mosaic Disease', \n           'Healthy']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = [None]*FOLDS#; GCS_PATH2 = [None]*FOLDS\nfor i,k in enumerate(IMG_SIZES):\n    GCS_PATH[i] = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-tfrecords-{k}x{k}')\n    #GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))\nfiles_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/*.tfrec')))\n# files_test  = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"database_base_path = '/kaggle/input/cassava-leaf-disease-classification/'\nsubmission = pd.read_csv(f'{database_base_path}sample_submission.csv')\ndisplay(submission.head())\n\nTEST_FILENAMES = tf.io.gfile.glob(f'{database_base_path}test_tfrecords/ld_test*.tfrec')\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nprint(f'GCS: test: {NUM_TEST_IMAGES}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr_callback(batch_size=8):\n    lr_start   = 0.000005\n    lr_max     = 0.000020 * REPLICAS * batch_size/16\n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n    return lr_callback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = KFold(n_splits=FOLDS,shuffle=True,random_state=42)\noof_pred = []; oof_tar = []; oof_val = []; oof_names = []\nfiles_test = np.sort(np.array(tf.io.gfile.glob(f'{database_base_path}test_tfrecords/ld_test*.tfrec')))\npreds = np.zeros((count_data_items(files_test),1))\noof_pred = []; oof_labels = []; history_list = []\n\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(15))):\n    \n    model_path = f'last_epoch_model_{fold}.h5'\n    # DISPLAY FOLD INFO\n    if DEVICE=='TPU':\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print('#'*25); print('#### FOLD',fold+1)\n    print('#### Image Size %i with EfficientNet B%i and batch_size %i'%\n          (IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/Id_train%.2i*.tfrec'%x for x in idxT])\n#     if INC2019[fold]:\n#         files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])\n#         print('#### Using 2019 external data')\n#     if INC2018[fold]:\n#         files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])\n#         print('#### Using 2018+2017 external data')\n    np.random.shuffle(files_train); print('#'*25)\n    files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/Id_train%.2i*.tfrec'%x for x in idxV])\n    #files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))\n\n    files_test = np.sort(np.array(tf.io.gfile.glob(f'{database_base_path}test_tfrecords/Id_test*.tfrec')))\n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():\n        model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])\n        \n    # SAVE BEST MODEL EACH FOLD\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_loss', verbose=1, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n   \n    ##TRAIN\n#     history = model.fit(\n#         get_dataset(files_train, augment=True, ordered=True, repeat=True,\n#                 dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]), \n#         epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])], \n#         #steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,\n#         steps_per_epoch = 2,\n#         validation_data=get_dataset(files_valid,augment=False,ordered=False,\n#                 repeat=False,dim=IMG_SIZES[fold]), #class_weight = {0:1,1:2},\n#         verbose=1\n#     )\n\n    ct_train = count_data_items(files_train)\n    es = EarlyStopping(monitor='val_sparse_categorical_accuracy', mode='max', \n                       patience=ES_PATIENCE, restore_best_weights=True, verbose=1)\n    BATCH_SIZE = BATCH_SIZES[fold]\n    history = model.fit(x=get_dataset(files_train, augment=True, ordered=True, repeat=True,\n                                     dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]), \n                        validation_data=get_dataset(files_valid, labeled=True, ordered=True,\n                                                    repeat=False, dim=IMG_SIZES[fold],\n                                                    augment=False), \n                        steps_per_epoch=(ct_train // BATCH_SIZE), \n                        callbacks=[sv, es, LearningRateScheduler(lrfn, verbose=0)], \n                        epochs=EPOCHS[fold],  \n                        verbose=1).history\n    \n    ## Save last model weights\n    model.save_weights(model_path)\n\n    \n    ## Validation\n    # OOF predictions\n    history_list.append(history)\n    ds_valid = get_dataset(files_valid, labeled=True, ordered=True,\n                                                    repeat=False, dim=IMG_SIZES[fold],\n                                                    augment=False)\n    oof_labels.append([target.numpy() for img, target in iter(ds_valid.unbatch())])\n    x_oof = ds_valid.map(lambda image, image_name: image)\n    oof_pred.append(np.argmax(model.predict(x_oof), axis=-1))\n    \n    ## RESULTS\n    print(f\"#### FOLD {fold+1} OOF Accuracy = {np.max(history['val_sparse_categorical_accuracy']):.3f}\")\n    #break\n    \n    ## Testing\n    \n#     print('Loading model...')\n#     #model.load_weights('fold-%i.h5'%fold)\n    \n#     model.load_weights('../input/melanoma-weights/fold-%i.h5'%fold)\n    \n#     # PREDICT OOF USING TTA\n#     print('Predicting OOF with TTA...')\n#     ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,\n#             repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n#     ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS\n#     pred = model.predict(ds_valid,steps=STEPS,verbose=1)[:TTA*ct_valid,] \n#     oof_pred.append( np.mean(pred.reshape((ct_valid,TTA),order='F'),axis=1) )                 \n#     #oof_pred.append(model.predict(get_dataset(files_valid,dim=IMG_SIZES[fold]),verbose=1))\n    \n#     # GET OOF TARGETS AND NAMES\n#     ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n#             labeled=True, return_image_names=True)\n#     oof_tar.append( np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) )\n#     ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n#                 labeled=False, return_image_names=True)\n#     oof_names.append( np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(ds.unbatch())]))\n    \n#     # PREDICT TEST USING TTA\n#     print('Predicting Test with TTA...')\n#     ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,\n#             repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n#     ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS\n#     pred = model.predict(ds_test,steps=STEPS,verbose=1)[:TTA*ct_test,] \n#     preds[:,0] += np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1) * WGTS[fold]\n    \n#     # REPORT RESULTS\n#     auc = roc_auc_score(oof_tar[-1],oof_pred[-1])\n#     #oof_val.append(np.max( history.history['val_auc'] ))\n    \n    \n#     #print('#### FOLD %i OOF AUC with TTA = %.3f, without TTA = %.3f'%(fold+1,auc,oof_val[-1]))\n    \n#     print('#### FOLD %i OOF AUC with TTA = %.3f, without TTA = --'%(fold+1,auc))\n#     print()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nfor fold, history in enumerate(history_list):\n    print(f'\\nFOLD: {fold+1}')\n    plot_metrics(history)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Model evaluation\n\nNow we can evaluate the performance of the model, first, we can evaluate the usual metrics like, accuracy, precision, recall, and f1-score, scikit-learn provides the perfect function for this classification_report.\n\nWe are evaluating the model on the OOF predictions, it stands for Out Of Fold, since we are training using K-Fold our model will see all the data, and the correct way to evaluate each fold is by looking at the predictions that are not from that fold.\nOOF metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_true = np.concatenate(oof_labels)\ny_preds = np.concatenate(oof_pred)\n\nprint(classification_report(y_true, y_preds, target_names=CLASSES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\nConfusion matrix\n\nLet's also take a look at the confusion matrix, this will give us an idea about what classes the model is mixing or having a hard time.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nfig, ax = plt.subplots(1, 1, figsize=(20, 12))\ntrain_cfn_matrix = confusion_matrix(y_true, y_preds, labels=range(len(CLASSES)))\ntrain_cfn_matrix = (train_cfn_matrix.T / train_cfn_matrix.sum(axis=1)).T\ntrain_df_cm = pd.DataFrame(train_cfn_matrix, index=CLASSES, columns=CLASSES)\nax = sns.heatmap(train_df_cm, cmap='Blues', annot=True, fmt='.2f', linewidths=.5).set_title('Train', fontsize=30)\nplt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Visualize predictions\n\nFinally, it is a good practice to always inspect some of the model's prediction by looking at the data, this can give an idea if the model is getting some predictions wrong because the data is really hard, of if it is because the model is actually bad.\nClass map\n\n0: Cassava Bacterial Blight (CBB)\n1: Cassava Brown Streak Disease (CBSD)\n2: Cassava Green Mottle (CGM)\n3: Cassava Mosaic Disease (CMD)\n4: Healthy"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_dataset(TRAINING_FILENAMES, ordered=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_samp, y_samp = dataset_to_numpy_util(train_dataset, 25)\n\nx_samp_1, y_samp_1 = x_samp[:9,:,:,:], y_samp[:9]\nsamp_preds_1 = model.predict(x_samp_1, batch_size=9)\ndisplay_9_images_with_predictions(x_samp_1, samp_preds_1, y_samp_1)\n\nx_samp_2, y_samp_2 = x_samp[9:,:,:,:], y_samp[9:]\nsamp_preds_2 = model.predict(x_samp_2, batch_size=9)\ndisplay_9_images_with_predictions(x_samp_2, samp_preds_2, y_samp_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}