{"cells":[{"metadata":{},"cell_type":"markdown","source":"My main goal of this competition is:\n* Integrate a custom TensorFlow training loop where image is augmented each epoch\n* Create a working end-to-end pipeline for preprocessing and infering the images on TPUs\n* I'll be updating this workbook regularly.\n"},{"metadata":{},"cell_type":"markdown","source":"What still needs to be implemented:\n* Make sure that dataset is reset and is applying the map functin on each step, as tensorflow does not do it automatically."},{"metadata":{"trusted":true},"cell_type":"code","source":"#install efficientnet keras model\n!pip install --quiet efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip uninstall --quiet albumentations -y\n#!pip uninstall --quiet tensorflow -y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --quiet albumentations==0.5.1\n#!pip install --quiet tensorflow==2.3.0\n#!pip install --quiet cloud-tpu-client\n\n#import tensorflow as tf\n#from cloud_tpu_client import Client\n#print(tf.__version__)\n\n#Client().configure_tpu_version(tf.__version__, restart_type='ifNeeded')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#import used libraries\nimport math, os, re, warnings, random, time\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.utils import class_weight\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras import optimizers, applications, Sequential, losses, metrics\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\nimport efficientnet.tfkeras as efn\nfrom matplotlib.pyplot import imread\nimport keras\nimport albumentations\nimport functools\nfrom multiprocessing.dummy import Pool\n\n#make sure everything is seeded so the models are reproduceable\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ndef_strat = tf.distribute.get_strategy()\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#TPUs use less RAM so the BATCH SIZE can be increased\nif tpu:\n    BATCH_SIZE = 8 * REPLICAS\nelse:\n    BATCH_SIZE = 4 * REPLICAS\n#Maximum Learning Rate\nLEARNING_RATE = 1e-5 * REPLICAS\n#Maximum number of Epochs in Training\nEPOCHS = 18\n#The following 4 variables determine the size of the tensors definining the images for training\nHEIGHT = 512\nWIDTH = 512\nIMAGE_SIZE = [HEIGHT,WIDTH]\nCHANNELS = 3\n#The number of classes we are trying to classify\nN_CLASSES = 5\n#The number of epochs we wait before stopping the training run after not improving the model\nES_PATIENCE = 4\n#Number of KFolds we are doing. e.g. How many models we are training\nN_FOLDS = 3\n#Number of test time augmentation, used for the test set\nN_TTA = 5\n\nONE_HOT = True\n\nT1 = 0.2\nT2 = 1.0\nLABEL_SMOOTH = 0.15","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Path to the datasets\ndatabase_base_path = '/kaggle/input/cassava-leaf-disease-classification/'\n\n#Read the CSV with the image names and labels associated with them for training\ntrain = pd.read_csv(database_base_path + 'train.csv')\nprint('Train samples: %d' % len(train))\n\n\nGCS_PATH = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification')\ndisplay(train.head())\n\nTRAIN_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train_tfrecords/ld_train*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test_tfrecords/ld_test*.tfrec')\nTRAIN_IMAGES = os.listdir(database_base_path+'/train_images/')\nTEST_IMAGES = os.listdir(database_base_path+'/test_images/')\n\n#Names of our 5 classes for classification\nCLASSES = ['Cassava Bacterial Blight', \n           'Cassava Brown Streak Disease', \n           'Cassava Green Mottle', \n           'Cassava Mosaic Disease', \n           'Healthy']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms = albumentations.Compose([\n    albumentations.Flip(always_apply=False,p=0.9),\n    albumentations.RandomResizedCrop(height=HEIGHT,width=WIDTH,always_apply = True),\n    albumentations.Blur(blur_limit=7,p=0.3),\n    albumentations.ColorJitter(brightness=0.1, contrast=0.15, saturation=0.15, hue=0.0, always_apply=False, p=0.5),\n    #albumentations.RandomBrightnessContrast(brightness_limit=0.15,contrast_limit=0.2,brightness_by_max=True,always_apply=False,p=0.3),\n    albumentations.Cutout(num_holes=4,max_h_size=int(HEIGHT*0.1),max_w_size=int(WIDTH*0.1),fill_value=0,always_apply=False,p=0.2,),\n    albumentations.Cutout(num_holes=4,max_h_size=int(HEIGHT*0.1),max_w_size=int(WIDTH*0.1),fill_value=0,always_apply=False,p=0.2,)])\ndef aug_fn(image):\n    aug_data = transforms(image=image)\n    aug_img = aug_data['image']\n    aug_img = tf.cast(aug_img,tf.float32)\n    return aug_img\ndef aug_fn_label(image,label):\n    aug_data = transforms(image=image)\n    aug_img = aug_data['image']\n    aug_img = tf.cast(aug_img,tf.float32)\n    return aug_img,label\ndef process_data(image, label):\n    aug_img = tf.numpy_function(func=aug_fn, inp=[image], Tout=tf.float32)\n    return aug_img, label\ndef set_shapes(img, label):\n    img.set_shape((HEIGHT,WIDTH,CHANNELS))\n    label.set_shape([])\n    return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_cutout = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_randcrop = tf.random.uniform([],0,1.0,dtype = tf.float32)\n    \n    if p_randcrop > .4:\n        image = tf.image.random_crop(image, size= [HEIGHT,WIDTH,CHANNELS])\n        image = tf.reshape(image,shape = (HEIGHT,WIDTH,CHANNELS))\n    else:\n        image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n        image = tf.reshape(image,shape = (HEIGHT,WIDTH,CHANNELS))\n    \n    # Shear\n    if p_shear > .2:\n        if p_shear > .6:\n            image = transform_shear(image, HEIGHT, shear=20.)\n        else:\n            image = transform_shear(image, HEIGHT, shear=-20.)\n            \n    # Rotation\n    if p_rotation > .2:\n        if p_rotation > .6:\n            image = transform_rotation(image, HEIGHT, rotation=45.)\n        else:\n            image = transform_rotation(image, HEIGHT, rotation=-45.)\n            \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > .75:\n        image = tf.image.transpose(image)\n        \n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n        \n    # Pixel-level transforms\n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n        \n    # Crops\n    if p_crop > .6:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.5)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.6)\n        elif p_crop > .7:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.8)\n    elif p_crop > .3:\n        crop_size = tf.random.uniform([], int(HEIGHT*.6), HEIGHT, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n            \n    image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n\n    if p_cutout > .5:\n        image = data_augment_cutout(image)\n    image = tf.reshape(image,shape = (HEIGHT,WIDTH,CHANNELS))\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\n# CutOut\ndef data_augment_cutout(image, min_mask_size=(int(HEIGHT * .1), int(HEIGHT * .1)), \n                        max_mask_size=(int(HEIGHT * .125), int(HEIGHT * .125))):\n    p_cutout = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    if p_cutout > .85: # 10~15 cut outs\n        n_cutout = tf.random.uniform([], 10, 15, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    elif p_cutout > .6: # 5~10 cut outs\n        n_cutout = tf.random.uniform([], 5, 10, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    elif p_cutout > .25: # 2~5 cut outs\n        n_cutout = tf.random.uniform([], 2, 5, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    else: # 1 cut out\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=1)\n\n    return image\n\ndef random_cutout(image, height, width, channels=3, min_mask_size=(10, 10), max_mask_size=(80, 80), k=1):\n    assert height > min_mask_size[0]\n    assert width > min_mask_size[1]\n    assert height > max_mask_size[0]\n    assert width > max_mask_size[1]\n\n    for i in range(k):\n        mask_height = tf.random.uniform(shape=[], minval=min_mask_size[0], maxval=max_mask_size[0], dtype=tf.int32)\n        mask_width = tf.random.uniform(shape=[], minval=min_mask_size[1], maxval=max_mask_size[1], dtype=tf.int32)\n\n        pad_h = height - mask_height\n        pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)\n        pad_bottom = pad_h - pad_top\n\n        pad_w = width - mask_width\n        pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)\n        pad_right = pad_w - pad_left\n\n        cutout_area = tf.zeros(shape=[mask_height, mask_width, channels], dtype=tf.uint8)\n\n        cutout_mask = tf.pad([cutout_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)\n        cutout_mask = tf.squeeze(cutout_mask, axis=0)\n        image = tf.multiply(tf.cast(image, tf.float32), tf.cast(cutout_mask, tf.float32))\n\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nplt.figure(figsize=(10, 10))\nfor i in range(16):\n    ax = plt.subplot(4, 4, i + 1)\n    image = imread(database_base_path+'train_images/'+train['image_id'].iloc[i])\n    aug_image,a = data_augment(image,label = 0)\n    plt.imshow(tf.cast(aug_image,dtype = tf.int32))\n    label = train['label'].iloc[i]\n    plt.title(label)\n    plt.axis(\"off\")\nprint(time.time()-start_time)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from functools import partial\n\ndef decode_image(image_data):\n    \"\"\"\n        1. Decode a JPEG-encoded image to a uint8 tensor.\n        2. Cast tensor to float and normalizes (range between 0 and 1).\n        3. Resize and reshape images to the expected size.\n    \"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32)/255\n\n    return image\n\ndef correct_shapes(image,label):\n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image,label\n\ndef random_crop(image,label):\n    image = tf.image.random_crop(image, size= [HEIGHT,WIDTH,CHANNELS])\n    return image,label\n\ndef process_path(file_path):\n    name = get_name(file_path)\n    img = tf.io.read_file(file_path)\n    img = decode_image(img)\n    return img, name\n\ndef get_name(file_path):\n    parts = tf.strings.split(file_path, os.path.sep)\n    name = parts[-1]\n    return name\n\ndef read_tfrecord(example, labeled=True):\n    \"\"\"\n        1. Parse data based on the 'TFREC_FORMAT' map.\n        2. Decode image.\n        3. If 'labeled' returns (image, label) if not (image, name).\n    \"\"\"\n    if labeled:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'target': tf.io.FixedLenFeature([], tf.int64), \n        }\n    else:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'image_name': tf.io.FixedLenFeature([], tf.string), \n        }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    if labeled:\n        label_or_name = tf.cast(example['target'], tf.int32)\n    else:\n        label_or_name =  example['image_name']\n    return image, label_or_name\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    \"\"\"\n        Create a Tensorflow dataset from TFRecords.\n    \"\"\"\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda x: read_tfrecord(x, labeled=labeled), num_parallel_calls=AUTO)\n    return dataset\n\ndef one_hot_fn(x,y):\n    y = tf.one_hot(y,N_CLASSES)\n    tf.ensure_shape(y,[N_CLASSES])\n    return x,y\n\n\ndef get_training_dataset(dataset, do_aug = True,repeat = False,is_one_hot = ONE_HOT): # trainingfiles changed to dataset\n    #dataset = load_dataset(training_files, labeled=True)\n    if do_aug:\n        dataset = dataset.map(data_augment)\n    if is_one_hot:\n        dataset = dataset.map(one_hot_fn)\n    dataset = dataset.shuffle(2048,reshuffle_each_iteration=True)\n    if repeat: dataset = dataset.repeat()\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(validation_files , ordered=False,is_crop = False,is_one_hot = ONE_HOT):\n    dataset = load_dataset(validation_files, labeled=True, ordered=ordered) \n    if is_crop:\n        dataset = dataset.map(random_crop)\n    else:\n        dataset = dataset.map(correct_shapes)\n    if is_one_hot:\n        dataset = dataset.map(one_hot_fn)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n\ndef get_test_dataset(files_path, shuffled=False, tta=True, extension='jpg'):\n    dataset = tf.data.Dataset.list_files(f'{files_path}*{extension}', shuffle=shuffled)\n    dataset = dataset.map(process_path, num_parallel_calls=AUTO)\n    if tta:\n        dataset = dataset.map(process_data, num_parallel_calls=AUTO)\n        dataset = dataset.map(set_shapes, num_parallel_calls=AUTO)\n    dataset = dataset.batch(1)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LR_START = 5e-6\nLR_MIN = 1e-5\nLR_MAX = LEARNING_RATE\nLR_RAMPUP_EPOCHS = 3\nLR_SUSTAIN_EPOCHS = 0\nN_CYCLES = .5\n\ndef lrfn(epoch,change_on_step = False,step = 0,max_steps = 10):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        progress = (epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) / (EPOCHS - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)\n        lr = LR_MAX * (0.5 * (1.0 + tf.math.cos(math.pi * N_CYCLES * 2.0 * progress)))\n        if LR_MIN is not None:\n            lr = tf.math.maximum(LR_MIN, lr)\n    if change_on_step:\n        next_lr = lrfn(epoch+1)\n        return lr+step/max_steps*(next_lr-lr) \n    return lr\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\n\nsns.set(style='whitegrid')\nfig, ax = plt.subplots(figsize=(20, 6))\nplt.plot(rng, y)\n\nprint(f'{EPOCHS} total epochs and {len(train)//BATCH_SIZE} steps per epoch')\nprint(f'Learning rate schedule: {y[0]:.3g} to {max(y):.3g} to {y[-1]:.3g}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def for_loop(num_iters, body, initial_args):\n    \"\"\"Runs a simple for-loop with given body and initial_args.\n    Args:\n    num_iters: Maximum number of iterations.\n    body: Body of the for-loop.\n    initial_args: Args to the body for the first iteration.\n    Returns:\n    Output of the final iteration.\n    \"\"\"\n    for i in range(num_iters):\n        if i == 0:\n            outputs = body(*initial_args)\n        else:\n            outputs = body(*outputs)\n    return outputs\ndef log_t(u, t):\n    \"\"\"Compute log_t for `u`.\"\"\"\n\n    def _internal_log_t(u, t):\n        return (u**(1.0 - t) - 1.0) / (1.0 - t)\n\n    return tf.cond(\n      tf.equal(t, 1.0), lambda: tf.math.log(u),\n      functools.partial(_internal_log_t, u, t))\ndef exp_t(u, t):\n    \"\"\"Compute exp_t for `u`.\"\"\"\n\n    def _internal_exp_t(u, t):\n        return tf.nn.relu(1.0 + (1.0 - t) * u)**(1.0 / (1.0 - t))\n\n    return tf.cond(\n      tf.equal(t, 1.0), lambda: tf.math.exp(u),\n      functools.partial(_internal_exp_t, u, t))\n\ndef compute_normalization_fixed_point(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example (t > 1.0).\n    Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    t: Temperature 2 (> 1.0 for tail heaviness).\n    num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n\n    mu = tf.math.reduce_max(activations, -1, keepdims=True)\n    normalized_activations_step_0 = activations - mu\n    shape_normalized_activations = tf.shape(normalized_activations_step_0)\n\n    def iter_body(i, normalized_activations):\n        logt_partition = tf.math.reduce_sum(\n            exp_t(normalized_activations, t), -1, keepdims=True)\n        normalized_activations_t = tf.reshape(\n            normalized_activations_step_0 * tf.math.pow(logt_partition, 1.0 - t),\n            shape_normalized_activations)\n        return [i + 1, normalized_activations_t]\n\n    _, normalized_activations_t = for_loop(num_iters, iter_body,\n                                         [0, normalized_activations_step_0])\n    logt_partition = tf.math.reduce_sum(\n      exp_t(normalized_activations_t, t), -1, keepdims=True)\n    return -log_t(1.0 / logt_partition, t) + mu\n\ndef compute_normalization_binary_search(activations, t, num_iters=10):\n    \"\"\"Returns the normalization value for each example (t < 1.0).\n    Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    t: Temperature 2 (< 1.0 for finite support).\n    num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    mu = tf.math.reduce_max(activations, -1, keepdims=True)\n    normalized_activations = activations - mu\n    shape_activations = tf.shape(activations)\n    effective_dim = tf.cast(\n      tf.math.reduce_sum(\n          tf.cast(\n              tf.greater(normalized_activations, -1.0 / (1.0 - t)), tf.int32),\n          -1,\n          keepdims=True), tf.float32)\n    shape_partition = tf.concat([shape_activations[:-1], [1]], 0)\n    lower = tf.zeros(shape_partition)\n    upper = -log_t(1.0 / effective_dim, t) * tf.ones(shape_partition)\n\n    def iter_body(i, lower, upper):\n        logt_partition = (upper + lower)/2.0\n        sum_probs = tf.math.reduce_sum(exp_t(\n            normalized_activations - logt_partition, t), -1, keepdims=True)\n        update = tf.cast(tf.less(sum_probs, 1.0), tf.float32)\n        lower = tf.reshape(lower * update + (1.0 - update) * logt_partition,\n                           shape_partition)\n        upper = tf.reshape(upper * (1.0 - update) + update * logt_partition,\n                           shape_partition)\n        return [i + 1, lower, upper]\n\n    _, lower, upper = for_loop(num_iters, iter_body, [0, lower, upper])\n    logt_partition = (upper + lower)/2.0\n    return logt_partition + mu\n\ndef compute_normalization(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example.\n    Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    t: Temperature 2 (< 1.0 for finite support, > 1.0 for tail heaviness).\n    num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    return tf.cond(\n      tf.less(t, 1.0),\n      functools.partial(compute_normalization_binary_search, activations, t,\n                        num_iters),\n      functools.partial(compute_normalization_fixed_point, activations, t,\n                        num_iters))\n\ndef _internal_bi_tempered_logistic_loss(activations, labels, t1, t2):\n    \"\"\"Computes the Bi-Tempered logistic loss.\n    Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    labels: batch_size\n    t1: Temperature 1 (< 1.0 for boundedness).\n    t2: Temperature 2 (> 1.0 for tail heaviness).\n    Returns:\n    A loss tensor for robust loss.\n    \"\"\"\n    if t2 == 1.0:\n        normalization_constants = tf.math.log(\n            tf.math.reduce_sum(tf.exp(activations), -1, keepdims=True))\n        if t1 == 1.0:\n            return normalization_constants + tf.math.reduce_sum(\n                tf.multiply(labels, tf.log(labels + 1e-10) - activations), -1)\n        else:\n            shifted_activations = tf.math.exp(activations - normalization_constants)\n            one_minus_t1 = (1.0 - t1)\n            one_minus_t2 = 1.0\n    else:\n        one_minus_t1 = (1.0 - t1)\n        one_minus_t2 = (1.0 - t2)\n        normalization_constants = compute_normalization(\n        activations, t2, num_iters=5)\n        shifted_activations = tf.nn.relu(1.0 + one_minus_t2 *\n                             (activations - normalization_constants))\n\n    if t1 == 1.0:\n        return tf.math.reduce_sum(\n            tf.math.multiply(\n            tf.math.log(labels + 1e-10) -\n            tf.math.log(tf.math.pow(shifted_activations, 1.0 / one_minus_t2)), labels),\n            -1)\n    else:\n        beta = 1.0 + one_minus_t1\n        logt_probs = (tf.math.pow(shifted_activations, one_minus_t1 / one_minus_t2) -\n          1.0) / one_minus_t1\n        return tf.math.reduce_sum(\n            tf.math.multiply(log_t(labels, t1) - logt_probs, labels) - 1.0 / beta *\n            (tf.math.pow(labels, beta) -\n            tf.math.pow(shifted_activations, beta / one_minus_t2)), -1)\n    \ndef tempered_sigmoid(activations, t, num_iters=5):\n    \"\"\"Tempered sigmoid function.\n    Args:\n    activations: Activations for the positive class for binary classification.\n    t: Temperature tensor > 0.0.\n    num_iters: Number of iterations to run the method.\n    Returns:\n    A probabilities tensor.\n    \"\"\"\n    t = tf.convert_to_tensor(t)\n    input_shape = tf.shape(activations)\n    activations_2d = tf.reshape(activations, [-1, 1])\n    internal_activations = tf.concat(\n      [tf.zeros_like(activations_2d), activations_2d], 1)\n    normalization_constants = tf.cond(\n      # pylint: disable=g-long-lambda\n      tf.equal(t, 1.0),\n      lambda: tf.math.log(\n          tf.math.reduce_sum(tf.exp(internal_activations), -1, keepdims=True)),\n      functools.partial(compute_normalization, internal_activations, t,\n                        num_iters))\n    internal_probabilities = exp_t(internal_activations - normalization_constants,\n                                 t)\n    one_class_probabilities = tf.split(internal_probabilities, 2, axis=1)[1]\n    return tf.reshape(one_class_probabilities, input_shape)\n\ndef tempered_softmax(activations, t, num_iters=5):\n    \"\"\"Tempered softmax function.\n    Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    t: Temperature tensor > 0.0.\n    num_iters: Number of iterations to run the method.\n    Returns:\n    A probabilities tensor.\n    \"\"\"\n    t = tf.convert_to_tensor(t)\n    normalization_constants = tf.cond(\n      tf.equal(t, 1.0),\n      lambda: tf.math.log(tf.math.reduce_sum(tf.exp(activations), -1, keepdims=True)),\n      functools.partial(compute_normalization, activations, t, num_iters))\n    return exp_t(activations - normalization_constants, t)\n\ndef bi_tempered_logistic_loss(activations,\n                              labels,\n                              t1=T1,\n                              t2=T2,\n                              label_smoothing=LABEL_SMOOTH,\n                              num_iters=5):\n    #\"\"\"Bi-Tempered Logistic Loss with custom gradient.\n    #Args:\n    #  activations: A multi-dimensional tensor with last dimension `num_classes`.\n    #  labels: A tensor with shape and dtype as activations.\n    #  t1: Temperature 1 (< 1.0 for boundedness).\n    #  t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n    #  label_smoothing: Label smoothing parameter between [0, 1).\n    #  num_iters: Number of iterations to run the method.\n    #Returns:\n    #  A loss tensor.\n    #\"\"\"\n    with tf.name_scope('bitempered_logistic'):\n        t1 = tf.convert_to_tensor(t1)\n        t2 = tf.convert_to_tensor(t2)\n        if label_smoothing > 0.0:\n            num_classes = tf.cast(tf.shape(labels)[-1], tf.float32)\n            labels = (\n              1 - num_classes /\n              (num_classes - 1) * label_smoothing) * labels + label_smoothing / (\n                  num_classes - 1)\n\n        @tf.custom_gradient\n        def _custom_gradient_bi_tempered_logistic_loss(activations):\n        #\"\"\"Bi-Tempered Logistic Loss with custom gradient.\n        #Args:\n        #activations: A multi-dimensional tensor with last dim `num_classes`.\n        #Returns:\n        #A loss tensor, grad.\n        #\"\"\"\n            with tf.name_scope('gradient_bitempered_logistic'):\n                probabilities = tempered_softmax(activations, t2, num_iters)\n                loss_values = tf.multiply(\n                labels,\n                log_t(labels + 1e-10, t1) - log_t(probabilities, t1)) - 1.0 / (2.0 - t1) * (tf.pow(labels, 2.0 - t1) - tf.pow(probabilities, 2.0 - t1))\n\n                def grad(d_loss):\n                    #\"\"\"Explicit gradient calculation.\n                    #Args:\n                    #d_loss: Infinitesimal change in the loss value.\n                    #Returns:\n                    #Loss gradient.\n                    #\"\"\"\n                    delta_probs = probabilities - labels\n                    forget_factor = tf.math.pow(probabilities, t2 - t1)\n                    delta_probs_times_forget_factor = tf.math.multiply(delta_probs,\n                                                                forget_factor)\n                    delta_forget_sum = tf.math.reduce_sum(\n                      delta_probs_times_forget_factor, -1, keepdims=True)\n                    escorts = tf.math.pow(probabilities, t2)\n                    escorts = escorts / tf.math.reduce_sum(escorts, -1, keepdims=True)\n                    derivative = delta_probs_times_forget_factor - tf.multiply(\n                      escorts, delta_forget_sum)\n                    return tf.multiply(d_loss, derivative)\n\n            return loss_values, grad\n\n    loss_values = tf.cond(tf.logical_and(tf.equal(t1, 1.0), tf.equal(t2, 1.0)),\n                          functools.partial(\n                              tf.nn.softmax_cross_entropy_with_logits,\n                              labels=labels,\n                              logits=activations),\n                          functools.partial(\n                              _custom_gradient_bi_tempered_logistic_loss,\n                              activations))\n    reduce_sum_last = lambda x: tf.math.reduce_sum(x, -1)\n    loss_values = tf.cond(tf.logical_and(tf.equal(t1, 1.0), tf.equal(t2, 1.0)),\n                          functools.partial(tf.identity, loss_values),\n                          functools.partial(reduce_sum_last, loss_values))\n    return loss_values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define the model\ndef model_fn(input_shape, N_CLASSES):\n    with strategy.scope():\n        input_image = L.Input(shape=input_shape, name='input_image')\n        base_model = efn.EfficientNetB3(input_tensor=input_image, \n                                        include_top=False, \n                                        weights='imagenet', \n                                        pooling='avg')\n\n        model = tf.keras.Sequential([\n            base_model,\n            L.Dropout(.25),\n            L.Dense(N_CLASSES, activation='softmax', name='output')\n        ])\n\n        optimizer = optimizers.Adam(LR_START)\n        lrschedule = LearningRateScheduler(lrfn, verbose=1) \n        model.compile(optimizer=optimizer, \n                      loss=keras.losses.CategoricalCrossentropy(label_smoothing= LABEL_SMOOTH), \n                      metrics=['categorical_accuracy'])\n\n        return model\nmodel = model_fn(input_shape=(HEIGHT,WIDTH,CHANNELS), N_CLASSES=N_CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_step(x_batch_train, y_batch_train):\n    with tf.GradientTape() as tape:\n\n        # Run the forward pass of the layer.\n        # The operations that the layer applies\n        # to its inputs are going to be recorded\n        # on the GradientTape.\n        preds = model(x_batch_train, training=True)  # Logits for this minibatch\n\n        # Compute the loss value for this minibatch.\n        if tpu:\n            loss_value = loss_fn(y_batch_train, preds)\n        else:\n            loss_value = bi_tempered_logistic_loss(preds,\n                              y_batch_train,\n                              t1=T1,\n                              t2=T2,\n                              label_smoothing=LABEL_SMOOTH,\n                              num_iters=5)\n\n    # Use the gradient tape to automatically retrieve\n    # the gradients of the trainable variables with respect to the loss.\n    grads = tape.gradient(loss_value, model.trainable_weights)\n\n    # Run one step of gradient descent by updating\n    # the value of the variables to minimize the loss.\n    optimizer.apply_gradients(zip(grads, model.trainable_weights))\n    epoch_accuracy.update_state(y_batch_train, preds)\n    return loss_value\n\ndef test_step(x_test,y_test):\n    preds = model(x_test, training=False)  # Logits for this minibatch\n    # Compute the loss value for this minibatch.\n    val_loss_value = loss_fn(y_test, preds)\n    val_loss.update_state(val_loss_value)\n    val_accuracy.update_state(y_test, preds)\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not tpu:\n    print_time = True\n    tf.config.run_functions_eagerly(True)\n    step_print = 500\nelse:\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    print_time = True\n    step_print = 25\nimport gc\nimport functools\nmodels = []\nhistories = []\n#\nskf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)\n# Loop for each model fold\nfor fold,(idxT, idxV) in enumerate(skf.split(np.arange(15))):\n    @tf.function\n    def distributed_train_step(x_batch_train, y_batch_train):\n        per_replica_losses = strategy.run(train_step, args=(x_batch_train, y_batch_train))\n        return strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_losses,axis=0) / BATCH_SIZE / REPLICAS\n    @tf.function\n    def distributed_test_step(x_test, y_test):\n        return strategy.run(test_step, args=(x_test,y_test,))\n    K.clear_session()\n    fold_time = time.time()\n    print(\"Start of FOLD %d.\" % (fold))\n    print(\"Training: {} Validation: {}\".format(idxT,idxV))\n    # Instantiate the model for this fold\n    model = model_fn(input_shape=(HEIGHT,WIDTH,CHANNELS), N_CLASSES=N_CLASSES)\n    # Instantiate fold history\n    train_loss_history = []\n    train_acc_history = []\n    val_loss_history = []\n    val_acc_history = []\n    # Instantiate an optimizer.\n    optimizer = optimizers.Adam(LR_START)\n    \n    \n    #____________________DATASETS INITIALIZATION\n    TRAIN_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/train_tfrecords/ld_train%.2i*.tfrec' % x for x in idxT])\n    VALID_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/train_tfrecords/ld_train%.2i*.tfrec' % x for x in idxV])\n    TRAIN_COUNT = count_data_items(TRAIN_FILENAMES)\n    steps = math.floor(TRAIN_COUNT/BATCH_SIZE)\n    np.random.shuffle(TRAIN_FILENAMES)\n    train_dataset = get_training_dataset(load_dataset(TRAIN_FILENAMES,labeled = True),do_aug = True,is_one_hot = True)\n    val_dataset = get_validation_dataset(VALID_FILENAMES,is_crop = True,is_one_hot = True)\n    \n    \n    \n    best_val_loss = 1e3\n    fold_patience = 0\n    with strategy.scope():\n        loss_fn = keras.losses.CategoricalCrossentropy(from_logits=True,reduction=tf.keras.losses.Reduction.NONE,label_smoothing= LABEL_SMOOTH)\n        epoch_accuracy = tf.keras.metrics.CategoricalAccuracy()\n        val_accuracy = tf.keras.metrics.CategoricalAccuracy()\n        val_loss = tf.keras.metrics.Mean()\n    for epoch in range(EPOCHS):\n        step = 0\n        loss = 0\n        start_time = time.time()\n        print(\"Start of epoch %d. LR: %f \\n\" % (epoch,optimizer.learning_rate))\n        for x_batch_train,y_batch_train in train_dataset:\n            optimizer.learning_rate = lrfn(epoch = epoch,change_on_step = True,step = step,max_steps = steps)\n            with strategy.scope():\n                loss_value = distributed_train_step(\n                    x_batch_train,\n                    y_batch_train)\n            # Log every step_print batches.\n            loss += loss_value\n            step += 1\n            if step % step_print == 0 or step == steps or step == 1:\n                \n                if print_time or step == steps - 1:\n                    print(\"Time taken this epoch: %.2fs per step: %.2fs current lr: %f\" % (\n                        time.time() - start_time,(time.time() - start_time)/(step),\n                        optimizer.learning_rate))\n                print(\n                    \"Training loss at step %d/%d: %.4f  Accuracy: %.4f\"\n                    % ((step),steps,loss/step,epoch_accuracy.result()))                                                                        \n        val_time = time.time()\n        for x_val,y_val in val_dataset:\n            with strategy.scope():\n                distributed_test_step(x_val,y_val)\n                train_loss_history.append(loss/step)\n                train_acc_history.append(epoch_accuracy.result())\n                val_loss_history.append(val_loss.result())\n                val_acc_history.append(val_accuracy.result())\n        print(\"\\nValidation for epoch %d:    Time taken for validation: %.2fs\"%(epoch,time.time() - val_time))\n        print(\"val_acc : %.4f val_loss : %.4f\"%(val_accuracy.result(),val_loss.result()))\n        if val_loss.result().numpy() < best_val_loss:\n            best_val_loss = val_loss.result().numpy()\n            path  = f'Best_Save_EFFNETB3:_Fold:{fold}.hdf5'\n            model.save(path)\n            print('Model Saved \\n')\n            fold_patience = 0\n        else:\n            fold_patience += 1\n            if fold_patience == ES_PATIENCE:\n                print('Early Stepping due to model not improving for %f epochs'.format(ES_PATIENCE))\n                break\n        \n        print(\"\\nTime taken for this fold so far : %.2fs \\n\"%(time.time() - fold_time))\n        \n        with strategy.scope():\n            epoch_accuracy.reset_states()\n            val_accuracy.reset_states()\n            val_loss.reset_states()\n    models.append(model)\n    histories.append([train_loss_history,train_acc_history,val_loss_history,val_acc_history])\n    path  = f'Cassava_Model_EFFNETB3:_Fold:{fold}.hdf5'\n    model.save(path)\n    #del(model,optimizer,loss_fn,history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"models = []\nhistories = []\n#\nskf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)\n# Loop for each model fold\nfor fold,(idxT, idxV) in enumerate(skf.split(np.arange(15))):\n    optimizer = optimizers.Adam(LR_START)\n    # Instantiate a loss function.\n    with strategy.scope():\n        model = model_fn(input_shape=(HEIGHT,WIDTH,CHANNELS), N_CLASSES=N_CLASSES)\n    TRAIN_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/train_tfrecords/ld_train%.2i*.tfrec' % x for x in idxT])\n    VALID_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/train_tfrecords/ld_train%.2i*.tfrec' % x for x in idxV])\n    TRAIN_COUNT = count_data_items(TRAIN_FILENAMES)\n    steps = math.floor(TRAIN_COUNT/BATCH_SIZE)\n    np.random.shuffle(TRAIN_FILENAMES)\n    train_dataset = get_training_dataset(load_dataset(TRAIN_FILENAMES,labeled = True),do_aug = True,is_one_hot = True)\n    val_dataset = get_validation_dataset(VALID_FILENAMES,is_crop = True,is_one_hot = True)\n    es = EarlyStopping(monitor='val_loss', mode='min', \n                       patience=ES_PATIENCE, restore_best_weights=True, verbose=1)\n    history = model.fit(x=train_dataset, \n                        validation_data=val_dataset, \n                        steps_per_epoch=steps, \n                        callbacks=[es, LearningRateScheduler(lrfn, verbose=0)], \n                        epochs=EPOCHS,  \n                        verbose=1).history"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(histories[0][0])\nplt.plot(histories[0][2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(histories[0][1])\nplt.plot(histories[0][3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(histories[1][0])\nplt.plot(histories[1][2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(histories[1][1])\nplt.plot(histories[1][3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(histories[2][0])\nplt.plot(histories[2][2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(histories[2][1])\nplt.plot(histories[2][3])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}