{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Install and Import packages","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U git+https://github.com/qubvel/efficientnet >> /dev/null","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\n\nimport efficientnet.tfkeras as efn\n\nfrom sklearn.metrics import *\n\nfrom kaggle_datasets import KaggleDatasets\n\nimport gc\nimport time\nimport re\nimport math\n\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Setup TPUs","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Detect TPU, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() \n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Enable Mixed Precision\n\nStill need to validate if it makes it faster\nWhen you use this, the output values are in format bfloat16","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"MIXED_PRECISION = True\nXLA_ACCELERATE = True\n\nif MIXED_PRECISION:\n    from tensorflow.keras.mixed_precision import experimental as mixed_precision\n    if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')\n    else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n    mixed_precision.set_policy(policy)\n    print('Mixed precision enabled')\n\nif XLA_ACCELERATE:\n    tf.config.optimizer.set_jit(True)\n    print('Accelerated Linear Algebra enabled')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Setup Image sizes and paths","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO : Try with bigger image size\n# TODO : Try with larger batch sizes\n\nIMAGE_SIZES = [[512, 512], [331, 331], [224, 224], [192, 192]]\nIMAGE_SIZE = IMAGE_SIZES[2]\n\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\n\nprint('Using IMAGE_SIZE', IMAGE_SIZE, 'and BATCH_SIZE', BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use_external = True\n\nBASE_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')\nEXT_PATH = KaggleDatasets().get_gcs_path('tf-flower-photo-tfrec')\n\ndef get_path(image_size=IMAGE_SIZE):\n    # image_size is [512, 512] for example\n    return f'{BASE_PATH}/tfrecords-jpeg-{image_size[0]}x{image_size[1]}'\n\ndef get_ext_path(folder, image_size=IMAGE_SIZE):\n    return f'{EXT_PATH}/{folder}/tfrecords-jpeg-{image_size[0]}x{image_size[1]}'\n\nTRAINING_FILES = tf.io.gfile.glob(get_path() + '/train/*.tfrec')\nVALIDATION_FILES = tf.io.gfile.glob(get_path() + '/val/*.tfrec')\nTEST_FILES = tf.io.gfile.glob(get_path() + '/test/*.tfrec')\n\nif use_external:\n#     IMAGENET = tf.io.gfile.glob(get_ext_path('imagenet_no_test') + '/*.tfrec')\n#     TRAINGING_FILES += IMAGENET\n    INATURALIST = tf.io.gfile.glob(get_ext_path('inaturalist_no_test') + '/*.tfrec')\n    TRAINING_FILES += INATURALIST\n#     OPENIMAGE = tf.io.gfile.glob(get_ext_path('openimage_no_test') + '/*.tfrec')\n#     TRAINING_FILES += OPENIMAGE\n    OXFORD = tf.io.gfile.glob(get_ext_path('oxford_102_no_test') + '/*.tfrec')\n    TRAINING_FILES += OXFORD\n    TFFLOWERS = tf.io.gfile.glob(get_ext_path('tf_flowers_no_test') + '/*.tfrec')\n    TRAINING_FILES += TFFLOWERS\n\nprint('Loaded Training:', len(TRAINING_FILES), 'files, Validation:', len(VALIDATION_FILES), 'files, Test:', len(TEST_FILES), 'files')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102\n\nprint(len(CLASSES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augment images\n\nTried and failed to use albumenations with tensorflow. It works on GPUs but not on TPUs. More specifically, it throws errors when called via `tf.numpy_function` ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO : Find other augmentations that can be used with TPU with TF\n\nclass Augmentor():\n    \n    @staticmethod\n    def process(img, size):\n        img = Augmentor.transform(img, IMAGE_SIZE[0])\n        img = tf.image.random_brightness(img, 0.3)\n        img = tf.image.random_contrast(img, 0.8, 1.2)\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n#         img = tf.image.random_hue(img, 0.2)\n        img = tf.image.random_saturation(img, 0.8, 1.2)\n        img = Augmentor.dropout(img, size)\n        return img\n\n    @staticmethod\n    def dropout(img, size, probability=0.5, count=2, ratio=0.4):\n        probability = tf.cast( \n            tf.random.uniform([],0,1) < probability, \n            tf.int32\n        )\n        if (probability == 0) | (count == 0) | (ratio == 0):\n            return img\n\n        for k in range(count):\n            x = tf.cast( tf.random.uniform([], 0, size), tf.int32)\n            y = tf.cast( tf.random.uniform([], 0, size), tf.int32)\n            \n            # COMPUTE SQUARE \n            width = tf.cast( ratio*size, tf.int32 ) * probability\n            ya = tf.math.maximum(0, y-width//2)\n            yb = tf.math.minimum(size, y+width//2)\n            xa = tf.math.maximum(0,x-width//2)\n            xb = tf.math.minimum(size, x+width//2)\n            \n            # DROPOUT IMAGE\n            one = img[ya:yb, 0:xa, :]\n            two = tf.random.normal([yb-ya, xb-xa, 3])\n            three = img[ya:yb, xb:size, :]\n            middle = tf.concat([one,two,three],axis=1)\n            img = tf.concat([img[0:ya,:,:], middle, img[yb:size,:,:]], axis=0)\n\n        # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n        img = tf.reshape(img, [size, size, 3])\n        return img\n\n    @staticmethod\n    def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n        # returns 3x3 transformmatrix which transforms indicies\n\n        # CONVERT DEGREES TO RADIANS\n        rotation = math.pi * rotation / 180.\n        shear    = math.pi * shear    / 180.\n\n        def get_3x3_mat(lst):\n            return tf.reshape(tf.concat([lst],axis=0), [3,3])\n\n        # ROTATION MATRIX\n        c1   = tf.math.cos(rotation)\n        s1   = tf.math.sin(rotation)\n        one  = tf.constant([1],dtype='float32')\n        zero = tf.constant([0],dtype='float32')\n\n        rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                       -s1,  c1,   zero, \n                                       zero, zero, one])    \n        # SHEAR MATRIX\n        c2 = tf.math.cos(shear)\n        s2 = tf.math.sin(shear)    \n\n        shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                    zero, c2,   zero, \n                                    zero, zero, one])        \n        # ZOOM MATRIX\n        zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                                   zero,            one/width_zoom, zero, \n                                   zero,            zero,           one])    \n        # SHIFT MATRIX\n        shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                    zero, one,  width_shift, \n                                    zero, zero, one])\n\n        return K.dot(K.dot(rotation_matrix, shear_matrix), \n                     K.dot(zoom_matrix,     shift_matrix))\n\n\n    @staticmethod\n    def transform(image, DIM=256):    \n        # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n        # output - image randomly rotated, sheared, zoomed, and shifted\n        XDIM = DIM%2 #fix for size 331\n\n        rot = 180.0 * tf.random.normal([1], dtype='float32')\n        shr = 2.0 * tf.random.normal([1], dtype='float32') \n        h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / 8.0\n        w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / 8.0\n        h_shift = 8.0 * tf.random.normal([1], dtype='float32') \n        w_shift = 8.0 * tf.random.normal([1], dtype='float32') \n\n        # GET TRANSFORMATION MATRIX\n        m = Augmentor.get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n        # LIST DESTINATION PIXEL INDICES\n        x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n        y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n        z   = tf.ones([DIM*DIM], dtype='int32')\n        idx = tf.stack( [x,y,z] )\n\n        # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n        idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n        idx2 = K.cast(idx2, dtype='int32')\n        idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n\n        # FIND ORIGIN PIXEL VALUES           \n        idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n        d    = tf.gather_nd(image, tf.transpose(idx3))\n\n        return tf.reshape(d,[DIM, DIM,3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class DS():\n    \n    def __init__(self, files, training=True, shuffle=False, augment=False, batch_size=16, repeat=False):\n        self.files = files\n        self.augment = augment\n\n        self.ds = tf.data.TFRecordDataset(files, num_parallel_reads = AUTO)\n        self.ds = self.ds.map(self.read_with_label if training else self.read_without_label)\n        \n        if self.augment:\n            self.ds = self.ds.map(self.augment_image)\n        \n        if shuffle:\n            options = tf.data.Options()\n            options.experimental_deterministic = False\n            self.ds = self.ds.with_options(options)\n            self.ds = self.ds.shuffle(2048)\n            \n        if repeat:\n            self.ds = self.ds.repeat()\n\n        self.ds = self.ds.batch(batch_size)\n        self.ds = self.ds.prefetch(AUTO) # Improves throughput\n        # self.ds = self.ds.cache() # Uses too much memory, ignore\n\n    def data(self):\n        return self.ds\n        \n    def read_with_label(self, example):\n        example = tf.io.parse_single_example(example, {\n            'image': tf.io.FixedLenFeature([], tf.string),\n            'class': tf.io.FixedLenFeature([], tf.int64),\n        })\n        return self.decode_image(example['image']), tf.cast(example['class'], tf.int32)\n    \n    def read_without_label(self, example):\n        example = tf.io.parse_single_example(example, {\n            'image': tf.io.FixedLenFeature([], tf.string),\n            'id': tf.io.FixedLenFeature([], tf.string),\n        })\n        return self.decode_image(example['image']), example['id']\n    \n    def decode_image(self, img):\n        img = tf.io.decode_jpeg(img, channels=3)\n        img = tf.cast(img, tf.float32) / 255.0\n        return tf.reshape(img, [*IMAGE_SIZE, 3])\n    \n    def augment_image(self, img, label):\n        return Augmentor.process(img, IMAGE_SIZE[0]), label\n    \n    def count(self):\n        # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n        n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in self.files]\n        return np.sum(n)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test loading times with different parameters\n\nds.cache is turned off because it leads to OOM errors. Prefetching is still turned on.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"items = 0\nbatches = 0\ntimes = { \n    'batches': list(),\n    'items': list(),\n    'batch_speed': list(),\n    'item_speed': list(),\n}\n\ndef loop(items, batches, times):\n    train_ds = DS(TRAINING_FILES, augment=True, batch_size=128).data()\n#     train_ds = DS(VALIDATION_FILES, batch_size=128).data()\n\n    start = time.time()\n    for element in train_ds:\n        so_far = time.time() - start\n        batches += 1\n        items += len(element[0])\n\n        times['batches'].append(batches)\n        times['items'].append(items)\n        times['batch_speed'].append((batches+1)/so_far)\n        times['item_speed'].append((items+1)/so_far)\n\n        print(items, so_far, (items)/so_far)\n\n    print(batches, items, so_far, (batches)/so_far, (items)/so_far)\n    gc.collect()\n\ndef show_graph(size):\n    plt.figure(figsize=(12,6))\n    plt.plot(times['batches'], times['batch_speed'], color='red')\n    plt.plot(times['items'], times['item_speed'], color='blue', linestyle='--')\n    plt.legend()\n    plt.title('Batch size' + str(size))\n    plt.show()\n    \n# loop(items, batches, times) # 270-280 images/sec, 170-180/sec with albumenations, 60 with tensorflow\n# show_graph(128)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test if the images are loading from the dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_dataset():\n    print('Training images')\n    show_images(DS(TRAINING_FILES, augment=True, batch_size=128).data())\n    print('Validation image')\n    show_images(DS(VALIDATION_FILES, batch_size=128).data())\n    print('Testing images')\n    show_images(DS(TEST_FILES, training=False, batch_size=128).data())\n    print('Testing Augmented images')\n    show_images(DS(TEST_FILES, training=False, augment=True, batch_size=128).data())\n    \ndef show_images(ds, rows=3, columns=6):\n    plt.figure(figsize=(12, 6))\n    count = 0\n    \n    for i, examples in enumerate(ds.take(1)):\n        images = examples[0]\n        labels = examples[1]\n        for j, image in enumerate(images):\n            if count == rows * columns:\n                break\n            plt.subplot(rows, columns, count+1, xticks=[], yticks=[])\n            plt.imshow(image)\n            if labels[0].dtype == tf.string:\n                plt.xlabel(labels[j].numpy().decode(\"utf-8\"))\n            else:\n                plt.xlabel(CLASSES[labels[j]])\n            count += 1\n    plt.tight_layout()\n    plt.show()\n    \n# test_dataset()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build the model\n\nEfficientNet takes up a lot of time to load. Build a silly model to test the pipeline","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"EFNS = [\n    efn.EfficientNetB0, \n    efn.EfficientNetB1, \n    efn.EfficientNetB2, \n    efn.EfficientNetB3,\n    efn.EfficientNetB4,\n    efn.EfficientNetB5,\n    efn.EfficientNetB6,\n    efn.EfficientNetB7,\n]\n\ndef create_efficientnet(index, weight='imagenet'):\n    pretrained_model = EFNS[index](\n        weights = weight,\n        include_top = False,\n        input_shape = [*IMAGE_SIZE, 3]\n    )\n    pretrained_model.trainable = True\n    \n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(1024, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.01)),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Dropout(0.5),\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax'),\n    ], name='efn' + str(index))\n    return model\n\ndef create_densenet():\n    pretrained_model = tf.keras.applications.DenseNet201(\n        weights = 'imagenet',\n        include_top = False,\n        input_shape = [*IMAGE_SIZE, 3]\n    )\n    pretrained_model.trainable = True\n    \n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(1024, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.01)),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Dropout(0.5),\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax'),\n    ], name='densenet')\n    return model\n\ndef create_cnn():\n    model = tf.keras.Sequential([\n        tf.keras.layers.Conv2D(16, kernel_size=3, strides=2, padding='same', activation='relu', input_shape=[*IMAGE_SIZE, 3]),\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n    ], name='simple')\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO : Make a model class\n\ndef compile_models(models):\n    for x in models:\n        models[x].compile(\n            optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3),\n            loss = tf.keras.losses.SparseCategoricalCrossentropy(),\n            metrics = ['sparse_categorical_accuracy'],\n        )\n        models[x].summary()\n        print('')\n\nmodels = {}\n\nwith strategy.scope():\n    \n#     start = time.time()\n#     models['simple'] = create_cnn()\n#     print(f'Created simple in {(time.time() - start):.4f}s')\n    \n#     start = time.time()\n#     models['simple2'] = create_cnn()\n#     print(f'Created simple2 in {(time.time() - start):.4f}s')\n\n    start = time.time()\n    models['efn7'] = create_efficientnet(7)\n    print(f'Created efn7 in {(time.time() - start):.4f}s')\n\n    start = time.time()\n    models['efn6'] = create_efficientnet(6)\n    print(f'Created efn6 in {(time.time() - start):.4f}s')\n    \n#     start = time.time()\n#     models['efn5'] = create_efficientnet(5)\n#     print(f'Created efn5 in {(time.time() - start):.4f}s')\n\n#     start = time.time()\n#     models['efn4'] = create_efficientnet(4)\n#     print(f'Created efn4 in {(time.time() - start):.4f}s')\n\n    start = time.time()\n    models['efn4-noisy'] = create_efficientnet(4, 'noisy-student')\n    print(f'Created efn4 in {(time.time() - start):.4f}s')\n\n#     start = time.time()\n#     models['efn3'] = create_efficientnet(3)\n#     print(f'Created efn3 in {(time.time() - start):.4f}s')\n\n    start = time.time()\n    models['densenet'] = create_densenet()\n    print(f'Created densenet in {(time.time() - start):.4f}s')\n    \n\ncompile_models(models)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Setup datasets for the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = DS(TRAINING_FILES, augment=True, shuffle=True, batch_size=BATCH_SIZE, repeat=True)\nval_ds = DS(VALIDATION_FILES, batch_size=BATCH_SIZE)\nval_aug_ds = DS(VALIDATION_FILES, augment=True, batch_size=BATCH_SIZE)\ntest_ds = DS(TEST_FILES, training=False, batch_size=BATCH_SIZE)\n\nprint(f'Training: {train_ds.count()} images, Validation: {val_ds.count()}, Test: {test_ds.count()} images')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Setup learning rate","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO : Try with CosineDecayRestarts in LearningRateScheduler\n# Didn't work : change lr_max to use \n#    LR_START = 0.00001\n#    LR_MAX = 0.00005 * strategy.num_replicas_in_sync\n#    LR_MIN = 0.00001\n\n#1 \nlr_start = 0.00005\nlr_max = 0.0000125 * strategy.num_replicas_in_sync #BATCH_SIZE\nlr_min = 0.0000001\nlr_ramp_ep = 5\nlr_sus_ep = 0\nlr_decay = 0.8\n\ndef lrfn(epoch):\n    if epoch < lr_ramp_ep:\n        lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n    elif epoch < lr_ramp_ep + lr_sus_ep:\n        lr = lr_max\n    else:\n        lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n    return lr\n\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\ncount = range(100)\nplt.figure(figsize=(12, 6))\nplt.subplot(1, 3, 1)\nplt.plot(count, [lrfn(x) for x in count])\n\n#2\ncosine_decay_learning_rate = tf.keras.experimental.CosineDecayRestarts(\n        0.001, 0.8, t_mul=2.0, m_mul=1.0, alpha=0.0,\n        name=None\n    )\n\n# lr_callback = tf.keras.callbacks.LearningRateScheduler(cosine_decay_learning_rate, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Visibility(tf.keras.callbacks.Callback):\n    \n    def __init__(self):\n        super(Visibility, self).__init__()\n        self.metrics = ['sparse_categorical_accuracy', 'loss']\n    \n    def on_train_begin(self, logs=None):\n        # Training starts here\n        self.train_begin = time.time()\n\n    def on_train_end(self, logs=None):\n        print('\\n Training took ', time.time() - self.train_begin, 'seconds')\n\n#     def on_epoch_begin(self, epoch, logs=None):\n#         pass\n\n    def on_epoch_end(self, epoch, logs=None):\n        output = \"\"\n        for metric in self.metrics:\n            diff = ((logs['val_'+metric] / logs[metric]) - 1)*100\n            output += f\"val/{metric}: {diff:.2f}% \"\n        print(\"\\n\" + output)\n\n    def on_train_batch_begin(self, batch, logs=None):\n        print(\"▪\", end=\"\")\n\n#     def on_test_begin(self, logs=None):\n#         # Validation starts\n#         pass\n#     def on_test_end(self, logs=None):\n#         pass\n#     def on_train_batch_end(self, batch, logs=None):\n#         pass\n#     def on_test_batch_begin(self, batch, logs=None):\n#         pass\n\n    def on_test_batch_end(self, batch, logs=None):\n        print(\"▫\", end=\"\")\n\n#     def on_predict_begin(self, logs=None):\n#         pass\n#     def on_predict_end(self, logs=None):\n#         pass\n#     def on_predict_batch_begin(self, batch, logs=None):\n#         pass\n#     def on_predict_batch_end(self, batch, logs=None):\n#         pass","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train the models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 50\nhistory = {}\n\ndef fit_models(models, history):\n    for x in models:\n        print(f'Starting training for {x}')\n        hist = models[x].fit(\n            train_ds.data(),\n            validation_data = val_ds.data(),\n            epochs = EPOCHS,\n            steps_per_epoch = train_ds.count() // BATCH_SIZE,\n            verbose=2,\n            callbacks=[\n                tf.keras.callbacks.ModelCheckpoint(\n                    monitor='val_sparse_categorical_accuracy', verbose=2, save_best_only=True, save_weights_only=True, mode='max', filepath=f'{x}.h5'),\n                lr_callback,\n                tf.keras.callbacks.EarlyStopping(\n                    monitor='val_sparse_categorical_accuracy', min_delta=0.001, patience=3, verbose=2, mode='max', baseline=None, restore_best_weights=True\n                ),\n                Visibility(),\n            ],\n        )\n        history[x] = hist\n\nfit_models(models, history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Plot metrics, val_metrics and overfitting\n\nPlotted based on the metric, \n* greater than 0 means overfitting\n* less than 0 means underfitting","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_metrics(name, h):\n    plt.figure(figsize=(14, 6))\n\n    plt.subplot(1,2,1)\n    plt.plot(h.epoch, h.history['loss'], color='blue')\n    plt.plot(h.epoch, h.history['val_loss'], color='blue', linestyle='--')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend(loc='upper right')\n    plt.title(name)\n\n    plt.subplot(1,2,2)\n    plt.plot(h.epoch, h.history['sparse_categorical_accuracy'], color='orange')\n    plt.plot(h.epoch, h.history['val_sparse_categorical_accuracy'], color='orange', linestyle='--')\n    plt.xlabel('Epochs')\n    plt.ylabel('Sparse Categorical Accuracy')\n    plt.legend(loc='upper right')\n    plt.title(name)\n\n    plt.show()\n    \ndef plot_overfitting(name, h):\n    loss_gap = np.array(h.history['val_loss']) - np.array(h.history['loss'])\n    acc_gap = 100*(np.array(h.history['sparse_categorical_accuracy']) \n                   - np.array(h.history['val_sparse_categorical_accuracy']))\n\n    plt.figure(figsize=(14, 6))\n\n    plt.subplot(1, 3, 1)\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss Gap')\n    plt.plot(h.epoch, loss_gap, color='blue')\n    plt.title(name)\n\n    plt.subplot(1, 3, 2)\n    plt.xlabel('Epochs')\n    plt.ylabel('Accuracy Gap')\n    plt.plot(h.epoch, acc_gap, color='orange')\n    plt.title(name)\n\n    plt.subplot(1, 3, 3)\n    plt.xlabel('Epochs')\n    plt.ylabel('Learning Rate')\n    plt.plot(h.epoch, h.history['lr'], color='red')\n    plt.title(name)\n\n    plt.legend()\n    plt.show()\n    \nfor x in history:\n    plot_metrics(x, history[x])\n    plot_overfitting(x, history[x])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validate the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"highest_f1 = 0\nhighest_f1_model = ''\npreds = list()\n\ndef get_y_true():\n\n    def get_val_classes():\n        all_classes = np.array([])\n        for examples in val_ds.data():\n            all_classes = np.concatenate([all_classes, examples[1].numpy()])\n        return all_classes\n\n    y_true = get_val_classes()\n    print('True Values:: Length:', len(y_true), ', List:', y_true)\n    return y_true\n\ndef predict_models(models, y_true):\n    global highest_f1\n    global highest_f1_model\n    global preds\n    \n     # TPU doesn't like strings, get rid of them\n    val_tpu_ds = val_aug_ds.data().map(lambda image, ids: image)\n    for x in models:\n        val_aug_count = 2\n        y_val_pred = np.zeros((3712,104), dtype=np.float64)\n        for i in range(val_aug_count):\n            y_val_loop = models[x].predict(val_tpu_ds, verbose=1)\n            y_val_loop = tf.cast(y_val_loop, tf.float32)\n            y_val_pred += y_val_loop.numpy()\n        y_val_pred /= val_aug_count\n        y_val = np.argmax(y_val_pred, axis=-1).astype(np.float32)\n\n        print(f'\\n\\nScores for {x}\\n\\n')\n        print(f'{x}: Pred Values:: Length:{len(y_val)} List: {y_val}')\n        \n        model_f1_score = f1_score(y_true, y_val, average='macro', zero_division=0)*100\n        if model_f1_score > highest_f1:\n            highest_f1 = model_f1_score\n            highest_f1_model = x\n\n        scores = {\n            \"metrics\": [\n                \"F1\",\n                \"Precision\",\n                \"Recall\",\n            ],\n            \"scores\": [\n                model_f1_score,\n                precision_score(y_true, y_val, average='macro', zero_division=0)*100,\n                recall_score(y_true, y_val, average='macro', zero_division=0)*100,\n            ]\n        }\n        \n        print(pd.DataFrame.from_dict(scores))\n\n        preds.append((\n            x,\n            y_val_pred,\n            model_f1_score,\n        ))\n        # print( classification_report(y_true, y_val, target_names=CLASSES) )\n\ny_true = get_y_true()\npredict_models(models, y_true)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# x, 1\n# y, alpha1\n# z, alpha2\n\ndef get_best_alpha(pred1, pred2):\n    # Given two predictions, find the highest alpha and f1\n    best_alpha = 0\n    best_f1 = 0\n    \n    for x in np.linspace(0, 1, 101):\n        mixed = (pred1 * x)  + (pred2 * (1-x))\n        y_mixed = np.argmax(mixed, axis=-1).astype(np.float32)\n        mixed_f1 = f1_score(y_true, y_mixed, average='macro', zero_division=0)*100        \n        if mixed_f1 > best_f1:\n            best_f1 = mixed_f1\n            best_alpha = x\n    return best_alpha, best_f1\n\n\ndef get_alpha(preds, y_true):\n    output = list()\n    \n    # sort the model by highest f1 score\n    preds = np.array(preds)\n    preds = preds[preds[:,2].argsort()[::-1]]\n    \n    model_name = preds[0][0]\n    predictions = preds[0][1]\n    model_f1 = preds[0][2]\n    output.append((model_name, 1))\n    \n    best_predictions = predictions\n    best_f1 = model_f1\n    print(best_f1, output)\n\n    for i in range(1, len(preds)):\n        i_name = preds[i][0]\n        i_preds = preds[i][1]\n        i_f1 = preds[i][1]\n        \n        alpha, f1 = get_best_alpha(best_predictions, i_preds)\n        if f1 > best_f1:\n            best_predictions = (alpha * i_preds) + ((1-alpha) * best_predictions)\n            best_f1 = f1\n            output.append((i_name, alpha))\n            print(best_f1, output)\n            \n    return output\n    \nalpha_chain = get_alpha(preds, y_true)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create final prediction and submit","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'{highest_f1_model} has the highest f1 score - {highest_f1:0.5f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO : Create an ensemble from multiple models\n\nds_submission = DS(TEST_FILES, training=False, augment=True, batch_size=BATCH_SIZE).data()\nds_submission = ds_submission.map(lambda image, ids: image)\n\ndef get_test_ids():\n    all_ids = np.array([])\n    for examples in test_ds.data():\n        all_ids = np.concatenate([all_ids, examples[1].numpy().astype('U')])\n    return all_ids\n\ndef get_prediction_with_tta(model, ds_submission):\n    \n    tta = 5\n    \n    y_pred = np.zeros((7382,104), dtype=np.float64)\n    \n    for x in range(tta):    \n        y_pred_loop = model.predict(ds_submission, verbose=2)\n        y_pred_loop = tf.cast(y_pred_loop, tf.float32)\n        y_pred += y_pred_loop.numpy()\n    \n    y_pred /= tta\n    return y_pred\n    \ndef create_submission(alpha_chain, models, ds_submission):\n    \n    y_pred = np.zeros((7382,104), dtype=np.float64)\n    \n    for x in alpha_chain:\n        model = x[0]\n        alpha = x[1]\n        \n        print('Predicting with', model, 'with alpha', alpha)\n        y_pred_chain = get_prediction_with_tta(models[model], ds_submission)\n        y_pred = (alpha * y_pred_chain) + ((1-alpha) * y_pred)\n    \n    y_pred = np.argmax(y_pred, axis=-1)\n\n    filename = f'submission.csv'\n    np.savetxt(\n        filename, \n        np.column_stack((get_test_ids(), y_pred)),\n        fmt=('%s', '%s'),\n        delimiter=',',\n        header='id,label',\n        comments=''\n    )\n\n# create_submission(models[highest_f1_model])\ncreate_submission(alpha_chain, models, ds_submission)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}