{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import math, re, os, time\nimport datetime\nimport tensorflow as tf\nimport numpy as np\nfrom collections import namedtuple, Counter\nimport json\nfrom matplotlib import pyplot as plt\nfrom matplotlib import gridspec\nimport itertools \nfrom kaggle_datasets import KaggleDatasets\nimport sklearn\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nimport pandas as pd\n\nprint(\"Tensorflow version \" + tf.__version__)\n\nfrom tensorflow.keras.applications import Xception\nfrom tensorflow.keras.applications import DenseNet121, DenseNet169, DenseNet201\nfrom tensorflow.keras.applications import ResNet50V2, ResNet101V2, ResNet152V2\nfrom tensorflow.keras.applications import InceptionV3\nfrom tensorflow.keras.applications import InceptionResNetV2\n\n# Only for tensorflow 2.3\n# from tensorflow.keras.applications import EfficientNetB0, EfficientNetB1, EfficientNetB2, EfficientNetB3, EfficientNetB4, EfficientNetB5, EfficientNetB6, EfficientNetB7\n\n!pip install -q efficientnet\nimport efficientnet.tfkeras as efn\n\nMODEL_CLASSES = {\n    'Xception': Xception,\n    'DenseNet121': DenseNet121,\n    'DenseNet169': DenseNet169,\n    'DenseNet201': DenseNet201,\n    'ResNet50V2': ResNet50V2,\n    'ResNet101V2': ResNet101V2,\n    'ResNet152V2': ResNet152V2,\n    'InceptionV3': InceptionV3,\n    'InceptionResNetV2': InceptionResNetV2,\n    'EfficientNetB0': efn.EfficientNetB0,\n    'EfficientNetB1': efn.EfficientNetB1,\n    'EfficientNetB2': efn.EfficientNetB2,\n    'EfficientNetB3': efn.EfficientNetB3,\n    'EfficientNetB4': efn.EfficientNetB4,\n    'EfficientNetB5': efn.EfficientNetB5,\n    'EfficientNetB6': efn.EfficientNetB6,\n    'EfficientNetB7': efn.EfficientNetB7,\n}\n\nimport gc\ngc.enable()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:23:56.945591Z","iopub.execute_input":"2021-05-30T09:23:56.945993Z","iopub.status.idle":"2021-05-30T09:24:03.392388Z","shell.execute_reply.started":"2021-05-30T09:23:56.945937Z","shell.execute_reply":"2021-05-30T09:24:03.391383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\n\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:03.393924Z","iopub.execute_input":"2021-05-30T09:24:03.394224Z","iopub.status.idle":"2021-05-30T09:24:08.883916Z","shell.execute_reply.started":"2021-05-30T09:24:03.394195Z","shell.execute_reply":"2021-05-30T09:24:08.882943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started') # you can list the bucket with \"!gsutil ls $GCS_DS_PATH\"\n\nprint(f'GCS_DS_PATH = {GCS_DS_PATH}\\n')","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:08.885935Z","iopub.execute_input":"2021-05-30T09:24:08.886355Z","iopub.status.idle":"2021-05-30T09:24:09.224164Z","shell.execute_reply.started":"2021-05-30T09:24:08.886315Z","shell.execute_reply":"2021-05-30T09:24:09.223479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# At size `512`, a GPU will run out of memory, so we use the TPU.\n# For GPU training, please select 224 x 224 px image size.\nIMAGE_SIZE = [331, 331] \n\nGCS_PATH_SELECT = { # available image sizes\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\n# Select the dataset containing the size we chose above\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec') # predictions on this dataset should be submitted for the competition","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:11.396471Z","iopub.execute_input":"2021-05-30T09:24:11.397016Z","iopub.status.idle":"2021-05-30T09:24:11.597553Z","shell.execute_reply.started":"2021-05-30T09:24:11.396984Z","shell.execute_reply":"2021-05-30T09:24:11.596538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH_EXT = KaggleDatasets().get_gcs_path('tf-flower-photo-tfrec')\n# External data\nGCS_PATH_SELECT_EXT = {\n    192: '/tfrecords-jpeg-192x192',\n    224: '/tfrecords-jpeg-224x224',\n    331: '/tfrecords-jpeg-331x331',\n    512: '/tfrecords-jpeg-512x512'\n}\nGCS_PATH_EXT = GCS_PATH_SELECT_EXT[IMAGE_SIZE[0]]\n\nIMAGENET_FILES = tf.io.gfile.glob(GCS_DS_PATH_EXT + '/imagenet' + GCS_PATH_EXT + '/*.tfrec')\nINATURELIST_FILES = tf.io.gfile.glob(GCS_DS_PATH_EXT + '/inaturalist' + GCS_PATH_EXT + '/*.tfrec')\nOPENIMAGE_FILES = tf.io.gfile.glob(GCS_DS_PATH_EXT + '/openimage' + GCS_PATH_EXT + '/*.tfrec')\nOXFORD_FILES = tf.io.gfile.glob(GCS_DS_PATH_EXT + '/oxford_102' + GCS_PATH_EXT + '/*.tfrec')\nTENSORFLOW_FILES = tf.io.gfile.glob(GCS_DS_PATH_EXT + '/tf_flowers' + GCS_PATH_EXT + '/*.tfrec')\n\nADDITIONAL_TRAINING_FILENAMES = IMAGENET_FILES + INATURELIST_FILES + OPENIMAGE_FILES + OXFORD_FILES + TENSORFLOW_FILES  ","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:15.27243Z","iopub.execute_input":"2021-05-30T09:24:15.272779Z","iopub.status.idle":"2021-05-30T09:24:16.033146Z","shell.execute_reply.started":"2021-05-30T09:24:15.27275Z","shell.execute_reply":"2021-05-30T09:24:16.032046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAINING_FILENAMES = TRAINING_FILENAMES + ADDITIONAL_TRAINING_FILENAMES","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:19.731777Z","iopub.execute_input":"2021-05-30T09:24:19.732254Z","iopub.status.idle":"2021-05-30T09:24:19.738056Z","shell.execute_reply.started":"2021-05-30T09:24:19.73222Z","shell.execute_reply":"2021-05-30T09:24:19.737149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 103","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:24.352983Z","iopub.execute_input":"2021-05-30T09:24:24.353324Z","iopub.status.idle":"2021-05-30T09:24:24.361063Z","shell.execute_reply.started":"2021-05-30T09:24:24.353295Z","shell.execute_reply":"2021-05-30T09:24:24.360305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"number of flower classes: {len(CLASSES)}\")","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:25.374537Z","iopub.execute_input":"2021-05-30T09:24:25.37506Z","iopub.status.idle":"2021-05-30T09:24:25.379736Z","shell.execute_reply.started":"2021-05-30T09:24:25.375016Z","shell.execute_reply":"2021-05-30T09:24:25.379061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#numpy and matplotlib defaults\nnp.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    \n    if type(data) == tuple:\n        images, labels = data\n    else:\n        images = data\n        labels = None\n    \n    numpy_images = images.numpy()\n    \n    numpy_labels = [None for _ in enumerate(numpy_images)]\n    if labels is not None:\n        numpy_labels = labels.numpy()\n        if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n            numpy_labels = [None for _ in enumerate(numpy_images)]\n    \n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return '{} {}'.format(CLASSES[label], label) , True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(\n        '{} {}'.format(CLASSES[label], label),\n        'OK' if correct else 'NO',\n        u\"\\u2192\" if not correct else '',\n        '{} {}'.format(CLASSES[correct_label], correct_label) if not correct else ''\n    ), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n    \ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    \n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else '{} {}'.format(CLASSES[label], label)\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n    plt.close()\n\ndef get_title(label, prediction):\n\n    title = '' if label is None else '{} {}'.format(CLASSES[label], label)\n    correct = True\n    if prediction is not None:\n        title, correct = title_from_label_and_target(prediction, label)\n    return title, correct\n\ndef display_one_flower_ax(image, label, prediction, ax, red=False, titlesize=16):\n\n    title, correct = get_title(label, prediction)\n    red = not correct\n    \n    ax.axis('off')\n    ax.imshow(image)\n    if len(title) > 0:\n        ax.set_title(title, fontsize=int(titlesize) if not red else int(titlesize / 1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize / 1.5))        \n        \ndef display_pairs_of_image_batch(databatch, databatch_2=None, predictions=None, predictions_2=None, ds_name_1=None, ds_name_2=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n\n    nb_databatch = 1\n    if databatch_2:\n        nb_databatch = 2\n    \n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n\n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images) // rows        \n\n    gs0 = gridspec.GridSpec(1, nb_databatch)\n    gs00 = gridspec.GridSpecFromSubplotSpec(rows, cols, subplot_spec=gs0[0])\n    \n    if databatch_2:\n        images_2, labels_2 = batch_to_numpy_images_and_labels(databatch_2)\n        if labels_2 is None:\n            labels_2 = [None for _ in enumerate(images_2)]\n        gs01 = gridspec.GridSpecFromSubplotSpec(rows, cols, subplot_spec=gs0[1]) \n\n    # size and spacing\n    FIGSIZE = 24.0\n    SPACING = 0.10\n    subplot=(rows, cols, 1)\n\n    if rows < cols:\n        fig = plt.figure(figsize=(FIGSIZE, FIGSIZE / nb_databatch / cols * rows))\n    else:\n        fig = plt.figure(figsize=(FIGSIZE / rows * cols, FIGSIZE / nb_databatch))\n    \n    if ds_name_1 and ds_name_2:\n        fig.suptitle('2 batch of images. [Left: {}]   vs.   [Right: {}]'.format(ds_name_1, ds_name_2), y=-0.05, verticalalignment='bottom', fontsize=24)\n    elif ds_name_1:\n        fig.suptitle('1 batch of images from {}'.format(ds_name_1), y=-0.05, verticalalignment='bottom', fontsize=24)\n        \n    dynamic_titlesize = FIGSIZE * SPACING / max(rows, 2 * cols) * 40 + 3 # magic formula tested to work from 1x1 to 10x10 images          \n        \n    # display\n    for row, col in itertools.product(range(rows), range(cols)):\n        \n        idx = row * cols + col\n\n        image = images[idx]\n        label = labels[idx]\n        prediction = None if predictions is None else predictions[idx]\n        ax = fig.add_subplot(gs00[row, col])\n        display_one_flower_ax(image, label, prediction, ax, titlesize=dynamic_titlesize)\n        \n        if databatch_2:\n            image = images_2[idx]\n            label = labels_2[idx]\n            prediction = None if predictions_2 is None else predictions_2[idx]\n            ax = fig.add_subplot(gs01[row, col])\n            display_one_flower_ax(image, label, prediction, ax, titlesize=dynamic_titlesize)\n\n    #layout\n    plt.tight_layout()\n    \n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=SPACING / 2, hspace=SPACING / 2)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    \n    plt.show()\n    plt.close()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:26.693692Z","iopub.execute_input":"2021-05-30T09:24:26.694052Z","iopub.status.idle":"2021-05-30T09:24:26.731056Z","shell.execute_reply.started":"2021-05-30T09:24:26.694021Z","shell.execute_reply":"2021-05-30T09:24:26.729878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_dataset = tf.data.TFRecordDataset(TRAINING_FILENAMES)\nraw_dataset","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:28.449632Z","iopub.execute_input":"2021-05-30T09:24:28.449994Z","iopub.status.idle":"2021-05-30T09:24:28.465043Z","shell.execute_reply.started":"2021-05-30T09:24:28.449947Z","shell.execute_reply":"2021-05-30T09:24:28.46411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"serialized_example = next(iter(raw_dataset))\n\nprint('A serialized example looks like:\\n\\n' + str(serialized_example)[:100] + '...' * 5 + str(serialized_example)[-100:] + '\\n')\nprint(type(serialized_example))","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:29.812723Z","iopub.execute_input":"2021-05-30T09:24:29.81343Z","iopub.status.idle":"2021-05-30T09:24:29.853935Z","shell.execute_reply.started":"2021-05-30T09:24:29.813379Z","shell.execute_reply":"2021-05-30T09:24:29.853037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example = tf.train.Example()\nexample.ParseFromString(serialized_example.numpy())\nprint(str(example)[:300] + ' ...')","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:31.47824Z","iopub.execute_input":"2021-05-30T09:24:31.478565Z","iopub.status.idle":"2021-05-30T09:24:31.486299Z","shell.execute_reply.started":"2021-05-30T09:24:31.478539Z","shell.execute_reply":"2021-05-30T09:24:31.485083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image_data):\n    \"\"\"\n    Args:\n        image_data: A `tf.string` obtained from `tf.io.encode_jpeg()`.\n    \"\"\"\n    \n    # image is now of type `tf.uint8`\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    \n    # convert image to floats in [0, 1] range\n    image = tf.cast(image, tf.float32) / 255.0  \n    \n    # explicit size needed for TPU\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) \n    \n    return image\n\ndef read_labeled_tfrecord(example):\n    \n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    parsed_example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(parsed_example['image'])\n    label = tf.cast(parsed_example['class'], tf.int32)\n    \n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    \n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n    }\n    parsed_example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(parsed_example['image'])\n    idnum = parsed_example['id']\n    \n    return image, idnum # returns a dataset of (image, id) pairs","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:32.383893Z","iopub.execute_input":"2021-05-30T09:24:32.384284Z","iopub.status.idle":"2021-05-30T09:24:32.393441Z","shell.execute_reply.started":"2021-05-30T09:24:32.384251Z","shell.execute_reply":"2021-05-30T09:24:32.392408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parsed_example = read_labeled_tfrecord(serialized_example)\nprint('A parsed example looks like\\n\\n' + str(parsed_example)[:200] + '\\n...')","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:33.475472Z","iopub.execute_input":"2021-05-30T09:24:33.476039Z","iopub.status.idle":"2021-05-30T09:24:33.494067Z","shell.execute_reply.started":"2021-05-30T09:24:33.475993Z","shell.execute_reply":"2021-05-30T09:24:33.493062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    \"\"\"Read from TFRecords.\n    \n    For optimal performance, reading from multiple files at once and disregarding data order (if `ordered=False`).\n\n    Order does not matter since we will be shuffling the data anyway (for training dataset).\n    \"\"\"\n\n    options = tf.data.Options()\n    if not ordered:\n        # disable order, increase speed\n        options.experimental_deterministic = False\n\n    # Read in an automatically interleaving way from multiple tfrecord files.\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=tf.data.experimental.AUTOTUNE)\n    \n    # Uses data as soon as it streams in, rather than in its original order.\n    dataset = dataset.with_options(options) \n    \n    # parse and return a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    dataset = dataset.map(\n        read_labeled_tfrecord if labeled else read_unlabeled_tfrecord,\n        num_parallel_calls=tf.data.experimental.AUTOTUNE,\n    )\n    \n    return dataset\n\ndef get_training_dataset(batch_size, shuffle_buffer_size, repeat_dataset=False, ordered=False, drop_remainder=True):\n    \"\"\"\n    Set `shuffle_buffer_size` to `1` to have no shuffling.\n    \"\"\"\n    \n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=ordered)\n    \n    # Repeat the training dataset. We will determine the number of steps (or updates) later for 1 training epoch.\n    if repeat_dataset:\n        dataset = dataset.repeat()\n    \n    # Shuffling\n    if not ordered:\n        dataset = dataset.shuffle(shuffle_buffer_size)\n    \n    # Batching\n    dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)\n    \n    # prefetch next batch while training (autotune prefetch buffer size)\n    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n    \n    return dataset\n\ndef get_validation_dataset(batch_size):\n    \n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=True)\n    dataset = dataset.batch(batch_size)\n    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n    \n    return dataset\n\ndef get_test_dataset(batch_size):\n    \n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=True)\n    dataset = dataset.batch(batch_size)\n    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) \n    \n    return dataset\n\ndef count_data_items(filenames):\n    # For this flower dataset, the number of data items is written in the name of .tfrec files.\n    # For example, `flowers00-230.tfrec` means 230 data items in it.\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nORIGINAL_NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\n\nprint('Original Dataset:\\n\\n{} training images\\n{} validation images\\n{} unlabeled test images'.format(ORIGINAL_NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:35.112572Z","iopub.execute_input":"2021-05-30T09:24:35.112957Z","iopub.status.idle":"2021-05-30T09:24:35.12755Z","shell.execute_reply.started":"2021-05-30T09:24:35.112924Z","shell.execute_reply":"2021-05-30T09:24:35.126908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = get_training_dataset(batch_size=3, shuffle_buffer_size=1)\nds","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:41.419994Z","iopub.execute_input":"2021-05-30T09:24:41.420509Z","iopub.status.idle":"2021-05-30T09:24:41.587785Z","shell.execute_reply.started":"2021-05-30T09:24:41.420477Z","shell.execute_reply":"2021-05-30T09:24:41.5869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch = next(iter(ds))\nprint('The batch is a {} with {} components.'.format(type(batch).__name__, len(batch)))\nprint('\\nThe 1st compoent is a {} with shape {}'.format(type(batch[0]).__name__, batch[0].shape))\nprint('The 2nd compoent is a {} with shape {}\\n'.format(type(batch[1]).__name__, batch[1].shape))\n\nprint('The 2nd compoent looks like')\nbatch[1]","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:42.651225Z","iopub.execute_input":"2021-05-30T09:24:42.651575Z","iopub.status.idle":"2021-05-30T09:24:45.027054Z","shell.execute_reply.started":"2021-05-30T09:24:42.651547Z","shell.execute_reply":"2021-05-30T09:24:45.026213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = get_test_dataset(batch_size=3)\nds","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:45.028271Z","iopub.execute_input":"2021-05-30T09:24:45.028709Z","iopub.status.idle":"2021-05-30T09:24:45.128026Z","shell.execute_reply.started":"2021-05-30T09:24:45.028663Z","shell.execute_reply":"2021-05-30T09:24:45.126994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch = next(iter(ds))\nprint('The batch is a {} with {} components.'.format(type(batch).__name__, len(batch)))\nprint('\\nThe 1st compoent is a {} with shape {}'.format(type(batch[0]).__name__, batch[0].shape))\nprint('The 2nd compoent is a {} with shape {}'.format(type(batch[1]).__name__, batch[1].shape))\n\nprint('\\nThe 2nd compoent looks like')\nbatch[1]","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:51.347984Z","iopub.execute_input":"2021-05-30T09:24:51.348411Z","iopub.status.idle":"2021-05-30T09:24:52.898226Z","shell.execute_reply.started":"2021-05-30T09:24:51.348374Z","shell.execute_reply":"2021-05-30T09:24:52.897136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Peek the training data\ntrain_dataset = get_training_dataset(batch_size=16, shuffle_buffer_size=1, ordered=True, drop_remainder=False)\ntrain_iter = iter(train_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:55.049709Z","iopub.execute_input":"2021-05-30T09:24:55.050064Z","iopub.status.idle":"2021-05-30T09:24:55.077121Z","shell.execute_reply.started":"2021-05-30T09:24:55.050034Z","shell.execute_reply":"2021-05-30T09:24:55.076184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run this cell again for next set of images\nbatch = next(train_iter)\ndisplay_batch_of_images(batch)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:24:56.116584Z","iopub.execute_input":"2021-05-30T09:24:56.116926Z","iopub.status.idle":"2021-05-30T09:24:59.461363Z","shell.execute_reply.started":"2021-05-30T09:24:56.116896Z","shell.execute_reply":"2021-05-30T09:24:59.460574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# peek the validation data\nvalid_dataset = get_validation_dataset(batch_size=16)\nvalid_iter = iter(valid_dataset)# peek the validation data\nvalid_dataset = get_validation_dataset(batch_size=16)\nvalid_iter = iter(valid_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:25:05.02027Z","iopub.execute_input":"2021-05-30T09:25:05.020627Z","iopub.status.idle":"2021-05-30T09:25:05.073462Z","shell.execute_reply.started":"2021-05-30T09:25:05.020597Z","shell.execute_reply":"2021-05-30T09:25:05.072423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run this cell again for next set of images\ndisplay_batch_of_images(next(valid_iter))","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:25:08.787004Z","iopub.execute_input":"2021-05-30T09:25:08.787356Z","iopub.status.idle":"2021-05-30T09:25:12.204581Z","shell.execute_reply.started":"2021-05-30T09:25:08.787328Z","shell.execute_reply":"2021-05-30T09:25:12.20108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# peek the test data\ntest_dataset = get_test_dataset(batch_size=16)\ntest_iter = iter(test_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:25:14.866659Z","iopub.execute_input":"2021-05-30T09:25:14.867104Z","iopub.status.idle":"2021-05-30T09:25:15.144984Z","shell.execute_reply.started":"2021-05-30T09:25:14.867059Z","shell.execute_reply":"2021-05-30T09:25:15.144101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run this cell again for next set of images\ndisplay_batch_of_images(next(test_iter))","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:25:16.767251Z","iopub.execute_input":"2021-05-30T09:25:16.767591Z","iopub.status.idle":"2021-05-30T09:25:19.074171Z","shell.execute_reply.started":"2021-05-30T09:25:16.767564Z","shell.execute_reply":"2021-05-30T09:25:19.072759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Original Dataset:\\n\\ntraining images: {}\\nvalidation images: {}\\ntest images {}'.format(ORIGINAL_NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:25:21.549501Z","iopub.execute_input":"2021-05-30T09:25:21.549862Z","iopub.status.idle":"2021-05-30T09:25:21.5545Z","shell.execute_reply.started":"2021-05-30T09:25:21.549833Z","shell.execute_reply":"2021-05-30T09:25:21.553439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.autograph.experimental.do_not_convert\ndef get_label_counting(labeled_dataset):\n    \n    c = Counter()\n    labels = []\n    for batch in labeled_dataset.map(lambda image, label: label, num_parallel_calls=tf.data.experimental.AUTOTUNE):\n        labels.append(batch)\n    \n    labels = tf.concat(labels, axis=0).numpy()\n    c.update(labels)\n\n    return labels, c","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:25:22.451246Z","iopub.execute_input":"2021-05-30T09:25:22.451648Z","iopub.status.idle":"2021-05-30T09:25:22.457967Z","shell.execute_reply.started":"2021-05-30T09:25:22.451614Z","shell.execute_reply":"2021-05-30T09:25:22.456853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels, train_counter = get_label_counting(train_dataset)\nvalid_labels, valid_counter = get_label_counting(valid_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:25:23.376986Z","iopub.execute_input":"2021-05-30T09:25:23.377365Z","iopub.status.idle":"2021-05-30T09:25:33.768416Z","shell.execute_reply.started":"2021-05-30T09:25:23.377331Z","shell.execute_reply":"2021-05-30T09:25:33.767003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_label_dist(labels, dist_1, dist_2, dist_label_1, dist_label_2, title=''):\n    \n    x = np.arange(len(labels)) # the label locations\n    width = 0.4 # the width of the bars\n\n    fig, ax = plt.subplots(figsize=(15, 5))\n    rects1 = ax.bar(x - width / 2, dist_1, width, label=dist_label_1)\n    rects2 = ax.bar(x + width / 2, dist_2, width, label=dist_label_2)\n\n    # Add some text for labels, title and custom x-axis tick labels, etc.\n    ax.set_ylabel('portion in dataset')\n    ax.set_title(title)\n    ax.set_xticks(x)\n    ax.set_xticklabels([str(x) if x % 5 in [0] else '' for x in range(len(labels))] )\n    ax.legend()\n\n    plt.show()\n    plt.close()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:25:33.770162Z","iopub.execute_input":"2021-05-30T09:25:33.770743Z","iopub.status.idle":"2021-05-30T09:25:33.779618Z","shell.execute_reply.started":"2021-05-30T09:25:33.770699Z","shell.execute_reply":"2021-05-30T09:25:33.778666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = list(range(len(CLASSES)))\ndist_train = [train_counter[x] / ORIGINAL_NUM_TRAINING_IMAGES for x in labels]\ndist_valid = [valid_counter[x] / NUM_VALIDATION_IMAGES for x in labels]    \n    \nhalf = len(labels) // 2\nplot_label_dist(\n    labels[:half],\n    dist_train[:half],\n    dist_valid[:half],\n    'Train',\n    'Valid',\n    title='Label distribution in Train/Valid datasets: Labels 0-{}'.format(half - 1)\n)\n\nplot_label_dist(\n    labels[half:],\n    dist_train[half:],\n    dist_valid[half:],\n    'Train',\n    'Valid',    \n    title='Label distribution in Train/Valid datasets: Labels {}-{}'.format(half, len(labels) - 1)\n)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:25:33.781972Z","iopub.execute_input":"2021-05-30T09:25:33.782406Z","iopub.status.idle":"2021-05-30T09:25:34.898533Z","shell.execute_reply.started":"2021-05-30T09:25:33.782365Z","shell.execute_reply":"2021-05-30T09:25:34.897435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"labels in the original training dataset, sorted by occurrence\\n\")\nprint(\"pairs of (class id, counting)\\n\")\nprint(train_counter.most_common())","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:25:34.90065Z","iopub.execute_input":"2021-05-30T09:25:34.901105Z","iopub.status.idle":"2021-05-30T09:25:34.907032Z","shell.execute_reply.started":"2021-05-30T09:25:34.90106Z","shell.execute_reply":"2021-05-30T09:25:34.905931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_num_to_repeat_for_class(class_id, target_counting):\n    \"\"\"Compute the (ideal) number of times a training example with\n       label `class_id` should repeat in order to get a dataset where\n       each class occur `target_counting` times.\n       \n    The return value is a float number. The actual number to repeat will\n    be determined in the function `get_nums_to_repeat` in a randomized way,\n    in order to make a better approximation.\n       \n    Args:\n    \n        class_id: int, the id of a class.\n        target_counting: int, the targeted occurrence number.\n        \n    Returns:\n        A float, the number of times an example with label `class_id` to repeat.\n    \"\"\"\n    \n    # Use the counter computed in `Label distribution` subsection`.\n    counting = train_counter[class_id]\n    \n    # No need to repeat for a class having already the desired occurrecne.\n    if counting >= target_counting:\n        return 1.0\n    \n    num_to_repeat = target_counting / counting\n    \n    return num_to_repeat\n\ndef get_nums_to_repeat(target_counting):\n    \"\"\"Compute a tabel that stores the results of `get_num_to_repeat_for_class`\n       for every class and for the given `target_counting`.\n    \n    Args:\n        target_counting: int, the targeted occurrence number.\n    \n    Returns:\n        table: A `tf.lookup.StaticHashTable`.\n        d: A dictionary storing the same information as `table`.\n    \"\"\"\n    \n    keys = range(len(CLASSES))\n    values = [get_num_to_repeat_for_class(x, target_counting) for x in keys]\n\n    keys_tensor = tf.constant(keys)\n    vals_tensor = tf.constant(values)\n    \n    table_initializers = tf.lookup.KeyValueTensorInitializer(keys_tensor, vals_tensor)\n    table = tf.lookup.StaticHashTable(table_initializers, default_value=0.0)\n    \n    d = {k: v for k, v in zip(keys, values)}\n\n    return table, d\n\ndef get_num_to_repeat_for_example(example, table):\n    \"\"\"Compute the actual number of times a training example will repeat\n       in order to get a dataset where each class occur <approximately> \n       N times with N being a pre-defined number that is used for constructing\n       `table`.\n\n    Args:\n        example: A tuple of 2 tensors, which is a labeled training example and\n            represented as (image, label).\n                          \n        tabel: A tf.lookup.StaticHashTable, as obtained from `get_nums_to_repeat`.\n                          \n    Returns:\n        A tf.int64 scalar tensor, the number of times `example` will repeat.\n    \"\"\"\n    \n    image, label = example\n\n    num_to_repeat = table.lookup(label)    \n    \n    # This part is deterministic.\n    num_to_repeat_integral = tf.cast(int(num_to_repeat), tf.float32)\n    \n    # With a probability `residue`, we allow `example` to repeat one more time.\n    residue = num_to_repeat - num_to_repeat_integral\n    num_to_repeat = num_to_repeat_integral + tf.cast(tf.random.uniform(shape=()) <= residue, tf.float32)\n    \n    return tf.cast(num_to_repeat, tf.int64)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:31:12.495044Z","iopub.execute_input":"2021-05-30T09:31:12.495408Z","iopub.status.idle":"2021-05-30T09:31:12.505977Z","shell.execute_reply.started":"2021-05-30T09:31:12.495379Z","shell.execute_reply":"2021-05-30T09:31:12.505167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, d = get_nums_to_repeat(782)\nd = sorted(d.items(), key=lambda x: x[1], reverse=True)\n\nprint('pair of (class id, num to repeat)\\n')\nfor x in d:\n    print(x)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:31:18.689949Z","iopub.execute_input":"2021-05-30T09:31:18.690461Z","iopub.status.idle":"2021-05-30T09:31:18.715322Z","shell.execute_reply.started":"2021-05-30T09:31:18.69043Z","shell.execute_reply":"2021-05-30T09:31:18.714114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_oversampled_training_dataset(\n        target_counting, batch_size, shuffle_buffer_size,\n        repeat_dataset=False, ordered=False,\n        oversample=True, augmentation_fn=None, probability=1.0\n    ):\n    \"\"\"\n    Construct an oversampled dataset in which each class occurs approximately\n    `target_counting` times.\n    \n    (Special) Args:\n    \n        target_counting: int, the target occurrence.\n        oversampe: bool, if to use oversampling. If `False`, no oversampliing and\n            the arguement `target_counting` has no effect.\n        augmentation_fn: A funtion used to map the dataset for data augmentation.\n        probability: float, the probability to perform the augmentation\n        \n    Returns:\n        A tf.data.Dataset.\n    \"\"\"\n    \n    table, d = get_nums_to_repeat(target_counting)\n    \n    nb_examples = ORIGINAL_NUM_TRAINING_IMAGES\n    \n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=ordered)\n\n    if oversample:\n        \n        # This is only approximation, but good enough.\n        nb_examples = int(sum([train_counter[k] *  v for k, v in d.items()]))\n        \n        dataset = dataset.flat_map(\n            lambda image, label: tf.data.Dataset.from_tensors((image, label)).repeat(get_num_to_repeat_for_example((image, label), table))\n        )\n        \n    if repeat_dataset:\n        dataset = dataset.repeat()\n \n    if not ordered:\n        if not shuffle_buffer_size:\n            shuffle_buffer_size = nb_examples\n        dataset = dataset.shuffle(shuffle_buffer_size)\n    \n    dataset = dataset.batch(batch_size, drop_remainder=True)\n    \n    if augmentation_fn:\n        probability = tf.constant(probability, dtype=tf.float32)\n        dataset = dataset.map(\n            lambda images, labels: augmentation_fn(images, labels, probability=probability),\n            num_parallel_calls=tf.data.experimental.AUTOTUNE\n        )\n        \n    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n    \n    return dataset, nb_examples","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:31:20.509376Z","iopub.execute_input":"2021-05-30T09:31:20.509762Z","iopub.status.idle":"2021-05-30T09:31:20.520184Z","shell.execute_reply.started":"2021-05-30T09:31:20.509709Z","shell.execute_reply":"2021-05-30T09:31:20.519142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oversampled_train_dataset, _ = get_oversampled_training_dataset(target_counting=782, batch_size=16, shuffle_buffer_size=1, repeat_dataset=False, ordered=True, oversample=True, augmentation_fn=None)\n\n_, oversampled_train_counter = get_label_counting(oversampled_train_dataset)\n\nprint('Oversampled training dataset:\\ntraining images: {}\\n'.format(sum(oversampled_train_counter.values())))\n\nprint(\"labels in the oversampled training dataset, sorted by occurrence: pairs of (label_id, label_counting)\\n\")\nprint(oversampled_train_counter.most_common())\n\nprint('\\n' + 'averaged number of occurrences: ', np.array(list(oversampled_train_counter.values())).mean())","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:31:21.152018Z","iopub.execute_input":"2021-05-30T09:31:21.15239Z","iopub.status.idle":"2021-05-30T09:32:46.077917Z","shell.execute_reply.started":"2021-05-30T09:31:21.152357Z","shell.execute_reply":"2021-05-30T09:32:46.076787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dist_train_oversampled = np.array([oversampled_train_counter[x] for x in labels]) / sum(oversampled_train_counter.values())\n\nhalf = len(labels) // 2\nplot_label_dist(\n    labels[:half],\n    dist_train[:half],\n    dist_train_oversampled[:half],\n    'original',\n    'oversampled',\n    title='Label distribution in train datasets with/without oversampling: Labels 0-{}'.format(half - 1)\n)\n\nplot_label_dist(\n    labels[half:],\n    dist_train[half:],\n    dist_train_oversampled[half:],\n    'original',\n    'oversampled',    \n    title='Label distribution in train datasets with/without oversampling: Labels {}-{}'.format(half, len(labels) - 1)\n)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:32:46.0797Z","iopub.execute_input":"2021-05-30T09:32:46.080121Z","iopub.status.idle":"2021-05-30T09:32:47.250757Z","shell.execute_reply.started":"2021-05-30T09:32:46.08008Z","shell.execute_reply":"2021-05-30T09:32:47.249703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Peek the oversampled training data\ntrain_iter = iter(train_dataset)\noversampled_train_iter = iter(oversampled_train_dataset)\n\ndisplay_pairs_of_image_batch(next(train_iter), next(oversampled_train_iter), ds_name_1='original dataset', ds_name_2='oversampled dataset')\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:32:47.253178Z","iopub.execute_input":"2021-05-30T09:32:47.253588Z","iopub.status.idle":"2021-05-30T09:32:51.955467Z","shell.execute_reply.started":"2021-05-30T09:32:47.253544Z","shell.execute_reply":"2021-05-30T09:32:51.954168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Flower_Trainer:\n    \n    def __init__(self, batch_size_per_replica=16, prediction_batch_size_per_replica=64, shuffle_buffer_size=1, oversample=False, target_counting=1, grad_acc_steps=1, augmentation_fn=None, probability=1.0, log_interval=1):\n    \n        self.batch_size_per_replica = batch_size_per_replica\n        self.prediction_batch_size_per_replica = prediction_batch_size_per_replica\n        \n        self.batch_size = batch_size_per_replica * strategy.num_replicas_in_sync\n        self.prediction_batch_size = prediction_batch_size_per_replica * strategy.num_replicas_in_sync\n\n        self.grad_acc_steps = grad_acc_steps\n        self.update_size = self.batch_size * self.grad_acc_steps\n        \n        self.shuffle_buffer_size = shuffle_buffer_size\n        self.oversample = oversample\n        self.target_counting = target_counting\n        \n        self.augmentation_fn = augmentation_fn\n        \n        self.train_ds, self.nb_examples_approx = get_oversampled_training_dataset(\n            self.target_counting, self.update_size, self.shuffle_buffer_size,\n            repeat_dataset=True, ordered=False,\n            oversample=self.oversample, augmentation_fn=self.augmentation_fn,\n            probability=probability\n        )\n\n        self.updates_per_epoch = self.nb_examples_approx // self.update_size        \n        \n        self.valid_ds = get_validation_dataset(self.prediction_batch_size)\n        self.test_ds = get_test_dataset(self.prediction_batch_size)\n        \n        self.log_interval = log_interval\n         \n    def train(self, train_name, model_name, epochs, start_lr, max_lr, end_lr, warmup, lr_scaling, optimized_loop=False, verbose=False):\n        \n        update_steps = epochs * self.updates_per_epoch\n        warmup_steps = int(update_steps * warmup)\n        \n        model, loss_fn, optimizer, gradient_accumulator, metrics = get_model(model_name, update_steps, warmup_steps, start_lr, max_lr, end_lr, lr_scaling, verbose=verbose)\n        \n        dist_train_1_epoch_optimized, dist_train_1_epoch_normal, dist_predict_step  = get_routines(\n            model, loss_fn, optimizer, gradient_accumulator, metrics, self.batch_size_per_replica, self.update_size, self.grad_acc_steps, self.updates_per_epoch\n        )\n\n        dist_train_1_epoch = dist_train_1_epoch_normal\n        if optimized_loop:\n             dist_train_1_epoch = dist_train_1_epoch_optimized\n        \n        train_fn = get_train_fn(dist_train_1_epoch, dist_predict_step, loss_fn, metrics, log_interval=self.log_interval)\n        history, valid_labels, valid_preds = train_fn(train_name, epochs, self.train_ds, self.valid_ds, self.test_ds, self.updates_per_epoch)\n        \n        return history, valid_labels, valid_preds","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:32:51.957027Z","iopub.execute_input":"2021-05-30T09:32:51.957493Z","iopub.status.idle":"2021-05-30T09:32:51.971219Z","shell.execute_reply.started":"2021-05-30T09:32:51.95744Z","shell.execute_reply":"2021-05-30T09:32:51.97015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class WarmupLearningRateSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n\n    def __init__(self, backend_schedule, start_lr, max_lr, end_lr, opt_steps, warmup_steps, lr_scaling):\n\n        self.start_lr = start_lr\n        self.max_lr = max_lr\n        self.end_lr = end_lr\n        self.opt_steps = tf.cast(opt_steps, tf.float32)\n        self.warmup_steps = tf.cast(warmup_steps, tf.float32)\n        self.lr_scaling = tf.cast(lr_scaling, tf.float32)\n        self.backend_lr = backend_schedule\n\n        self.warmup_incremental = (self.max_lr - self.start_lr) / tf.math.reduce_max([self.warmup_steps, 1.0]) * tf.cast(self.warmup_steps > 0.0, tf.float32)\n\n    def __call__(self, step):\n\n        is_warmup = tf.cast(step < self.warmup_steps, tf.float32)\n        warmup_lr = self.warmup_incremental * step + self.start_lr\n        decay_lr = self.backend_lr(step - self.warmup_steps)\n        lr = (1.0 - is_warmup) * decay_lr + is_warmup * warmup_lr\n\n        return lr * self.lr_scaling\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:32:51.972579Z","iopub.execute_input":"2021-05-30T09:32:51.97295Z","iopub.status.idle":"2021-05-30T09:32:51.989635Z","shell.execute_reply.started":"2021-05-30T09:32:51.972919Z","shell.execute_reply":"2021-05-30T09:32:51.988403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_lr_schedule(lr_schedule, n_steps):\n\n    steps = [i for i in range(n_steps)]\n    lrs = [lr_schedule(x) for x in steps]\n    plt.plot(steps, lrs)\n    print(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(lrs[0], max(lrs), lrs[-1])) \n    plt.show()\n    \ndef plot_lr_schedule_pair(lr_schedule_1, lr_schedule_2, n_steps):\n    \n    steps = [i for i in range(n_steps)]\n    \n    plt.figure(figsize=(11.0, 6.0 / 2))\n    \n    lrs = [lr_schedule_1(x) for x in steps]\n    plt.subplot(1, 2, 1)\n    plt.plot(steps, lrs)\n    plt.title('original lr', fontsize=14, color='black', fontdict={'verticalalignment':'center'}, pad=12.0)\n          \n    lrs = [lr_schedule_2(x) for x in steps]\n    plt.subplot(1, 2, 2)\n    plt.plot(steps, lrs)\n    plt.title('warmup lr', fontsize=14, color='black', fontdict={'verticalalignment':'center'}, pad=12.0)\n    \n    plt.show()\n    \n    \nopt_steps, start_lr, max_lr, end_lr, lr_scaling = 1000, 1e-7, 1e-5, 1e-6, 1\n\nwarmup_steps = 0\nbackend_lr = tf.keras.optimizers.schedules.ExponentialDecay(\n    max_lr, opt_steps - warmup_steps, decay_rate=(end_lr / max_lr),\n)\nlr_rate1 = WarmupLearningRateSchedule(backend_lr, start_lr, max_lr, end_lr, opt_steps, warmup_steps, lr_scaling)\n\nwarmup_steps = 0\nbackend_lr = tf.keras.optimizers.schedules.PolynomialDecay(\n    initial_learning_rate=max_lr, decay_steps=(opt_steps - warmup_steps), end_learning_rate=end_lr, power=1.0\n)\nlr_rate3 = WarmupLearningRateSchedule(backend_lr, start_lr, max_lr, end_lr, opt_steps, warmup_steps, lr_scaling)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:10:51.619093Z","iopub.execute_input":"2021-05-30T06:10:51.619523Z","iopub.status.idle":"2021-05-30T06:10:51.644231Z","shell.execute_reply.started":"2021-05-30T06:10:51.619446Z","shell.execute_reply":"2021-05-30T06:10:51.643048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warmup_steps = 200\nbackend_lr = tf.keras.optimizers.schedules.ExponentialDecay(\n    max_lr, opt_steps - warmup_steps, decay_rate=(end_lr / max_lr),\n)\nlr_rate2 = WarmupLearningRateSchedule(backend_lr, start_lr, max_lr, end_lr, opt_steps, warmup_steps, lr_scaling)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:10:51.645689Z","iopub.execute_input":"2021-05-30T06:10:51.645969Z","iopub.status.idle":"2021-05-30T06:10:51.660499Z","shell.execute_reply.started":"2021-05-30T06:10:51.645941Z","shell.execute_reply":"2021-05-30T06:10:51.659403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_lr_schedule_pair(lr_rate1, lr_rate2, opt_steps)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:10:51.661878Z","iopub.execute_input":"2021-05-30T06:10:51.662157Z","iopub.status.idle":"2021-05-30T06:11:01.060787Z","shell.execute_reply.started":"2021-05-30T06:10:51.662129Z","shell.execute_reply":"2021-05-30T06:11:01.059268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warmup_steps = 200\nbackend_lr = tf.keras.optimizers.schedules.PolynomialDecay(\n    initial_learning_rate=max_lr, decay_steps=(opt_steps - warmup_steps), end_learning_rate=end_lr, power=1.0\n)\nlr_rate4 = WarmupLearningRateSchedule(backend_lr, start_lr, max_lr, end_lr, opt_steps, warmup_steps, lr_scaling)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:01.06455Z","iopub.execute_input":"2021-05-30T06:11:01.065166Z","iopub.status.idle":"2021-05-30T06:11:01.073919Z","shell.execute_reply.started":"2021-05-30T06:11:01.065111Z","shell.execute_reply":"2021-05-30T06:11:01.072826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_lr_schedule_pair(lr_rate3, lr_rate4, opt_steps)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:01.07513Z","iopub.execute_input":"2021-05-30T06:11:01.075467Z","iopub.status.idle":"2021-05-30T06:11:11.56596Z","shell.execute_reply.started":"2021-05-30T06:11:01.075419Z","shell.execute_reply":"2021-05-30T06:11:11.564786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(model_name, update_steps, warmup_steps, start_lr, max_lr, end_lr, lr_scaling, verbose=False):\n\n    with strategy.scope():\n\n        model_class = MODEL_CLASSES[model_name]\n        pretrained_model = model_class(weights='imagenet', include_top=False, input_shape=[*IMAGE_SIZE, 3])\n        \n        # False = transfer learning, True = fine-tuning\n        pretrained_model.trainable = True \n\n        model = tf.keras.Sequential([\n            pretrained_model,\n            tf.keras.layers.Dropout(rate=0.05),\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(len(CLASSES))\n        ])\n        \n        if verbose:\n            model.summary()\n\n        backend_lr = tf.keras.optimizers.schedules.ExponentialDecay(\n            max_lr, opt_steps - warmup_steps, decay_rate=(end_lr / max_lr),\n        )\n        lr_rate = WarmupLearningRateSchedule(backend_lr, start_lr, max_lr, end_lr, update_steps, warmup_steps, lr_scaling)\n        \n        # Instiate an optimizer with a learning rate schedule\n        optimizer = tf.keras.optimizers.Adam(lr_rate)\n\n        # Only `NONE` and `SUM` are allowed, and it has to be explicitly specified.\n        loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.SUM)\n\n        # Instantiate metrics\n        metrics = {\n            'train loss': tf.keras.metrics.Sum(),\n            'train acc': tf.keras.metrics.SparseCategoricalAccuracy(),\n            'valid acc': tf.keras.metrics.SparseCategoricalAccuracy()\n        }\n        \n        gradient_accumulator = None\n\n        return model, loss_fn, optimizer, gradient_accumulator, metrics","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:11.567394Z","iopub.execute_input":"2021-05-30T06:11:11.567735Z","iopub.status.idle":"2021-05-30T06:11:11.57838Z","shell.execute_reply.started":"2021-05-30T06:11:11.567702Z","shell.execute_reply":"2021-05-30T06:11:11.577298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = get_training_dataset(batch_size=9, shuffle_buffer_size=1)\ndist_ds = strategy.experimental_distribute_dataset(ds)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:11.580871Z","iopub.execute_input":"2021-05-30T06:11:11.58151Z","iopub.status.idle":"2021-05-30T06:11:11.636191Z","shell.execute_reply.started":"2021-05-30T06:11:11.581442Z","shell.execute_reply":"2021-05-30T06:11:11.635432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dist_batch = next(iter(dist_ds))\nprint('The distributed batch is a {} with {} components.'.format(type(dist_batch).__name__, len(dist_batch)))\nprint('\\nThe 1st compoent is a {}'.format(type(dist_batch[0]).__name__))\nprint('The 2nd compoent is a {}'.format(type(dist_batch[1]).__name__))","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:11.637841Z","iopub.execute_input":"2021-05-30T06:11:11.638425Z","iopub.status.idle":"2021-05-30T06:11:12.256467Z","shell.execute_reply.started":"2021-05-30T06:11:11.638378Z","shell.execute_reply":"2021-05-30T06:11:12.255439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dist_batch[1]","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:12.257725Z","iopub.execute_input":"2021-05-30T06:11:12.257998Z","iopub.status.idle":"2021-05-30T06:11:12.350887Z","shell.execute_reply.started":"2021-05-30T06:11:12.25797Z","shell.execute_reply":"2021-05-30T06:11:12.349854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('The values contained inside dist_batch[0] (which is a `{}` object) are packed in a {} with {} components.\\n'.format(type(dist_batch[0]).__name__, type(dist_batch[0].values).__name__, len(dist_batch[0].values)))\nprint('The 1st component in `dist_batch[0].values` is a {} with shape {}'.format(type(dist_batch[0].values[0]).__name__, dist_batch[0].values[0].shape))\nprint('The 4th component in `dist_batch[0].values` is a {} with shape {}'.format(type(dist_batch[0].values[4]).__name__, dist_batch[0].values[4].shape))\nprint('The last component in `dist_batch[0].values` is a {} with shape {}'.format(type(dist_batch[0].values[-1]).__name__, dist_batch[0].values[-1].shape))","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:12.352073Z","iopub.execute_input":"2021-05-30T06:11:12.352354Z","iopub.status.idle":"2021-05-30T06:11:12.360905Z","shell.execute_reply.started":"2021-05-30T06:11:12.352327Z","shell.execute_reply":"2021-05-30T06:11:12.359503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('The values contained inside dist_batch[1] (which is a `{}` object) are packed in a {} with {} components.\\n'.format(type(dist_batch[1]).__name__, type(dist_batch[1].values).__name__, len(dist_batch[1].values)))\nprint('The first component in `dist_batch[1].values` is a {} with shape {}'.format(type(dist_batch[1].values[0]).__name__, dist_batch[1].values[0].shape))\nprint('The 4th component in `dist_batch[1].values` is a {} with shape {}'.format(type(dist_batch[1].values[4]).__name__, dist_batch[1].values[4].shape))\nprint('The last component in `dist_batch[1].values` is a {} with shape {}'.format(type(dist_batch[1].values[-1]).__name__, dist_batch[1].values[-1].shape))\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:12.362604Z","iopub.execute_input":"2021-05-30T06:11:12.363101Z","iopub.status.idle":"2021-05-30T06:11:12.37577Z","shell.execute_reply.started":"2021-05-30T06:11:12.363056Z","shell.execute_reply":"2021-05-30T06:11:12.374593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef dummy_run(images, labels):\n    \n    images = images + 1\n    labels = labels * 0\n    \n    return images, labels\n    \ndummy_images, dummy_labels = strategy.run(dummy_run, args=dist_batch)\ndummy_labels","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:12.377038Z","iopub.execute_input":"2021-05-30T06:11:12.377324Z","iopub.status.idle":"2021-05-30T06:11:12.971312Z","shell.execute_reply.started":"2021-05-30T06:11:12.377296Z","shell.execute_reply":"2021-05-30T06:11:12.970323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy.experimental_local_results(dummy_labels)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:12.974699Z","iopub.execute_input":"2021-05-30T06:11:12.975028Z","iopub.status.idle":"2021-05-30T06:11:12.993351Z","shell.execute_reply.started":"2021-05-30T06:11:12.974986Z","shell.execute_reply":"2021-05-30T06:11:12.992021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.concat(strategy.experimental_local_results(dummy_labels), axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:12.994866Z","iopub.execute_input":"2021-05-30T06:11:12.996465Z","iopub.status.idle":"2021-05-30T06:11:13.00957Z","shell.execute_reply.started":"2021-05-30T06:11:12.996426Z","shell.execute_reply":"2021-05-30T06:11:13.008563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_routines(model, loss_fn, optimizer, gradient_accumulator, metrics, batch_size_per_replica, update_size, grad_acc_steps, updates_per_epoch):\n\n    def train_1_forward_backward(images, labels):\n\n        with tf.GradientTape() as tape:\n\n            logits = model(images, training=True)\n            # Remember that we use the `SUM` reduction when we define the loss object.\n            loss = loss_fn(labels, logits) / update_size\n\n        grads = tape.gradient(loss, model.trainable_variables)\n        \n        # shape = [batch_size_per_replica]\n        preds = tf.cast(tf.math.argmax(logits, axis=-1), dtype=tf.int32)\n\n        # update metrics\n        metrics['train loss'].update_state(loss)\n        metrics['train acc'].update_state(labels, logits)\n\n        return grads, preds\n\n    def train_1_update(images, labels):\n        \"\"\"\n        gradient accumulation.\n        \"\"\"\n        \n        accumulated_grads = [tf.zeros_like(var, dtype=tf.float32) for var in model.trainable_variables]\n        \n        # Used for collecting the predictions.\n        preds = tf.zeros_like(labels)\n        \n        for idx in tf.range(grad_acc_steps):\n\n            # Take the 1st `batch_size_per_replica` examples.\n            _images = images[:batch_size_per_replica]\n            _labels = labels[:batch_size_per_replica]\n\n            # Get the gradients\n            grads, _preds = train_1_forward_backward(_images, _labels)\n            preds = tf.concat([preds[batch_size_per_replica:], _preds], axis=0)\n\n            # accumulated the gradients\n            accumulated_grads = [x + y for x, y in zip(accumulated_grads, grads)]\n\n            # Move the leading part to the end, so the shape is not changed.\n            images = tf.concat([images[batch_size_per_replica:], _images], axis=0)\n            labels = tf.concat([labels[batch_size_per_replica:], _labels], axis=0)\n            \n        # Update the model's parameters.\n        optimizer.apply_gradients(zip(accumulated_grads, model.trainable_variables))\n\n        return labels, preds\n        \n    @tf.function\n    def dist_train_step(dist_batch):\n        \n        labels, preds = strategy.run(train_1_update, args=dist_batch)        \n        \n        return labels, preds\n        \n    def dist_train_1_epoch(data_iter):\n        \"\"\"\n        Iterating outside `tf.function`.\n        \"\"\"\n        \n        labels = tf.zeros(shape=[updates_per_epoch * update_size], dtype=tf.int32)\n        preds = tf.zeros(shape=[updates_per_epoch * update_size], dtype=tf.int32)\n        \n        for _ in range(updates_per_epoch):\n            \n            _labels, _preds = dist_train_step(next(data_iter))\n            \n            # these are tuples of tensors\n            _labels = strategy.experimental_local_results(_labels)\n            _preds = strategy.experimental_local_results(_preds)\n            \n            # convert each to a single tensor\n            _labels = tf.concat(_labels, axis=0)\n            _preds = tf.concat(_preds, axis=0)\n            \n            # collect the results\n            labels = tf.concat([labels[update_size:], _labels], axis=0)\n            preds = tf.concat([preds[update_size:], _preds], axis=0)\n            \n        return labels, preds\n        \n    @tf.function\n    def dist_train_1_epoch_optimized(data_iter):\n        \"\"\"\n        Iterating inside `tf.function` to optimized training time.\n        \"\"\"\n\n        labels = tf.zeros(shape=[updates_per_epoch * update_size], dtype=tf.int32)\n        preds = tf.zeros(shape=[updates_per_epoch * update_size], dtype=tf.int32)        \n        \n        for _ in tf.range(updates_per_epoch):\n            \n            _labels, _preds = dist_train_step(next(data_iter))\n            \n            # tuple of tensors\n            _labels = strategy.experimental_local_results(_labels)\n            _preds = strategy.experimental_local_results(_preds)\n            \n            # to a single tensor\n            _labels = tf.concat(_labels, axis=0)\n            _preds = tf.concat(_preds, axis=0)           \n            \n            # collect\n            labels = tf.concat([labels[update_size:], _labels], axis=0)\n            preds = tf.concat([preds[update_size:], _preds], axis=0)\n            \n        return labels, preds\n            \n    def predict_step(images):\n\n        logits = model(images, training=False)\n        return logits\n\n    @tf.function\n    def dist_predict_step(images):\n\n        logits = strategy.run(predict_step, [images])\n        return logits\n\n    return dist_train_1_epoch_optimized, dist_train_1_epoch, dist_predict_step\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:13.01105Z","iopub.execute_input":"2021-05-30T06:11:13.011362Z","iopub.status.idle":"2021-05-30T06:11:13.034854Z","shell.execute_reply.started":"2021-05-30T06:11:13.011332Z","shell.execute_reply":"2021-05-30T06:11:13.033696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_results(train_name, history, valid_labels, valid_preds, test_idx, test_preds):\n\n    with open(f'history-{train_name}.json', 'w', encoding='UTF-8') as fp:\n        json.dump(history, fp, indent=4, ensure_ascii=False)\n\n    with open(f'valid-labels-{train_name}.json', 'w', encoding='UTF-8') as fp:\n        json.dump(valid_labels, fp, indent=4, ensure_ascii=False)\n\n    with open(f'valid-preds-{train_name}.json', 'w', encoding='UTF-8') as fp:\n        json.dump(valid_preds, fp, ensure_ascii=False)\n\n    with open(f'test-preds-{train_name}.json', 'w', encoding='UTF-8') as fp:\n        json.dump(test_preds, fp, indent=4, ensure_ascii=False)\n        \n    submission = pd.DataFrame(test_idx, columns=['id'])\n    submission['label'] = test_preds\n    submission.to_csv(f'submission.csv', index=False)\n    \ndef print_metrics(history, epochs, log_interval):\n    \n    epoch = len(history) - 1\n    \n    if epoch in [0, epochs-1] or (epoch + 1) % log_interval == 0:\n\n        print('epoch: {}'.format(epoch + 1))\n        print('elapsed: {}\\n'.format(history[epoch]['train timing']))\n\n        print('train loss: {}'.format(history[epoch]['train loss']))\n        print('train acc: {}'.format(history[epoch]['train acc']))                \n        print('train recall: {}'.format(history[epoch]['train recall']))\n        print('train precision: {}'.format(history[epoch]['train precision']))\n        print('train f1: {}\\n'.format(history[epoch]['train f1']))           \n\n        print('valid loss: {}'.format(history[epoch]['valid loss']))\n        print('valid acc: {}'.format(history[epoch]['valid acc']))        \n        print('valid recall: {}'.format(history[epoch]['valid recall']))\n        print('valid precision: {}'.format(history[epoch]['valid precision']))\n        print('valid f1: {}'.format(history[epoch]['valid f1']))\n        \n        print('-' * 40)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:13.036709Z","iopub.execute_input":"2021-05-30T06:11:13.037156Z","iopub.status.idle":"2021-05-30T06:11:13.053571Z","shell.execute_reply.started":"2021-05-30T06:11:13.037096Z","shell.execute_reply":"2021-05-30T06:11:13.052659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_fn(dist_train_1_epoch, dist_predict_step, loss_fn, metrics, log_interval=1):\n\n    def predict_fn(dist_image_ds):\n\n        all_logits = []\n        for images in dist_image_ds:\n\n            # PerReplica object\n            logits = dist_predict_step(images)\n\n            # Tuple of tensors\n            logits = strategy.experimental_local_results(logits)\n\n            # tf.Tensor\n            logits = tf.concat(logits, axis=0)\n\n            all_logits.append(logits)\n\n        # tf.Tensor\n        logits = tf.concat(all_logits, axis=0)\n        preds = tf.math.argmax(logits, axis=-1)\n\n        return logits, preds\n\n    def valid_fn(dist_image_ds, labels, epoch):\n\n        logits, preds = predict_fn(dist_image_ds)\n\n        loss = loss_fn(labels, logits) / NUM_VALIDATION_IMAGES\n\n        # update metrics\n        metrics['valid acc'].update_state(labels, logits)\n\n        # get metrics\n        acc = metrics['valid acc'].result()\n\n        recall = sklearn.metrics.recall_score(labels, preds, average='macro')\n        precision = sklearn.metrics.precision_score(labels, preds, average='macro')\n        f1 = sklearn.metrics.f1_score(labels, preds, average='macro')\n        \n        # reset metrics\n        metrics['valid acc'].reset_states()\n        \n        return {'loss': float(loss), 'acc': float(acc), 'recall': recall, 'precision': precision, 'f1': f1, 'preds': preds}\n    \n    def train_fn(train_name, epochs, train_ds, valid_ds, test_ds, updates_per_epoch):    \n        \n        valid_image_ds = valid_ds.map(lambda image, label: image)   \n        test_image_ds = test_ds.map(lambda image, idx: image)\n    \n        train_dist_ds = strategy.experimental_distribute_dataset(train_ds)        \n        valid_dist_image_ds = strategy.experimental_distribute_dataset(valid_image_ds)\n        test_dist_image_ds = strategy.experimental_distribute_dataset(test_image_ds)\n        \n        train_data_iter = iter(train_dist_ds)\n        \n        valid_label_ds = valid_ds.map(lambda image, label: label)\n        valid_labels = next(iter(valid_label_ds.unbatch().batch(NUM_VALIDATION_IMAGES)))        \n        \n        test_idx_ds = test_ds.map(lambda image, idx: idx)\n        test_idx = next(iter(test_idx_ds.unbatch().batch(NUM_TEST_IMAGES))).numpy().astype('U')\n      \n        history = {}\n        valid_preds = {}\n    \n        for epoch in range(epochs):\n            \n            s = datetime.datetime.now()\n\n            labels, preds = dist_train_1_epoch(train_data_iter)\n\n            # get metrics\n            train_loss = metrics['train loss'].result() / updates_per_epoch\n            train_acc = metrics['train acc'].result()\n\n            # reset metrics\n            metrics['train loss'].reset_states()\n            metrics['train acc'].reset_states()\n            \n            recall = sklearn.metrics.recall_score(labels, preds, average='macro')\n            precision = sklearn.metrics.precision_score(labels, preds, average='macro')\n            f1 = sklearn.metrics.f1_score(labels, preds, average='macro')\n            \n            e = datetime.datetime.now()\n            elapsed = (e - s).total_seconds()\n            \n            valid_results = valid_fn(valid_dist_image_ds, valid_labels, epoch)\n            \n            history[epoch] = {\n                'train loss': float(train_loss),\n                'train acc': float(train_acc),\n                'train recall': recall,\n                'train precision': precision,\n                'train f1': f1,                \n                'valid loss': valid_results['loss'],\n                'valid acc': valid_results['acc'],\n                'valid recall': valid_results['recall'],\n                'valid precision': valid_results['precision'],\n                'valid f1': valid_results['f1'],\n                'train timing': elapsed\n            }\n            valid_preds[epoch] = valid_results['preds'].numpy().tolist()\n            \n            print_metrics(history, epochs, log_interval)\n            \n        _, test_preds = predict_fn(test_dist_image_ds)\n        \n        valid_labels = valid_labels.numpy().tolist()\n        test_preds = test_preds.numpy().tolist()\n        test_idx = test_idx.tolist()\n        \n        save_results(train_name, history, valid_labels, valid_preds, test_idx, test_preds)\n        \n        return history, valid_labels, valid_preds\n                \n    return train_fn","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:13.054874Z","iopub.execute_input":"2021-05-30T06:11:13.055282Z","iopub.status.idle":"2021-05-30T06:11:13.078878Z","shell.execute_reply.started":"2021-05-30T06:11:13.055237Z","shell.execute_reply":"2021-05-30T06:11:13.077835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name = 'EfficientNetB7'\n\nepochs = 40\nlr_scaling = 8\nlog_interval = 10","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:13.079959Z","iopub.execute_input":"2021-05-30T06:11:13.080292Z","iopub.status.idle":"2021-05-30T06:11:13.100557Z","shell.execute_reply.started":"2021-05-30T06:11:13.080261Z","shell.execute_reply":"2021-05-30T06:11:13.09917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#trainer = Flower_Trainer(\n#    batch_size_per_replica=16, prediction_batch_size_per_replica=64, shuffle_buffer_size=None,\n#    oversample=False, target_counting=1, grad_acc_steps=1, augmentation_fn=None, log_interval=log_interval)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:13.101847Z","iopub.execute_input":"2021-05-30T06:11:13.102206Z","iopub.status.idle":"2021-05-30T06:11:13.116209Z","shell.execute_reply.started":"2021-05-30T06:11:13.102176Z","shell.execute_reply":"2021-05-30T06:11:13.114999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def print_config(trainer):\n\n    print('use oversampling: {}'.format(trainer.oversample))\n    \n    if trainer.oversample:\n        print('target counting of each class for oversampling {}: '.format(trainer.target_counting))    \n    \n    print('(approximated) nb. of training examples used: {}'.format(trainer.nb_examples_approx))\n    \n    print('per replica batch size for training: {}'.format(trainer.batch_size_per_replica))\n    print('batch size for training: {}'.format(trainer.batch_size))    \n    print('gradient accumulation steps: {}'.format(trainer.grad_acc_steps))\n    print('update size: {}'.format(trainer.update_size))\n    print('updates per epoch: {}'.format(trainer.updates_per_epoch))\n    \n    print('per replica batch size for prediction: {}'.format(trainer.prediction_batch_size_per_replica))\n    print('batch size for prediction: {}'.format(trainer.prediction_batch_size))\n\n    print('use data augmentation: {}'.format(trainer.augmentation_fn is not None))\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:13.117728Z","iopub.execute_input":"2021-05-30T06:11:13.118122Z","iopub.status.idle":"2021-05-30T06:11:13.131054Z","shell.execute_reply.started":"2021-05-30T06:11:13.118091Z","shell.execute_reply":"2021-05-30T06:11:13.129805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print_config(trainer)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:13.132529Z","iopub.execute_input":"2021-05-30T06:11:13.132923Z","iopub.status.idle":"2021-05-30T06:11:13.143893Z","shell.execute_reply.started":"2021-05-30T06:11:13.132889Z","shell.execute_reply":"2021-05-30T06:11:13.142774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(history, desc=''):\n    \n    fig, _ = plt.subplots(figsize=(25, 14.28))\n    # fig, _ = plt.subplots(figsize=(17.25, 10))\n    # plt.tight_layout()\n\n    if desc:\n        fig.suptitle('{}'.format(desc), fontsize=24, y=0.95)\n    \n    xs = range(1, len(history) + 1)\n\n    subplot = (2, 2, 1)\n    ax = plt.subplot(*subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(xs, [history[k]['train loss'] for k in history])\n    ax.plot(xs, [history[k]['valid loss'] for k in history])\n    ax.set_title('model loss', fontsize=20)\n    ax.set_xlabel('epoch', fontsize=16)\n    ax.set_ylabel('loss', fontsize=16)\n    ax.legend(['train loss', 'valid loss'], fontsize=16)\n\n    subplot = (2, 2, 2)\n    ax = plt.subplot(*subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(xs, [history[k]['train acc'] for k in history])\n    ax.plot(xs, [history[k]['valid acc'] for k in history])\n    ax.set_title('model accuracy', fontsize=20)\n    ax.set_xlabel('epoch', fontsize=16)\n    ax.set_ylabel('accuracy', fontsize=16)\n    ax.legend(['train acc', 'valid acc'], fontsize=16)\n\n    subplot = (2, 2, 3)\n    ax = plt.subplot(*subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(xs, [history[k]['train recall'] for k in history])\n    ax.plot(xs, [history[k]['train precision'] for k in history])\n    ax.plot(xs, [history[k]['train f1'] for k in history])\n    ax.set_title('train - recall, precision, f1', fontsize=20)\n    ax.set_xlabel('epoch', fontsize=16)\n    ax.set_ylabel('train metrics', fontsize=16)\n\n    ax.legend(['train recall', 'train precision', 'train f1'], fontsize=16)\n\n    subplot = (2, 2, 4)\n    ax = plt.subplot(*subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(xs, [history[k]['valid recall'] for k in history])\n    ax.plot(xs, [history[k]['valid precision'] for k in history])\n    ax.plot(xs, [history[k]['valid f1'] for k in history])\n    ax.set_title('valid - recall, precision, f1', fontsize=20)\n    ax.set_xlabel('epoch', fontsize=16)\n    ax.set_ylabel('valid metrics', fontsize=16)    \n    \n    ax.legend(['valid recall', 'valid precision', 'valid f1'], fontsize=16)    \n    \ndef plot_history_pair(history_1, history_2, desc_1='', desc_2='', short_desc_1='history 1', short_desc_2='history 2'):\n    \n    nb_epochs_1 = len(history_1)\n    nb_epochs_2 = len(history_2)\n    \n    # extend by the last epoch\n    if nb_epochs_1 < nb_epochs_2:\n        for epoch in range(nb_epochs_1, nb_epochs_2):\n            history_1[epoch] = history_1[nb_epochs_1 - 1]\n    elif nb_epochs_1 > nb_epochs_2:\n        for epoch in range(nb_epochs_2, nb_epochs_1):\n            history_2[epoch] = history_2[nb_epochs_2 - 1]        \n    \n    fig, _ = plt.subplots(figsize=(25, 14.28))\n    # fig, _ = plt.subplots(figsize=(17.25, 10))\n    # plt.tight_layout()\n    \n    if desc_1 and desc_2:\n        fig.suptitle('{}   vs.   {}'.format(desc_1, desc_2), fontsize=24, y=0.95)\n    elif desc_1:\n        fig.suptitle('{}'.format(desc_1), fontsize=24, y=0.95)\n\n    xs = range(1, len(history_1) + 1)\n\n    subplot = (2, 2, 1)\n    ax = plt.subplot(*subplot)\n    ax.set_facecolor('#F8F8F8')\n\n    ax.plot(xs, [history_2[k]['train loss'] for k in history_2], color='b')\n    ax.plot(xs, [history_2[k]['valid loss'] for k in history_2], color='g')     \n\n    ax.plot([], [], linestyle='--', color='k')     \n    ax.plot([], [], linestyle='-', color='k')       \n    \n    ax.plot(xs, [history_1[k]['train loss'] for k in history_1], linestyle='--', color='b')\n    ax.plot(xs, [history_1[k]['valid loss'] for k in history_1], linestyle='--', color='g')\n    \n    ax.set_title('model loss', fontsize=20)\n    ax.set_xlabel('epoch', fontsize=16)\n    ax.set_ylabel('loss', fontsize=16)\n    ax.legend(['train loss', 'valid loss', short_desc_1, short_desc_2], fontsize=16)\n\n    subplot = (2, 2, 2)\n    ax = plt.subplot(*subplot)\n    ax.set_facecolor('#F8F8F8')\n    \n    ax.plot(xs, [history_2[k]['train acc'] for k in history_2], color='b')\n    ax.plot(xs, [history_2[k]['valid acc'] for k in history_2], color='g')     \n    \n    ax.plot([], [], linestyle='--', color='k')     \n    ax.plot([], [], linestyle='-', color='k')     \n    \n    ax.plot(xs, [history_1[k]['train acc'] for k in history_1], linestyle='--', color='b')\n    ax.plot(xs, [history_1[k]['valid acc'] for k in history_1], linestyle='--', color='g')\n    \n    ax.set_title('model accuracy', fontsize=20)\n    ax.set_xlabel('epoch', fontsize=16)\n    ax.set_ylabel('accuracy', fontsize=16)\n    ax.legend(['train acc', 'valid acc', short_desc_1, short_desc_2], fontsize=16)\n\n    subplot = (2, 2, 3)\n    ax = plt.subplot(*subplot)\n    ax.set_facecolor('#F8F8F8')\n\n    ax.plot(xs, [history_2[k]['train recall'] for k in history_2], color='b')\n    ax.plot(xs, [history_2[k]['train precision'] for k in history_2], color='g')\n    ax.plot(xs, [history_2[k]['train f1'] for k in history_2], color='r')    \n\n    ax.plot([], [], linestyle='--', color='k')     \n    ax.plot([], [], linestyle='-', color='k')     \n    \n    ax.plot(xs, [history_1[k]['train recall'] for k in history_1], linestyle='--', color='b')\n    ax.plot(xs, [history_1[k]['train precision'] for k in history_1], linestyle='--', color='g')\n    ax.plot(xs, [history_1[k]['train f1'] for k in history_1], linestyle='--', color='r')\n    \n    ax.set_title('train - recall, precision, f1', fontsize=20)\n    ax.set_xlabel('epoch', fontsize=16)\n    ax.set_ylabel('train metrics', fontsize=16)\n\n    ax.legend(['train recall', 'train precision', 'train f1', short_desc_1, short_desc_2], fontsize=16)\n    \n    subplot = (2, 2, 4)\n    ax = plt.subplot(*subplot)\n    ax.set_facecolor('#F8F8F8')\n    \n    ax.plot(xs, [history_2[k]['valid recall'] for k in history_2], color='b')\n    ax.plot(xs, [history_2[k]['valid precision'] for k in history_2], color='g')\n    ax.plot(xs, [history_2[k]['valid f1'] for k in history_2], color='r')    \n\n    ax.plot([], [], linestyle='--', color='k')     \n    ax.plot([], [], linestyle='-', color='k')      \n    \n    ax.plot(xs, [history_1[k]['valid recall'] for k in history_1], linestyle='--', color='b')\n    ax.plot(xs, [history_1[k]['valid precision'] for k in history_1], linestyle='--', color='g')\n    ax.plot(xs, [history_1[k]['valid f1'] for k in history_1], linestyle='--', color='r')\n    \n    ax.set_title('valid - recall, precision, f1', fontsize=20)\n    ax.set_xlabel('epoch', fontsize=16)\n    ax.set_ylabel('valid metrics', fontsize=16)\n\n    ax.legend(['valid recall', 'valid precision', 'valid f1', short_desc_1, short_desc_2], fontsize=16)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:13.145299Z","iopub.execute_input":"2021-05-30T06:11:13.145694Z","iopub.status.idle":"2021-05-30T06:11:13.193723Z","shell.execute_reply.started":"2021-05-30T06:11:13.145663Z","shell.execute_reply":"2021-05-30T06:11:13.192355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#history_1, valid_labels, valid_preds = trainer.train(train_name='original', model_name=model_name, epochs=epochs, start_lr=1e-5, max_lr=1e-5, end_lr=1e-5, warmup=0.2, lr_scaling=1, optimized_loop=False, verbose=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:13.195687Z","iopub.execute_input":"2021-05-30T06:11:13.196237Z","iopub.status.idle":"2021-05-30T06:11:13.208083Z","shell.execute_reply.started":"2021-05-30T06:11:13.196203Z","shell.execute_reply":"2021-05-30T06:11:13.207071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot_history(history_1, desc='original training')","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:13.20946Z","iopub.execute_input":"2021-05-30T06:11:13.209806Z","iopub.status.idle":"2021-05-30T06:11:13.221976Z","shell.execute_reply.started":"2021-05-30T06:11:13.209777Z","shell.execute_reply":"2021-05-30T06:11:13.221011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compare_training_time(history1, history2, title1, title2):\n\n    avg1 = sum([history1[k]['train timing'] for k in history1 if k != 0]) / (len(history1) - 1)\n    avg2 = sum([history2[k]['train timing'] for k in history2 if k != 0]) / (len(history2) - 1)\n\n    print('Training time per epoch\\n')\n    print('  for the 1st epoch')\n    print(f'    {title1}: {history1[0][\"train timing\"]}')\n    print(f'    {title2}: {history2[0][\"train timing\"]}\\n')\n    print('  for the remaining epoch')\n    print(f'    {title1}: {avg1}')\n    print(f'    {title2}: {avg2}')","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:13.223575Z","iopub.execute_input":"2021-05-30T06:11:13.223863Z","iopub.status.idle":"2021-05-30T06:11:13.23271Z","shell.execute_reply.started":"2021-05-30T06:11:13.223836Z","shell.execute_reply":"2021-05-30T06:11:13.231613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#trainer = Flower_Trainer(\n    #batch_size_per_replica=16, prediction_batch_size_per_replica=64, shuffle_buffer_size=None,\n    #oversample=True, target_counting=600, grad_acc_steps=8, augmentation_fn=None, log_interval=log_interval)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:13.234226Z","iopub.execute_input":"2021-05-30T06:11:13.234546Z","iopub.status.idle":"2021-05-30T06:11:13.333234Z","shell.execute_reply.started":"2021-05-30T06:11:13.234517Z","shell.execute_reply":"2021-05-30T06:11:13.332215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print_config(trainer)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:13.33461Z","iopub.execute_input":"2021-05-30T06:11:13.334893Z","iopub.status.idle":"2021-05-30T06:11:13.342081Z","shell.execute_reply.started":"2021-05-30T06:11:13.334866Z","shell.execute_reply":"2021-05-30T06:11:13.34076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#history_6, valid_labels, valid_preds = trainer.train(train_name='oversampling N=300', model_name=model_name, epochs=epochs, start_lr=1e-5, max_lr=1e-5, end_lr=1e-5, warmup=0.2, lr_scaling=lr_scaling, optimized_loop=True, verbose=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:11:13.34378Z","iopub.execute_input":"2021-05-30T06:11:13.344203Z","iopub.status.idle":"2021-05-30T06:45:36.4428Z","shell.execute_reply.started":"2021-05-30T06:11:13.344154Z","shell.execute_reply":"2021-05-30T06:45:36.441667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot_history_pair(history_1, history_6, desc_1='[Normal]', desc_2='[oversampling N=300]', short_desc_1='oversamp. N=100', short_desc_2='oversamp. N=300')","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:45:36.455793Z","iopub.execute_input":"2021-05-30T06:45:36.456178Z","iopub.status.idle":"2021-05-30T06:45:36.474268Z","shell.execute_reply.started":"2021-05-30T06:45:36.456148Z","shell.execute_reply":"2021-05-30T06:45:36.47291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot_history(history_6, desc='original training')","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:45:36.476053Z","iopub.execute_input":"2021-05-30T06:45:36.476503Z","iopub.status.idle":"2021-05-30T06:45:37.218756Z","shell.execute_reply.started":"2021-05-30T06:45:36.476443Z","shell.execute_reply":"2021-05-30T06:45:37.217624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def random_4_points_2D_batch(height, width, batch_size, probability=1.0):\n    \"\"\"Generate `batch_size * 4` random 2-D points.\n    \n    Each 4 points are inside a rectangle with the same center as the above rectangle\n    but with side length being approximately 1.5 times. This choice is to avoid the\n    image being transformed too disruptively.\n\n    Each point is created first by making it close to the corresponding corner points\n    determined by the rectangle, i.e [0, 0], [0, width], [height, width] and [height, 0]\n    respectively. Then the 4 points are randomly shifted module 4 and randomly flipped.\n    \n    Args:\n        height: 0-D tensor, height of a reference rectangle.\n        width: 0-D tensor, width of a reference rectangle.\n        batch_size: 0-D tensor, the number of 4 points to be generated.\n        probability: 0-D tensor, the probability to use perspective transformation.\n        \n    Returns:\n        points: 3-D tensor of shape [batch_size, 4, 2]\n    \"\"\"\n\n    probability = tf.constant(probability, dtype=tf.float32)\n    \n    sy = height // 4\n    sx = width // 4\n        \n    h, w = height, width\n    \n    # Each has shape [batch_size]\n    y1 = tf.random.uniform(minval = -sy, maxval = sy, shape=[batch_size], dtype=tf.int32)\n    x1 = tf.random.uniform(minval = -sx, maxval = sx, shape=[batch_size], dtype=tf.int32)\n\n    y2 = tf.random.uniform(minval = -sy, maxval = sy, shape=[batch_size], dtype=tf.int32)\n    x2 = tf.random.uniform(minval = 3 * sx, maxval = 5 * sx, shape=[batch_size], dtype=tf.int32)\n\n    y3 = tf.random.uniform(minval = 3 * sy, maxval = 5 * sy, shape=[batch_size], dtype=tf.int32)\n    x3 = tf.random.uniform(minval = 3 * sx, maxval = 5 * sx, shape=[batch_size], dtype=tf.int32)    \n\n    y4 = tf.random.uniform(minval = 3 * sy, maxval = 5 * sy, shape=[batch_size], dtype=tf.int32)\n    x4 = tf.random.uniform(minval = -sx, maxval = sx, shape=[batch_size], dtype=tf.int32)\n        \n    # shape = [4, 2, batch_size]\n    _points = tf.convert_to_tensor([[y1, x1], [y2, x2], [y3, x3], [y4, x4]])\n    \n    # shape = [batch_size, 4, 2]\n    #     Each _points[i, :, :] consists of 4 points\n    #         [y1, x1], [y2, x2], [y3, x3], [y4, x4],\n    #     with xj, yj as scalars this time.\n    _points = tf.transpose(_points, perm=[2, 0, 1])\n    \n    # shape = [4, 2]\n    _standard_points = tf.constant([[0, 0], [0, width], [height, width], [height, 0]], dtype=tf.int32)\n    # shape = [batch_size, 4, 2]\n    _standard_points = tf.broadcast_to(_standard_points[tf.newaxis, :, :], shape=[batch_size, 4, 2])\n    \n    # If to use perspective transformation\n    # shape = [batch_size, 1, 1]\n    do_perspective = tf.cast(tf.random.uniform(shape=[batch_size]) < probability, dtype=tf.int32)[:, tf.newaxis, tf.newaxis]\n    \n    # shape = [batch_size, 4, 2]\n    _points = do_perspective * _points + (1 - do_perspective) * _standard_points\n    \n    # ----------------------------------------\n    # Trick to get random rotations\n    \n    # shape = [batch_size]\n    # shift degree\n    shift_degree = tf.random.uniform(minval=0, maxval=4, shape=[batch_size], dtype=tf.int32)\n    \n    # shape = [batch_size, 4]\n    # Each `_indices[i, :]` is [0, 1, 2, 3] + a random integer in [0, 1, 2, 3]\n    # This shifts the indices of the 4 points, which corresponds to a rotation.\n    _indices = shift_degree[:, tf.newaxis] + tf.range(4, dtype=tf.int32)[tf.newaxis, :]\n    _indices = tf.math.floormod(_indices, 4)\n\n    # shape = [batch_size, 4, 2]\n    # Each `indices[i, :, 0]` is [i, i, i, i]\n    # Each `indices[i, :, :]` is [[i, k1], [i, k2] [i, k3], [i, k4]] where [k0, k1, k2, k3] is `_indices[i]`.\n    _indices = tf.stack(\n        [\n            tf.broadcast_to(\n                tf.range(batch_size, dtype=tf.int32)[:, tf.newaxis],\n                shape=[batch_size, 4]\n            ),\n            _indices\n        ],\n        axis=2\n    )\n\n    # Obtain a tensor `new _points` of shape [batch_size, 4, 2], where\n    # `new _points[i, j] = _points[indices[i, j]]`\n    \n    # shape = [batch_size, 4, 2]\n    _points = tf.gather_nd(_points, _indices)\n      \n    # ----------------------------------------\n    # Trick to get random reflections\n    \n    # All has shape [4]\n    # no reflection\n    reflection_0 = tf.constant([0, 1, 2, 3], dtype=tf.int32)\n    # flip up/down\n    reflection_1 = tf.constant([3, 2, 1, 0], dtype=tf.int32)\n    # flip left/right\n    reflection_2 = tf.constant([1, 0, 3, 2], dtype=tf.int32)\n    \n    # shape = [3, 4]\n    reflections = tf.stack([reflection_0, reflection_1, reflection_2], axis=0)\n    \n    # shape = [batch_size, 3]\n    reflection_types = tf.cast(\n        tf.one_hot(\n            tf.random.uniform(\n                minval=0, maxval=3, shape=[batch_size], dtype=tf.int32\n            ),\n            3\n        ),\n        dtype=tf.int32\n    )\n\n    # shape = [batch_size, 4]\n    selected_reflections = tf.linalg.matmul(reflection_types, reflections)\n        \n    # shape = [batch_size, 4, 2]\n    _indices = tf.stack(\n        [\n            tf.broadcast_to(\n                tf.range(batch_size, dtype=tf.int32)[:, tf.newaxis],\n                shape=[batch_size, 4]\n            ),\n            selected_reflections\n        ],\n        axis=2\n    )\n            \n    # shape = [batch_size, 4, 2]\n    _points = tf.gather_nd(_points, _indices)\n    \n    # ----------------------------------------\n    \n    return _points\n\n\ndef random_4_point_transform_2D_batch(images, probability=1.0):\n    \"\"\"Apply 4 point transformation on 2-D images `images` with randomly\n       generated 4 points on target spaces.\n    \n    On source space, the 4 points are the corner points, i.e [0, 0], [0, width],\n    [height, width] and [height, 0].\n    \n    On target space, the 4 points are randomly generated by `random_4_points_2D_batch()`.\n    \"\"\"\n\n    batch_size, height, width = images.shape[:3]\n\n    # 4 corner points in source image\n    # shape = [batch_size, 4, 2]\n    src_pts = tf.convert_to_tensor([[0, 0], [0, width], [height, width], [height, 0]])\n    src_pts = tf.broadcast_to(src_pts, shape=[batch_size, 4, 2])\n\n    # 4 points in target image\n    # shape = [batch_size, 4, 2]\n    tgt_pts = random_4_points_2D_batch(height, width, batch_size, probability=probability)\n    \n    tgt_images = four_point_transform_2D_batch(images, src_pts, tgt_pts)\n\n    return tgt_images\n\n\ndef four_point_transform_2D_batch(images, src_pts, tgt_pts):\n    \"\"\"Apply 4 point transformation determined by `src_pts` and `tgt_pts`\n       on 2-D images `images`.\n    \n    Args:\n        images: 3-D tensor of shape [batch_size, height, width], or 4-D tensor\n            of shape [batch_size, height, width, channels]\n        src_pts: 3-D tensor of shape [batch_size, 4, 2]\n        tgt_pts: 3-D tensor of shape [batch_size, 4, 2]\n        \n    Returns:\n        A tensor with the same shape as `images`.\n    \"\"\"\n    \n    src_to_tgt_mat = get_src_to_tgt_mat_2D_batch(src_pts, tgt_pts)\n    \n    tgt_images = transform_by_perspective_matrix_2D_batch(images, src_to_tgt_mat)\n    \n    return tgt_images\n\n\ndef transform_by_perspective_matrix_2D_batch(images, src_to_tgt_mat):\n    \"\"\"Transform 2-D images by prespective transformation matrices\n    \n    Args:\n        images: 3-D tensor of shape [batch_size, height, width], or 4-D tensor of\n            shape [batch_size, height, width, channels]\n        src_to_tgt_mat: 3-D tensor of shape [batch_size, 3, 3]. This is the\n            transformation matrix mapping the source space to the target space.\n        \n    Returns:\n        A tensor with the same shape as `image`.        \n    \"\"\"\n\n    batch_size, height, width = images.shape[:3]\n\n    # shape = (3, 3)\n    tgt_to_src_mat = tf.linalg.inv(src_to_tgt_mat)\n        \n    # prepare y coordinates\n    # shape = [height * width]\n    ys = tf.repeat(tf.range(height), width) \n    \n    # prepare x coordinates\n    # shape = [height * width]\n    xs = tf.tile(tf.range(width), [height])\n\n    # prepare indices in target space\n    # shape = [2, height * width]\n    tgt_indices = tf.stack([ys, xs], axis=0)\n    \n    # Change to projective coordinates in the target space by adding ones\n    # shape = [3, height * width]\n    tgt_indices_homo = tf.concat([tgt_indices, tf.ones(shape=[1, height * width], dtype=tf.int32)], axis=0)\n    \n    # Get the corresponding projective coordinate in the source space\n    # shape = [batch_size, 3, height * width]\n    src_indices_homo = tf.linalg.matmul(tgt_to_src_mat, tf.cast(tgt_indices_homo, dtype=tf.float64))\n    \n    # normalize the projective coordinates\n    # shape = [batch_size, 3, height * width]\n    src_indices_normalized = src_indices_homo[:, :3, :] / src_indices_homo[:, 2:, :]\n    \n    # Get the affine coordinate by removing ones\n    # shape = [batch_size, 2, height * width]\n    src_indices_affine = tf.cast(src_indices_normalized, dtype=tf.int32)[:, :2, :]\n    \n    # Mask the points outside the range\n    # shape = [batch_size, height * width]\n    y_mask = tf.logical_and(src_indices_affine[:, 0] >= 0, src_indices_affine[:, 0] <= height - 1)\n    x_mask = tf.logical_and(src_indices_affine[:, 1] >= 0, src_indices_affine[:, 1] <= width - 1)\n    mask = tf.logical_and(y_mask, x_mask)\n    \n    # clip the coordinates\n    # shape = [batch_size, 2, height * width]\n    src_indices = tf.clip_by_value(src_indices_affine, clip_value_min=0, clip_value_max=[[height - 1], [width - 1]])\n    \n    # Get a collection of (y_coord, x_coord)\n    # shape = [batch_size, height * width, 2]\n    src_indices = tf.transpose(src_indices, perm=[0, 2, 1])\n    \n    # shape = [batch_size, height * width, channels]\n    tgt_images = tf.gather_nd(images, src_indices, batch_dims=1)\n    \n    # Set pixel to 0 by using the mask\n    tgt_images = tgt_images * tf.cast(mask[:, :, tf.newaxis], tf.float32)\n    \n    # reshape to [height, width, channels]\n    tgt_images = tf.reshape(tgt_images, images.shape)\n\n    return tgt_images\n\n\ndef get_src_to_tgt_mat_2D_batch(src_pts, tgt_pts):\n    \"\"\"Get the perspective transformation matrix from the source space to the target space,\n       which maps the 4 source points to the 4 target points.\n    \n    Args:\n        src_pts: 3-D tensor of shape [batch_size, 4, 2]\n        tgt_pts: 3-D tensor of shape [batch_size, 4, 2]\n        \n    Returns:\n        3-D tensor of shape [batch_size, 3, 3]\n    \"\"\"\n    \n    src_pts = tf.cast(src_pts, tf.int32)\n    tgt_pts = tf.cast(tgt_pts, tf.int32)\n    \n    # The perspective transformation matrix mapping basis vectors and (1, 1, 1) to `src_pts`\n    # shape = [batch_size, 3, 3]\n    src_mat = get_transformation_mat_2D_batch(src_pts)\n    \n    # The perspective transformation matrix mapping basis vectors and (1, 1, 1) to `tgt_pts`\n    # shape = [batch_size, 3, 3]\n    tgt_mat = get_transformation_mat_2D_batch(tgt_pts)\n    \n    # The perspective transformation matrix mapping `src_pts` to `tgt_pts`\n    # shape = [batch_size, 3, 3]\n    src_to_tgt_mat = tf.linalg.matmul(tgt_mat, tf.linalg.inv(src_mat))\n    \n    return src_to_tgt_mat\n  \n    \ndef get_transformation_mat_2D_batch(four_pts):\n    \"\"\"Get the perspective transformation matrix from a space to another space,\n       which maps the basis vectors and (1, 1, 1) to the 4 points defined by `four_pts`.\n    \n    Args:\n        four_pts: 3-D tensor of shape [batch_size, 4, 2]\n        \n    Returns:\n        3-D tensor of shape [batch_size, 3, 3]        \n    \"\"\"\n    \n    batch_size = four_pts.shape[0]\n    \n    # Change to projective coordinates by adding ones\n    # shape = [batch_size, 3, 4]\n    pts_homo = tf.transpose(tf.concat([four_pts, tf.ones(shape=[batch_size, 4, 1], dtype=tf.int32)], axis=-1), perm=[0, 2, 1])\n    \n    pts_homo = tf.cast(pts_homo, tf.float64)\n    \n    # Find `scalars` such that: src_pts_homo[:, 3:] * scalars == src_pts_homo[:, 3:]\n    # shape = [batch_size 3, 3]\n    inv_mat = tf.linalg.inv(pts_homo[:, :, :3])\n    # shape = [batch_size, 3, 1]\n    scalars = tf.linalg.matmul(inv_mat, pts_homo[:, :, 3:])\n    \n    # Get the matrix transforming unit vectors to the 4 source points\n    # shape = [batch_size, 3, 3]    \n    mat = tf.transpose(tf.transpose(pts_homo[:, :, :3], perm=[0, 2, 1]) * scalars, perm=[0, 2, 1])\n    \n    return mat\n\ndef perspective_transform(images, labels, probability=1.0):\n    \"\"\"\n    This is the method used for dataset transformation (random perspective transformation on images).\n    \"\"\"\n        \n    transformed_images = random_4_point_transform_2D_batch(images, probability=probability)\n    \n    return transformed_images, labels","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:45:37.225139Z","iopub.execute_input":"2021-05-30T06:45:37.225507Z","iopub.status.idle":"2021-05-30T06:45:37.273407Z","shell.execute_reply.started":"2021-05-30T06:45:37.225458Z","shell.execute_reply":"2021-05-30T06:45:37.272331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = get_training_dataset(batch_size=16, shuffle_buffer_size=1, ordered=True)\ntrain_iter = iter(train_dataset)\n\ntransformed_train_dataset = train_dataset.map(lambda images, labels: perspective_transform(images, labels, probability=0.8))\ntransformed_train_iter = iter(transformed_train_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:45:37.274996Z","iopub.execute_input":"2021-05-30T06:45:37.27528Z","iopub.status.idle":"2021-05-30T06:45:38.764898Z","shell.execute_reply.started":"2021-05-30T06:45:37.275253Z","shell.execute_reply":"2021-05-30T06:45:38.763651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run this cell again for next set of images\nbatch = next(train_iter)\ntransformed_batch = next(transformed_train_iter)\n\ndisplay_pairs_of_image_batch(batch, transformed_batch, ds_name_1='original dataset', ds_name_2='transformed dataset')","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:45:38.769141Z","iopub.execute_input":"2021-05-30T06:45:38.769684Z","iopub.status.idle":"2021-05-30T06:45:42.116469Z","shell.execute_reply.started":"2021-05-30T06:45:38.769634Z","shell.execute_reply":"2021-05-30T06:45:42.11501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer = Flower_Trainer(\n    batch_size_per_replica=16, prediction_batch_size_per_replica=64, shuffle_buffer_size=None,\n    oversample=True, target_counting=600, grad_acc_steps=8, augmentation_fn=perspective_transform,\n    probability=0.35, log_interval=log_interval\n)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:45:42.118029Z","iopub.execute_input":"2021-05-30T06:45:42.118598Z","iopub.status.idle":"2021-05-30T06:45:42.476787Z","shell.execute_reply.started":"2021-05-30T06:45:42.118556Z","shell.execute_reply":"2021-05-30T06:45:42.475712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_8, valid_labels, valid_preds = trainer.train(train_name='with data augmentation', model_name=model_name, epochs=epochs, start_lr=1e-5, max_lr=1e-5, end_lr=1e-5, warmup=0.2, lr_scaling=lr_scaling, optimized_loop=True, verbose=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:45:42.480511Z","iopub.execute_input":"2021-05-30T06:45:42.480856Z","iopub.status.idle":"2021-05-30T07:20:25.867587Z","shell.execute_reply.started":"2021-05-30T06:45:42.480823Z","shell.execute_reply":"2021-05-30T07:20:25.866442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_pair(history_8, desc='original training')","metadata":{"execution":{"iopub.status.busy":"2021-05-30T07:20:25.882962Z","iopub.execute_input":"2021-05-30T07:20:25.883388Z","iopub.status.idle":"2021-05-30T07:20:26.730597Z","shell.execute_reply.started":"2021-05-30T07:20:25.883351Z","shell.execute_reply":"2021-05-30T07:20:26.729327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('results of the last epoch\\n')\n\nprint('    without data augmentation:\\n')\n\nprint('        valid recall:', history_6[len(history_6) - 1]['valid recall'])\nprint('        valid precision:', history_6[len(history_6) - 1]['valid precision'])\nprint('        valid f1:', history_6[len(history_6) - 1]['valid f1'])\n\nprint('\\n    with data augmentation:\\n')\n\nprint('        valid recall:', history_8[len(history_8) - 1]['valid recall'])\nprint('        valid precision:', history_8[len(history_8) - 1]['valid precision'])\nprint('        valid f1:', history_8[len(history_8) - 1]['valid f1'])","metadata":{"execution":{"iopub.status.busy":"2021-05-30T07:20:26.732107Z","iopub.execute_input":"2021-05-30T07:20:26.732458Z","iopub.status.idle":"2021-05-30T07:20:26.743671Z","shell.execute_reply.started":"2021-05-30T07:20:26.732426Z","shell.execute_reply":"2021-05-30T07:20:26.742568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}