{"cells":[{"metadata":{},"cell_type":"markdown","source":"# About this kernel\n"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import math, re, os, time\nimport datetime\nimport tensorflow as tf\n\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\n\npolicy = mixed_precision.Policy('mixed_bfloat16')\nmixed_precision.set_policy(policy)\ntf.config.optimizer.set_jit(True)\n\nprint('Compute dtype: %s' % policy.compute_dtype)\nprint('Variable dtype: %s' % policy.variable_dtype)\n\nimport numpy as np\nfrom collections import namedtuple\nfrom collections import Counter\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow.keras.backend as K\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE\n\n!pip install tensorflow-addons\nimport tensorflow_addons as tfa\n\n!pip install -q efficientnet\nimport efficientnet.tfkeras as efn\n\nimport gc\ngc.enable()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls -l /kaggle/input/keras-pretrained-models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls -l /kaggle/input/","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TPU or GPU detection"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Competition data access\nTPUs read data directly from Google Cloud Storage (GCS). This Kaggle utility will copy the dataset to a GCS bucket co-located with the TPU. If you have multiple datasets attached to the notebook, you can pass the name of a specific dataset to the get_gcs_path function. The name of the dataset is the name of the directory it is mounted in. Use `!ls /kaggle/input/` to list attached datasets."},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path('flower-classification') # you can list the bucket with \"!gsutil ls $GCS_DS_PATH\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 3\n\nEPOCH_START_TRAIN_ALL = 2\nEPOCH_END_TRAIN_ALL = 2\nEPOCH_SAVING_START = 2\n\ninclude_additional_files = True\n\nimage_size = 192\nIMAGE_SIZE = [image_size, image_size]\n\nOVERSAMPLE = True\nAUGUMENTATION = True\n\n# We want each class occur at least (approximately) `TARGET_MIN_COUNTING` times\nTARGET_MIN_COUNTING = 1000\nif not OVERSAMPLE:\n    TARGET_MIN_COUNTING = 1\n\nbackend_names = [\n    \"EfficientNetB7\",\n    \"DenseNet201\",\n    \"ResNet152V2\",\n    \"Xception\"\n]\n\nbackend_mapping = {\n    \"EfficientNetB7\": efn.EfficientNetB7,\n    \"DenseNet201\": tf.keras.applications.DenseNet201,\n    \"ResNet152V2\": tf.keras.applications.ResNet152V2,    \n    \"Xception\": tf.keras.applications.Xception\n}    \n    \n# backend_name = \"Xception\"\nbackend_name = \"EfficientNetB7\"\nbackend = backend_mapping[backend_name]\n    \n\nGCS_PATH_SELECT = { # available image sizes\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec') # predictions on this dataset should be submitted for the competition\n\nCLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102\n\n# Learning rate schedule for TPU, GPU and CPU.\n# Using an LR ramp up because fine-tuning a pre-trained model.\n# Starting with a high LR would break the pre-trained weights.\n\nLR_START = 1e-5\nLR_MAX = 5e-5 * strategy.num_replicas_in_sync\nLR_MIN = 1e-5\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = 0.85\n        \n@tf.function\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY ** (epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## More Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"MORE_IMAGES_GCS_DS_PATH = KaggleDatasets().get_gcs_path('tf-flower-photo-tfrec')\n\nMOREIMAGES_PATH_SELECT = {\n    192: '/tfrecords-jpeg-192x192',\n    224: '/tfrecords-jpeg-224x224',\n    331: '/tfrecords-jpeg-331x331',\n    512: '/tfrecords-jpeg-512x512'\n}\nMOREIMAGES_PATH = MOREIMAGES_PATH_SELECT[IMAGE_SIZE[0]]\n\nIMAGENET_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/imagenet' + MOREIMAGES_PATH + '/*.tfrec')\nINATURELIST_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/inaturalist' + MOREIMAGES_PATH + '/*.tfrec')\nOPENIMAGE_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/openimage' + MOREIMAGES_PATH + '/*.tfrec')\nOXFORD_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/oxford_102' + MOREIMAGES_PATH + '/*.tfrec')\nTENSORFLOW_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/tf_flowers' + MOREIMAGES_PATH + '/*.tfrec')\nADDITIONAL_TRAINING_FILENAMES = IMAGENET_FILES + INATURELIST_FILES + OPENIMAGE_FILES + OXFORD_FILES + TENSORFLOW_FILES\n\nif include_additional_files:\n    TRAINING_FILENAMES = TRAINING_FILENAMES + ADDITIONAL_TRAINING_FILENAMES","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualization utilities\ndata -> pixels, nothing of much interest for the machine learning practitioner in this section."},{"metadata":{"trusted":true},"cell_type":"code","source":"# numpy and matplotlib defaults\nnp.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n    \ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        ### title = '' if label is None else CLASSES[label]\n        title = \"\"\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols) * 40 + 3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n\ndef display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()\n    \ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Datasets"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    #image = tf.image.random_saturation(image, 0, 2)\n    return image, label   \n\ndef get_training_dataset(batch_size):\n\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    ### dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat()  # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(12753)\n    dataset = dataset.batch(batch_size, drop_remainder=True)  # slighly faster with fixed tensor sizes\n    dataset = dataset.prefetch(AUTO)  # prefetch next batch while training (autotune prefetch buffer size)\n    \n    return dataset\n\ndef get_validation_dataset(batch_size, ordered=False, repeated=False):\n    \n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    \n    if repeated:\n        dataset = dataset.repeat()\n        dataset = dataset.shuffle(3712)\n        \n    dataset = dataset.batch(batch_size, drop_remainder=repeated) # slighly faster with fixed tensor sizes\n    \n    # dataset = dataset.cache()  # seems this is problematic in the setting of this kernel\n    \n    dataset = dataset.prefetch(AUTO)  # prefetch next batch while training (autotune prefetch buffer size)\n    \n    return dataset\n\ndef get_test_dataset(batch_size, ordered=True):\n    \n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(batch_size)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    \n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\n\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Oversample\n\nAlso include Chris Deotte's data augmentation"},{"metadata":{},"cell_type":"markdown","source":"## 1 - Get labels and their countings"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get labels and their countings\n\ndef get_training_dataset_raw():\n\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=False)\n    return dataset\n\n\nraw_training_dataset = get_training_dataset_raw()\n\nlabel_counter = Counter()\nfor images, labels in raw_training_dataset:\n    label_counter.update([labels.numpy()])\n\ndel raw_training_dataset    \n    \nlabel_counting_sorted = label_counter.most_common()\n\nNUM_TRAINING_IMAGES = sum([x[1] for x in label_counting_sorted])\nprint(\"number of examples in the original training dataset: {}\".format(NUM_TRAINING_IMAGES))\n\nprint(\"labels in the original training dataset, sorted by occurrence\")\nlabel_counting_sorted","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2 - Define the number of repetitions for each class\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_num_of_repetition_for_class(class_id):\n    \n    counting = label_counter[class_id]\n    if counting >= TARGET_MIN_COUNTING:\n        return 1.0\n    \n    num_to_repeat = TARGET_MIN_COUNTING / counting\n    \n    return num_to_repeat\n\nnumbers_of_repetition_for_classes = {class_id: get_num_of_repetition_for_class(class_id) for class_id in range(104)}\n\nprint(\"number of repetitions for each class (if > 1)\")\n{k: v for k, v in sorted(numbers_of_repetition_for_classes.items(), key=lambda item: item[1], reverse=True) if v > 1}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3 - Define the number of repetitions for each training example"},{"metadata":{"trusted":true},"cell_type":"code","source":"# This will be called later in `get_training_dataset_with_oversample()`\n\nkeys_tensor = tf.constant([k for k in numbers_of_repetition_for_classes])\nvals_tensor = tf.constant([numbers_of_repetition_for_classes[k] for k in numbers_of_repetition_for_classes])\ntable = tf.lookup.StaticHashTable(tf.lookup.KeyValueTensorInitializer(keys_tensor, vals_tensor), -1)\n\ndef get_num_of_repetition_for_example(training_example):\n    \n    _, label = training_example\n    \n    num_to_repeat = table.lookup(label)\n    num_to_repeat_integral = tf.cast(int(num_to_repeat), tf.float32)\n    residue = num_to_repeat - num_to_repeat_integral\n    \n    num_to_repeat = num_to_repeat_integral + tf.cast(tf.random.uniform(shape=()) <= residue, tf.float32)\n    \n    return tf.cast(num_to_repeat, tf.int64)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4 - Use data augmentation to avoid (exactly) same images appear too many times"},{"metadata":{},"cell_type":"markdown","source":"## Transform labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"def label_transform(images, labels):\n\n    # Make labels\n    if len(labels.shape) == 1:\n        labels = tf.one_hot(labels, len(CLASSES))\n\n    return images, labels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Rotation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_batch_transformatioin_matrix(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    \"\"\"Returns a tf.Tensor of shape (batch_size, 3, 3) with each element along the 1st axis being\n       an image transformation matrix (which transforms indicies).\n\n    Args:\n        rotation: 1-D Tensor with shape [batch_size].\n        shear: 1-D Tensor with shape [batch_size].\n        height_zoom: 1-D Tensor with shape [batch_size].\n        width_zoom: 1-D Tensor with shape [batch_size].\n        height_shift: 1-D Tensor with shape [batch_size].\n        width_shift: 1-D Tensor with shape [batch_size].\n        \n    Returns:\n        A 3-D Tensor with shape [batch_size, 3, 3].\n    \"\"\"    \n\n    # A trick to get batch_size\n    batch_size = tf.cast(tf.reduce_sum(tf.ones_like(rotation)), tf.int32)    \n    \n    # CONVERT DEGREES TO RADIANS\n    rotation = tf.constant(math.pi) * rotation / 180.0\n    shear = tf.constant(math.pi) * shear / 180.0\n\n    # shape = (batch_size,)\n    one = tf.ones_like(rotation, dtype=tf.float32)\n    zero = tf.zeros_like(rotation, dtype=tf.float32)\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation) # shape = (batch_size,)\n    s1 = tf.math.sin(rotation) # shape = (batch_size,)\n\n    # Intermediate matrix for rotation, shape = (9, batch_size) \n    rotation_matrix_temp = tf.stack([c1, s1, zero, -s1, c1, zero, zero, zero, one], axis=0)\n    # shape = (batch_size, 9)\n    rotation_matrix_temp = tf.transpose(rotation_matrix_temp)\n    # Fianl rotation matrix, shape = (batch_size, 3, 3)\n    rotation_matrix = tf.reshape(rotation_matrix_temp, shape=(batch_size, 3, 3))\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear) # shape = (batch_size,)\n    s2 = tf.math.sin(shear) # shape = (batch_size,)\n    \n    # Intermediate matrix for shear, shape = (9, batch_size) \n    shear_matrix_temp = tf.stack([one, s2, zero, zero, c2, zero, zero, zero, one], axis=0)\n    # shape = (batch_size, 9)\n    shear_matrix_temp = tf.transpose(shear_matrix_temp)\n    # Fianl shear matrix, shape = (batch_size, 3, 3)\n    shear_matrix = tf.reshape(shear_matrix_temp, shape=(batch_size, 3, 3))    \n    \n\n    # ZOOM MATRIX\n    \n    # Intermediate matrix for zoom, shape = (9, batch_size) \n    zoom_matrix_temp = tf.stack([one / height_zoom, zero, zero, zero, one / width_zoom, zero, zero, zero, one], axis=0)\n    # shape = (batch_size, 9)\n    zoom_matrix_temp = tf.transpose(zoom_matrix_temp)\n    # Fianl zoom matrix, shape = (batch_size, 3, 3)\n    zoom_matrix = tf.reshape(zoom_matrix_temp, shape=(batch_size, 3, 3))\n    \n    # SHIFT MATRIX\n    \n    # Intermediate matrix for shift, shape = (9, batch_size) \n    shift_matrix_temp = tf.stack([one, zero, height_shift, zero, one, width_shift, zero, zero, one], axis=0)\n    # shape = (batch_size, 9)\n    shift_matrix_temp = tf.transpose(shift_matrix_temp)\n    # Fianl shift matrix, shape = (batch_size, 3, 3)\n    shift_matrix = tf.reshape(shift_matrix_temp, shape=(batch_size, 3, 3))    \n        \n    return tf.linalg.matmul(tf.linalg.matmul(rotation_matrix, shear_matrix), tf.linalg.matmul(zoom_matrix, shift_matrix))\n\n\ndef basic_transform(images, labels):\n    \"\"\"Returns a tf.Tensor of the same shape as `images`, represented a batch of randomly transformed images.\n\n    Args:\n        images: 4-D Tensor with shape (batch_size, width, hight, depth).\n            Currently, `depth` can only be 3.\n        \n    Returns:\n        A 4-D Tensor with the same shape as `images`.\n    \"\"\" \n    \n    # input `images`: a batch of images [batch_size, dim, dim, 3]\n    # output: images randomly rotated, sheared, zoomed, and shifted\n    DIM = images.shape[1]\n    XDIM = DIM % 2  # fix for size 331\n    \n    # A trick to get batch_size\n    batch_size = tf.cast(tf.reduce_sum(tf.ones_like(images)) / (images.shape[1] * images.shape[2] * images.shape[3]), tf.int32)\n    \n    rot = 15.0 * tf.random.normal([batch_size], dtype='float32')\n    shr = 5.0 * tf.random.normal([batch_size], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([batch_size], dtype='float32') / 10.0\n    w_zoom = 1.0 + tf.random.normal([batch_size], dtype='float32') / 10.0\n    h_shift = 16.0 * tf.random.normal([batch_size], dtype='float32') \n    w_shift = 16.0 * tf.random.normal([batch_size], dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    # shape = (batch_size, 3, 3)\n    m = get_batch_transformatioin_matrix(rot, shr, h_zoom, w_zoom, h_shift, w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat(tf.range(DIM // 2, -DIM // 2, -1), DIM)  # shape = (DIM * DIM,)\n    y = tf.tile(tf.range(-DIM // 2, DIM // 2), [DIM])  # shape = (DIM * DIM,)\n    z = tf.ones([DIM * DIM], dtype='int32')  # shape = (DIM * DIM,)\n    idx = tf.stack([x, y, z])  # shape = (3, DIM * DIM)\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = tf.linalg.matmul(m, tf.cast(idx, dtype='float32'))  # shape = (batch_size, 3, DIM ** 2)\n    idx2 = K.cast(idx2, dtype='int32')  # shape = (batch_size, 3, DIM ** 2)\n    idx2 = K.clip(idx2, -DIM // 2 + XDIM + 1, DIM // 2)  # shape = (batch_size, 3, DIM ** 2)\n    \n    # FIND ORIGIN PIXEL VALUES\n    # shape = (batch_size, 2, DIM ** 2)\n    idx3 = tf.stack([DIM // 2 - idx2[:, 0, ], DIM // 2 - 1 + idx2[:, 1, ]], axis=1)  \n    \n    # shape = (batch_size, DIM ** 2, 3)\n    d = tf.gather_nd(images, tf.cast(tf.transpose(idx3, perm=[0, 2, 1]), dtype=tf.int64), batch_dims=1)\n        \n    # shape = (batch_size, DIM, DIM, 3)\n    new_images = tf.reshape(d, (batch_size, DIM, DIM, 3))\n\n    return new_images, labels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CutMix + MixUp"},{"metadata":{"trusted":true},"cell_type":"code","source":"def batch_cutmix(images, labels):\n    \n    PROBABILITY = 0.1\n    \n    # A trick to get batch_size\n    batch_size = tf.cast(tf.reduce_sum(tf.ones_like(images)) / (images.shape[1] * images.shape[2] * images.shape[3]), tf.int32)  \n    \n    DIM = IMAGE_SIZE[0]\n    \n    # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n    # This is a tensor containing 0 or 1 -- 0: no cutmix.\n    # shape = [batch_size]\n    do_cutmix = tf.cast(tf.random.uniform([batch_size], 0, 1) <= PROBABILITY, tf.int32)\n    \n    # Choose random images in the batch for cutmix\n    # shape = [batch_size]\n    new_image_indices = tf.random.uniform([batch_size], minval=0, maxval=batch_size, dtype=tf.int32)\n    \n    # Choose random location in the original image to put the new images\n    # shape = [batch_size]\n    new_x = tf.random.uniform([batch_size], minval=0, maxval=DIM, dtype=tf.int32)\n    new_y = tf.random.uniform([batch_size], minval=0, maxval=DIM, dtype=tf.int32)\n    \n    # Random width for new images, shape = [batch_size]\n    b = tf.random.uniform([batch_size], 0, 1) # this is beta dist with alpha=1.0\n    new_width = tf.cast(DIM * tf.math.sqrt(1-b), tf.int32) * do_cutmix\n    \n    # shape = [batch_size]\n    new_y0 = tf.math.maximum(0, new_y - new_width // 2)\n    new_y1 = tf.math.minimum(DIM, new_y + new_width // 2)\n    new_x0 = tf.math.maximum(0, new_x - new_width // 2)\n    new_x1 = tf.math.minimum(DIM, new_x + new_width // 2)\n    \n    # shape = [batch_size, DIM]\n    target = tf.broadcast_to(tf.range(DIM), shape=(batch_size, DIM))\n    \n    # shape = [batch_size, DIM]\n    mask_y = tf.math.logical_and(new_y0[:, tf.newaxis] <= target, target <= new_y1[:, tf.newaxis])\n    \n    # shape = [batch_size, DIM]\n    mask_x = tf.math.logical_and(new_x0[:, tf.newaxis] <= target, target <= new_x1[:, tf.newaxis])    \n    \n    # shape = [batch_size, DIM, DIM]\n    mask = tf.cast(tf.math.logical_and(mask_y[:, :, tf.newaxis], mask_x[:, tf.newaxis, :]), tf.float32)\n\n    # All components are of shape [batch_size, DIM, DIM, 3]\n    new_images =  images * tf.broadcast_to(1 - mask[:, :, :, tf.newaxis], [batch_size, DIM, DIM, 3]) + \\\n                    tf.gather(images, new_image_indices) * tf.broadcast_to(mask[:, :, :, tf.newaxis], [batch_size, DIM, DIM, 3])\n\n    a = tf.cast(new_width ** 2 / DIM ** 2, tf.float32)    \n        \n    # Make labels\n    if len(labels.shape) == 1:\n        labels = tf.one_hot(labels, len(CLASSES))\n        \n    new_labels =  (1-a)[:, tf.newaxis] * labels + a[:, tf.newaxis] * tf.gather(labels, new_image_indices)        \n        \n    return new_images, new_labels\n\n\ndef batch_mixup(images, labels):\n\n    PROBABILITY = 0.1\n    \n    # A trick to get batch_size\n    batch_size = tf.cast(tf.reduce_sum(tf.ones_like(images)) / (images.shape[1] * images.shape[2] * images.shape[3]), tf.int32)  \n    \n    DIM = IMAGE_SIZE[0]\n\n    # Do `batch_mixup` with a probability = `PROBABILITY`\n    # This is a tensor containing 0 or 1 -- 0: no mixup.\n    # shape = [batch_size]\n    do_mixup = tf.cast(tf.random.uniform([batch_size], 0, 1) <= PROBABILITY, tf.int32)\n\n    # Choose random images in the batch for cutmix\n    # shape = [batch_size]\n    new_image_indices = tf.random.uniform([batch_size], minval=0, maxval=batch_size, dtype=tf.int32)\n    \n    # ratio of importance of the 2 images to be mixed up\n    # shape = [batch_size]\n    a = tf.random.uniform([batch_size], 0, 1) * tf.cast(do_mixup, tf.float32)  # this is beta dist with alpha=1.0\n                \n    # The second part corresponds to the images to be added to the original images `images`.\n    new_images =  (1-a)[:, tf.newaxis, tf.newaxis, tf.newaxis] * images + a[:, tf.newaxis, tf.newaxis, tf.newaxis] * tf.gather(images, new_image_indices)\n\n    # Make labels\n    if len(labels.shape) == 1:\n        labels = tf.one_hot(labels, len(CLASSES))\n    new_labels =  (1-a)[:, tf.newaxis] * labels + a[:, tf.newaxis] * tf.gather(labels, new_image_indices)\n\n    return new_images, new_labels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Perspective Transformation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def random_4_points_2D_batch(height, width, batch_size):\n    \"\"\"Generate `batch_size * 4` random 2-D points.\n    \n    Each 4 points are inside a rectangle with the same center as the above rectangle but with side length being approximately 1.5 times.\n    This choice is to avoid the image being transformed too disruptively.\n\n    Each point is created first by making it close to the corresponding corner points determined by the rectangle, i.e\n    [0, 0], [0, width], [height, width] and [height, 0] respectively. Then the 4 points are randomly shifted module 4.\n    \n    Args:\n        height: 0-D tensor, height of a reference rectangle.\n        width: 0-D tensor, width of a reference rectangle.\n        batch_size: 0-D tensor, the number of 4 points to be generated\n        \n    Returns:\n        points: 3-D tensor of shape [batch_size, 4, 2]\n    \"\"\"\n\n    sy = height // 4\n    sx = width // 4\n        \n    h, w = height, width\n    \n    y1 = tf.random.uniform(minval = -sy, maxval = sy, shape=[batch_size], dtype=tf.int64)\n    x1 = tf.random.uniform(minval = -sx, maxval = sx, shape=[batch_size], dtype=tf.int64)\n    \n    y2 = tf.random.uniform(minval = -sy, maxval = sy, shape=[batch_size], dtype=tf.int64)\n    x2 = tf.random.uniform(minval = 3 * sx, maxval = 5 * sx, shape=[batch_size], dtype=tf.int64)\n\n    y3 = tf.random.uniform(minval = 3 * sy, maxval = 5 * sy, shape=[batch_size], dtype=tf.int64)\n    x3 = tf.random.uniform(minval = 3 * sx, maxval = 5 * sx, shape=[batch_size], dtype=tf.int64)    \n    \n    y4 = tf.random.uniform(minval = 3 * sy, maxval = 5 * sy, shape=[batch_size], dtype=tf.int64)\n    x4 = tf.random.uniform(minval = -sx, maxval = sx, shape=[batch_size], dtype=tf.int64)\n            \n    # shape = [4, 2, batch_size]\n    points = tf.convert_to_tensor([[y1, x1], [y2, x2], [y3, x3], [y4, x4], [y1, x1], [y2, x2], [y3, x3], [y4, x4]])\n    \n    # shape = [batch_size, 4, 2]\n    points = tf.transpose(points, perm=[2, 0, 1])\n    \n    # Trick to get random rotation\n    # shape = [batch_size, 8, 2]\n    points = tf.tile(points, multiples=[1, 2, 1])    \n    # shape = [batch_size]\n    start_indices = tf.random.uniform(minval=0, maxval=4, shape=[batch_size], dtype=tf.int64)\n    # shape = [batch_size, 4]\n    indices = start_indices[:, tf.newaxis] + tf.range(4, dtype=tf.int64)[tf.newaxis, :]\n    # shape = [batch_size, 4, 2]\n    indices = tf.stack([tf.broadcast_to(tf.range(batch_size, dtype=tf.int64)[:, tf.newaxis], shape=[batch_size, 4]), indices], axis=2)    \n    \n    # shape = [batch_size, 4, 2]\n    points = tf.gather_nd(points, tf.cast(indices, dtype=tf.int64))\n        \n    return points\n\n\ndef random_4_point_transform_2D_batch(images):\n    \"\"\"Apply 4 point transformation on 2-D images `images` with randomly generated 4 points on target spaces.\n    \n    On source space, the 4 points are the corner points, i.e [0, 0], [0, width], [height, width] and [height, 0].\n    On target space, the 4 points are randomly generated by `random_4_points_2D_batch()`.\n    \"\"\"\n\n    batch_size, height, width = images.shape[:3]\n\n    # 4 corner points in source image\n    # shape = [batch_size, 4, 2]\n    src_pts = tf.convert_to_tensor([[0, 0], [0, width], [height, width], [height, 0]])\n    src_pts = tf.broadcast_to(src_pts, shape=[batch_size, 4, 2])\n\n    # 4 points in target image\n    # shape = [batch_size, 4, 2]\n    tgt_pts = random_4_points_2D_batch(height, width, batch_size)\n    \n    tgt_images = four_point_transform_2D_batch(images, src_pts, tgt_pts)\n\n    return tgt_images\n\n\ndef four_point_transform_2D_batch(images, src_pts, tgt_pts):\n    \"\"\"Apply 4 point transformation determined by `src_pts` and `tgt_pts` on 2-D images `images`.\n    \n    Args:\n        images: 3-D tensor of shape [batch_size, height, width], or 4-D tensor of shape [batch_size, height, width, channels]\n        src_pts: 3-D tensor of shape [batch_size, 4, 2]\n        tgt_pts: 3-D tensor of shape [batch_size, 4, 2]\n        \n    Returns:\n        A tensor with the same shape as `images`.\n    \"\"\"\n    \n    src_to_tgt_mat = get_src_to_tgt_mat_2D_batch(src_pts, tgt_pts)\n    \n    tgt_images = transform_by_perspective_matrix_2D_batch(images, src_to_tgt_mat)\n    \n    return tgt_images\n\n\ndef transform_by_perspective_matrix_2D_batch(images, src_to_tgt_mat):\n    \"\"\"Transform 2-D images by prespective transformation matrices\n    \n    Args:\n        images: 3-D tensor of shape [batch_size, height, width], or 4-D tensor of shape [batch_size, height, width, channels]\n        src_to_tgt_mat: 3-D tensor of shape [batch_size, 3, 3]. This is the transformation matrix mapping the source space to the target space.\n        \n    Returns:\n        A tensor with the same shape as `image`.        \n    \"\"\"\n\n    batch_size, height, width = images.shape[:3]\n\n    # shape = (3, 3)\n    tgt_to_src_mat = tf.linalg.inv(src_to_tgt_mat)\n        \n    # prepare y coordinates\n    # shape = [height * width]\n    ys = tf.repeat(tf.range(height), width) \n    \n    # prepare x coordinates\n    # shape = [height * width]\n    xs = tf.tile(tf.range(width), [height])\n\n    # prepare indices in target space\n    # shape = [2, height * width]\n    tgt_indices = tf.stack([ys, xs], axis=0)\n    \n    # Change to projective coordinates in the target space by adding ones\n    # shape = [3, height * width]\n    tgt_indices_homo = tf.concat([tgt_indices, tf.ones(shape=[1, height * width], dtype=tf.int32)], axis=0)\n    \n    # Get the corresponding projective coordinate in the source space\n    # shape = [batch_size, 3, height * width]\n    src_indices_homo = tf.linalg.matmul(tgt_to_src_mat, tf.cast(tgt_indices_homo, dtype=tf.float64))\n    \n    # normalize the projective coordinates\n    # shape = [batch_size, 3, height * width]\n    src_indices_normalized = src_indices_homo[:, :3, :] / src_indices_homo[:, 2:, :]\n    \n    # Get the affine coordinate by removing ones\n    # shape = [batch_size, 2, height * width]\n    src_indices_affine = tf.cast(src_indices_normalized, dtype=tf.int64)[:, :2, :]\n    \n    # Mask the points outside the range\n    # shape = [batch_size, height * width]\n    y_mask = tf.logical_and(src_indices_affine[:, 0] >= 0, src_indices_affine[:, 0] <= height - 1)\n    x_mask = tf.logical_and(src_indices_affine[:, 1] >= 0, src_indices_affine[:, 1] <= width - 1)\n    mask = tf.logical_and(y_mask, x_mask)\n    \n    # clip the coordinates\n    # shape = [batch_size, 2, height * width]\n    src_indices = tf.clip_by_value(src_indices_affine, clip_value_min=0, clip_value_max=[[height - 1], [width - 1]])\n    \n    # Get a collection of (y_coord, x_coord)\n    # shape = [batch_size, height * width, 2]\n    src_indices = tf.transpose(src_indices, perm=[0, 2, 1])\n    \n    # shape = [batch_size, height * width, channels]\n    tgt_images = tf.gather_nd(images, tf.cast(src_indices, dtype=tf.int64), batch_dims=1)\n    \n    # Set pixel to 0 by using the mask\n    tgt_images = tgt_images * tf.cast(mask[:, :, tf.newaxis], tf.float32)\n    \n    # reshape to [height, width, channels]\n    tgt_images = tf.reshape(tgt_images, images.shape)\n\n    return tgt_images\n\n\ndef get_src_to_tgt_mat_2D_batch(src_pts, tgt_pts):\n    \"\"\"Get the perspective transformation matrix from the source space to the target space, which maps the 4 source points to the 4 target points.\n    \n    Args:\n        src_pts: 3-D tensor of shape [batch_size, 4, 2]\n        tgt_pts: 3-D tensor of shape [batch_size, 4, 2]\n        \n    Returns:\n        2-D tensor of shape [batch_size, 3, 3]\n    \"\"\"\n    \n    src_pts = tf.cast(src_pts, tf.int64)\n    tgt_pts = tf.cast(tgt_pts, tf.int64)\n    \n    # The perspective transformation matrix mapping basis vectors and (1, 1, 1) to `src_pts`\n    # shape = [batch_size, 3, 3]\n    src_mat = get_transformation_mat_2D_batch(src_pts)\n    \n    # The perspective transformation matrix mapping basis vectors and (1, 1, 1) to `tgt_pts`\n    # shape = [3, 3]\n    tgt_mat = get_transformation_mat_2D_batch(tgt_pts)\n    \n    # The perspective transformation matrix mapping `src_pts` to `tgt_pts`\n    # shape = [3, 3]\n    src_to_tgt_mat = tf.linalg.matmul(tgt_mat, tf.linalg.inv(src_mat))\n    \n    return src_to_tgt_mat\n  \n    \ndef get_transformation_mat_2D_batch(four_pts):\n    \"\"\"Get the perspective transformation matrix from a space to another space, which maps the basis vectors and (1, 1, 1) to the 4 points defined by `four_pts`.\n    \n    Args:\n        four_pts: 3-D tensor of shape [batch_size, 4, 2]\n        \n    Returns:\n        3-D tensor of shape [batch_size, 3, 3]        \n    \"\"\"\n    \n    batch_size = four_pts.shape[0]\n    \n    # Change to projective coordinates by adding ones\n    # shape = [batch_size, 3, 4]\n    pts_homo = tf.transpose(tf.concat([four_pts, tf.ones(shape=[batch_size, 4, 1], dtype=tf.int64)], axis=-1), perm=[0, 2, 1])\n    \n    pts_homo = tf.cast(pts_homo, tf.float64)\n    \n    # Find `scalars` such that: src_pts_homo[:, 3:] * scalars == src_pts_homo[:, 3:]\n    # shape = [batch_size 3, 3]\n    inv_mat = tf.linalg.inv(pts_homo[:, :, :3])\n    # shape = [batch_size, 3, 1]\n    scalars = tf.linalg.matmul(inv_mat, pts_homo[:, :, 3:])\n    \n    # Get the matrix transforming unit vectors to the 4 source points\n    # shape = [batch_size, 3, 3]    \n    mat = tf.transpose(tf.transpose(pts_homo[:, :, :3], perm=[0, 2, 1]) * scalars, perm=[0, 2, 1])\n    \n    return mat\n\n\ndef batch_perspective(images, labels):\n    \n    PROBABILITY = 0.1\n    \n    # A trick to get batch_size\n    batch_size = tf.cast(tf.reduce_sum(tf.ones_like(images)) / (images.shape[1] * images.shape[2] * images.shape[3]), tf.int32)  \n    \n    # This is a tensor containing 0 or 1 -- 0: no perspective transformation.\n    # shape = [batch_size]\n    do_perspective = tf.cast(tf.random.uniform([batch_size], 0, 1) <= PROBABILITY, tf.float32)\n    \n    new_images = random_4_point_transform_2D_batch(images)\n    \n    new_images = images * (1 - do_perspective)[:, tf.newaxis, tf.newaxis, tf.newaxis] + new_images * do_perspective[:, tf.newaxis, tf.newaxis, tf.newaxis]\n    \n    return new_images, labels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 5 - A method to get oversampled training dataset\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset_with_oversample(batch_size, shuffle_size=None, repeat_dataset=True, oversample=False, augumentation=False, drop_remainder=False):\n\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n\n    if oversample:\n        dataset = dataset.flat_map(lambda image, label: tf.data.Dataset.from_tensors((image, label)).repeat(get_num_of_repetition_for_example((image, label))))\n\n    if repeat_dataset:\n        dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    \n    if shuffle_size is not None:\n        dataset = dataset.shuffle(shuffle_size)\n    \n    dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)\n    \n    if augumentation:\n            \n        # dataset = dataset.map(label_transform, num_parallel_calls=AUTO)    \n        dataset = dataset.map(basic_transform, num_parallel_calls=AUTO)   \n        # dataset = dataset.map(batch_cutmix, num_parallel_calls=AUTO)\n        # dataset = dataset.map(batch_mixup, num_parallel_calls=AUTO)\n        dataset = dataset.map(batch_perspective, num_parallel_calls=AUTO)\n    \n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 6 - Check oversampled dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"oversampled_training_dataset = get_training_dataset_with_oversample(batch_size=8, shuffle_size=None, repeat_dataset=False, oversample=True, augumentation=False)\n\nlabel_counter_2 = Counter()\nfor images, labels in oversampled_training_dataset:\n    label_counter_2.update(labels.numpy())\n\ndel oversampled_training_dataset\n\nlabel_counting_sorted_2 = label_counter_2.most_common()\n\nNUM_TRAINING_IMAGES_OVERSAMPLED = sum([x[1] for x in label_counting_sorted_2])\nprint(\"number of examples in the oversampled training dataset: {}\".format(NUM_TRAINING_IMAGES_OVERSAMPLED))\n\nprint(\"labels in the oversampled training dataset, sorted by occurrence\")\nlabel_counting_sorted_2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset visualizations"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Peek at training data\ntrain_ds = get_training_dataset_with_oversample(batch_size=8, shuffle_size=NUM_TRAINING_IMAGES_OVERSAMPLED, repeat_dataset=True, oversample=OVERSAMPLE, augumentation=AUGUMENTATION, drop_remainder=True)\ntraining_dataset = train_ds.unbatch().batch(8)\ntrain_batch = iter(training_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run this cell again for next set of images\ndisplay_batch_of_images(next(train_batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Peek at training data\ntraining_dataset = get_training_dataset_with_oversample(batch_size=8)\ntraining_dataset = training_dataset.unbatch().batch(8)\ntrain_batch = iter(training_dataset)\n\n# Used below\nfor images, labels in training_dataset:\n    dummy_images = images\n    print(dummy_images)\n    print(dummy_images.shape)\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run this cell again for next set of images\ndisplay_batch_of_images(next(train_batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# peer at test data\ntest_dataset = get_test_dataset(batch_size=8)\ntest_dataset = test_dataset.unbatch().batch(8)\ntest_batch = iter(test_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run this cell again for next set of images\ndisplay_batch_of_images(next(test_batch))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Batch Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_batch_configuration(batch_size_per_replica, batches_per_update):\n\n    with strategy.scope():\n\n        # The number of examples for which the training procedure running on a single replica will compute the gradients in order to accumulate them.\n        BATCH_SIZE_PER_REPLICA = batch_size_per_replica\n\n        # The total number of examples for which the training procedure will compute the gradients in order to accumulate them.\n        # This is also used for validation step.\n        BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n\n        # Accumulate `BATCHES_PER_UPDATE` of gradients before updating the model's parameters.\n        BATCHES_PER_UPDATE = batches_per_update\n\n        # The number of examples for which the training procedure will update the model's parameters once.\n        # This is the `effective` batch size, which will be used in tf.data.Dataset. \n        UPDATE_SIZE = BATCH_SIZE * BATCHES_PER_UPDATE\n\n        # The number of parameter updates in 1 epoch\n        UPDATES_PER_EPOCH = NUM_TRAINING_IMAGES_OVERSAMPLED // UPDATE_SIZE\n        \n        # The number of batches for a validation step.\n        VALID_BATCHES_PER_EPOCH = NUM_VALIDATION_IMAGES // BATCH_SIZE\n        \n        return BATCH_SIZE_PER_REPLICA, BATCH_SIZE, BATCHES_PER_UPDATE, UPDATE_SIZE, UPDATES_PER_EPOCH, VALID_BATCHES_PER_EPOCH","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Optimized custom training loop\nOptimized by calling the TPU less often and performing more steps per call\n"},{"metadata":{},"cell_type":"markdown","source":"## Soft Macro F1 Loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"def soft_f1_fn(labels_1_hot, prob_dist):\n\n    tp = tf.math.reduce_sum(labels_1_hot * prob_dist, axis=0)\n    fn = tf.math.reduce_sum(labels_1_hot * (1 - prob_dist), axis=0)\n    fp = tf.math.reduce_sum((1 - labels_1_hot) * prob_dist, axis=0)\n    \n    f1 = 2 * tp / (2 * tp + fn + fp + 1e-30)\n    recall = tp / (tp + fn + 1e-30)\n    precision = tp / (tp + fp + 1e-30)\n    \n    return f1, recall, precision\n\n\ndef soft_f1_from_probs(labels, prob_dist, n_classes):\n\n    labels_1_hot = tf.one_hot(labels, depth=n_classes)\n\n    return soft_f1_fn(labels_1_hot, prob_dist)\n\n\ndef soft_f1_from_probs_with_1_hot_labels(labels, prob_dist, n_classes):\n\n    labels_1_hot = labels\n    \n    return soft_f1_fn(labels_1_hot, prob_dist)\n\n\ndef soft_f1_from_logits(labels, logits, n_classes):\n\n    prob_dist = tf.math.softmax(logits, axis=-1)\n\n    return soft_f1_from_probs(labels, prob_dist, n_classes)\n\n\ndef hard_f1_from_logits(labels, logits, n_classes):\n\n    pred_labels = tf.math.argmax(logits, axis=-1)\n\n    pred_labels_1_hot = tf.one_hot(pred_labels, depth=n_classes)\n\n    return soft_f1_from_probs(labels, pred_labels_1_hot, n_classes)\n\n\ndef hard_f1_from_probs(labels, prob_dist, n_classes):\n\n    pred_labels = tf.math.argmax(prob_dist, axis=-1)\n\n    pred_labels_1_hot = tf.one_hot(pred_labels, depth=n_classes)\n\n    return soft_f1_from_probs(labels, pred_labels_1_hot, n_classes)\n\n\ndef soft_f1_loss_from_logits(labels, logits, n_classes):\n    \n    f1_scores, recalls, precisions = soft_f1_from_logits(labels, logits, n_classes)\n    f1_score = tf.math.reduce_sum(f1_scores)\n    \n    return 1 - f1_score\n\ndef soft_f1_loss_from_probs(labels, prob_dist, n_classes):\n    \n    f1_scores, recalls, precisions = soft_f1_from_probs(labels, prob_dist, n_classes)\n    f1_score = tf.math.reduce_mean(f1_scores)\n    \n    return 1 - f1_score\n\n\ndef hard_f1_from_probs_with_1_hot_labels(labels, prob_dist, n_classes):\n\n    pred_labels = tf.math.argmax(prob_dist, axis=-1)\n\n    pred_labels_1_hot = tf.one_hot(pred_labels, depth=n_classes)\n\n    return soft_f1_from_probs_with_1_hot_labels(labels, pred_labels_1_hot, n_classes)\n\n\ndef soft_f1_loss_from_probs_with_1_hot_labels(labels, prob_dist, n_classes):\n    \n    f1_scores, recalls, precisions = soft_f1_from_probs_with_1_hot_labels(labels, prob_dist, n_classes)\n    f1_score = tf.math.reduce_mean(f1_scores)\n    \n    return 1 - f1_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_model(learning_rate_scaling=1):\n\n    with strategy.scope():\n\n        weights = 'imagenet'\n        ### weights = '/kaggle/input/keras-pretrained-models/xception_weights_tf_dim_ordering_tf_kernels_notop.h5'\n        pretrained_model = backend(weights=weights, include_top=False ,input_shape=[*IMAGE_SIZE, 3])\n        pretrained_model.trainable = True  # False = transfer learning, True = fine-tuning\n\n        class Flower_Classifier(tf.keras.models.Model):\n            \n            def __init__(self, backend):\n                \n                super(Flower_Classifier, self).__init__()\n\n                self.backend = backend\n                self.pooling = tf.keras.layers.GlobalAveragePooling2D(name='flower/pooling')\n                ### self.linear = tf.keras.layers.Dense(len(CLASSES), name='linear', activation='relu')\n                self.logit = tf.keras.layers.Dense(len(CLASSES), name='logit')\n                self.prediction = tf.keras.layers.Softmax(dtype='float32', name='prediction')\n                \n            def train_call(self, images, training=False):\n                \n                embeddings = self.backend(images, training=training)\n                pooling = self.pooling(embeddings)\n                ### linear = self.linear(pooling)\n                linear = pooling\n                logit = self.logit(linear)\n                prediction = self.prediction(logit)\n                \n                return prediction, pooling\n                \n            def call(self, images, training=False):\n                \n                prediction, pooling = self.train_call(images, training=training)\n                return prediction\n                \n        model = Flower_Classifier(backend=pretrained_model)\n        \n        model(dummy_images)\n        model.summary()\n        \n        embedding_dim = model.backend(dummy_images).shape[-1]\n\n        # Instiate optimizer with learning rate schedule\n        class LRSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n            \n            def __init__(self, scaling):\n                \n                self.scaling = scaling\n            \n            def __call__(self, step):\n                \n                return self.scaling * lrfn(epoch=step // (UPDATES_PER_EPOCH))\n\n        optimizer = tf.keras.optimizers.Adam(learning_rate=LRSchedule(scaling=learning_rate_scaling))\n        optimizer_final = tf.keras.optimizers.Adam(learning_rate=LRSchedule(scaling=1))\n\n        optimizer = mixed_precision.LossScaleOptimizer(optimizer, loss_scale='dynamic')\n        optimizer_final = mixed_precision.LossScaleOptimizer(optimizer_final, loss_scale='dynamic')        \n        \n        # Instantiate metrics\n        train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()\n        valid_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()\n        \n        train_loss = tf.keras.metrics.Sum()\n        valid_loss = tf.keras.metrics.Sum()\n        \n        train_focal_loss = tf.keras.metrics.Sum()\n        valid_focal_loss = tf.keras.metrics.Sum()\n        \n        train_hard_f1 = tf.keras.metrics.Sum()\n        train_hard_recall = tf.keras.metrics.Sum()\n        train_hard_precision = tf.keras.metrics.Sum()\n        \n        train_soft_f1 = tf.keras.metrics.Sum()\n        train_soft_recall = tf.keras.metrics.Sum()\n        train_soft_precision = tf.keras.metrics.Sum()        \n        \n        train_soft_f1_loss = tf.keras.metrics.Sum()\n        \n        # Loss\n        # The recommendation from the Tensorflow custom training loop documentation is:\n        # loss_fn = lambda a,b: tf.nn.compute_average_loss(tf.keras.losses.sparse_categorical_crossentropy(a,b), global_batch_size=BATCH_SIZE)\n        # https://www.tensorflow.org/tutorials/distribute/custom_training#define_the_loss_function\n        # This works too and shifts all the averaging to the training loop which is easier:\n        \n        loss_fn = tf.keras.losses.categorical_crossentropy\n        \n        loss_fn_focal = tfa.losses.sigmoid_focal_crossentropy\n        \n        loss_fn_sparse = tf.keras.losses.sparse_categorical_crossentropy\n        \n        return model, loss_fn_sparse, loss_fn, loss_fn_focal, optimizer, train_accuracy, train_loss, valid_accuracy, valid_loss, train_hard_f1, train_hard_recall, train_hard_precision, train_soft_f1, train_soft_recall, train_soft_precision, train_soft_f1_loss, optimizer_final, embedding_dim, train_focal_loss, valid_focal_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training routines"},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_routines():\n\n    with strategy.scope():\n        \n        def train_step_1_forward(images, labels):\n            \n            probabilities, pooling = model.train_call(images, training=True)\n            loss = loss_fn_sparse(labels, probabilities)\n            loss_value = tf.math.reduce_sum(loss)\n            \n            focal_loss = loss_fn_focal(tf.one_hot(labels, len(CLASSES)), probabilities)\n            focal_loss_value = tf.math.reduce_sum(focal_loss)\n            \n            loss_scaled = loss_value / UPDATE_SIZE\n            focal_loss_scaled = focal_loss_value / UPDATE_SIZE\n                        \n            train_focal_loss.update_state(focal_loss_value)            \n            \n            loss_scaled += focal_loss_scaled\n            \n            return loss_scaled, pooling, probabilities, loss_value\n\n        def train_step_1_forward_backward(images, labels, epoch):\n\n            if epoch >= EPOCH_START_TRAIN_ALL and epoch <= EPOCH_END_TRAIN_ALL:\n            \n                with tf.GradientTape() as tape:\n                    loss_scaled, pooling, probabilities, loss_value = train_step_1_forward(images, labels)\n                    loss_dynamically_scaled = optimizer.get_scaled_loss(loss_scaled)\n                dynamically_scaled_gradients = tape.gradient(loss_dynamically_scaled, model.trainable_variables)   \n                grads = optimizer.get_unscaled_gradients(dynamically_scaled_gradients) \n        \n            else:\n                loss_scaled, pooling, probabilities, loss_value = train_step_1_forward(images, labels)\n                grads = [tf.zeros_like(var, dtype=tf.float32) for var in model.trainable_variables]\n\n            # update metrics\n            train_accuracy.update_state(labels, probabilities)\n            train_loss.update_state(loss_value)                \n                \n            return grads, pooling\n\n        def train_step_1_update(batch, epoch):\n            \"\"\"\n            \"\"\"\n\n            images, labels = batch\n            \n            accumulated_grads = [tf.zeros_like(var, dtype=tf.float32) for var in model.trainable_variables]\n            \n            # shape = [BATCH_SIZE_PER_REPLICA * BATCHES_PER_UPDATE, 2048]\n            total_pooling = tf.constant(0.0, shape=(BATCH_SIZE_PER_REPLICA * BATCHES_PER_UPDATE, embedding_dim))\n                                \n            for batch_idx in tf.range(BATCHES_PER_UPDATE):\n\n                # Take the 1st `BATCH_SIZE_PER_REPLICA` examples.\n                small_images = images[:BATCH_SIZE_PER_REPLICA]\n                small_labels = labels[:BATCH_SIZE_PER_REPLICA]      \n                \n                grads, pooling = train_step_1_forward_backward(small_images, small_labels, epoch)\n\n                if epoch >= EPOCH_START_TRAIN_ALL and epoch <= EPOCH_END_TRAIN_ALL:\n                    accumulated_grads = [x + y for x, y in zip(accumulated_grads, grads)]\n\n                # Move the leading part to the end, so the shape is not changed.\n                images = tf.concat([images[BATCH_SIZE_PER_REPLICA:], small_images], axis=0)\n                labels = tf.concat([labels[BATCH_SIZE_PER_REPLICA:], small_labels], axis=0)\n                                \n                total_pooling = tf.concat([total_pooling[BATCH_SIZE_PER_REPLICA:], tf.cast(pooling, tf.float32)], axis=0)\n                \n            with tf.GradientTape(watch_accessed_variables=False) as tape:\n                    \n                tape.watch(final_variables)\n                    \n                ### linear = model.linear(total_pooling)\n                linear = total_pooling\n                logit = model.logit(linear)\n                \n                # shape = [BATCH_SIZE_PER_REPLICA * BATCHES_PER_UPDATE, len(CLASSES)]\n                probabilities = model.prediction(logit)\n\n                soft_f1_loss = soft_f1_loss_from_probs(labels, probabilities, len(CLASSES))\n                soft_f1_loss_scaled = soft_f1_loss / strategy.num_replicas_in_sync\n                \n                soft_f1_loss_dynamically_scaled = optimizer_final.get_scaled_loss(soft_f1_loss_scaled)\n\n            dynamically_scaled_gradients = tape.gradient(soft_f1_loss_dynamically_scaled, final_variables)   \n            final_grads = optimizer_final.get_unscaled_gradients(dynamically_scaled_gradients)  \n            \n            # hard_f1, hard_recall, hard_precision = hard_f1_from_probs_with_1_hot_labels(labels, probabilities, len(CLASSES))\n            hard_f1, hard_recall, hard_precision = hard_f1_from_probs(labels, probabilities, len(CLASSES))\n            \n            hard_f1 = tf.reduce_mean(hard_f1)\n            hard_recall = tf.reduce_mean(hard_recall)\n            hard_precision = tf.reduce_mean(hard_precision)\n            \n            train_hard_f1.update_state(hard_f1)\n            train_hard_recall.update_state(hard_recall)\n            train_hard_precision.update_state(hard_precision)            \n            \n            # soft_f1, soft_recall, soft_precision = soft_f1_from_probs_with_1_hot_labels(labels, probabilities, len(CLASSES))\n            soft_f1, soft_recall, soft_precision = soft_f1_from_probs(labels, probabilities, len(CLASSES))\n            \n            soft_f1 = tf.reduce_mean(soft_f1)\n            soft_recall = tf.reduce_mean(soft_recall)\n            soft_precision = tf.reduce_mean(soft_precision)            \n            \n            train_soft_f1.update_state(soft_f1)\n            train_soft_recall.update_state(soft_recall)\n            train_soft_precision.update_state(soft_precision)\n            \n            train_soft_f1_loss.update_state(soft_f1_loss_scaled)\n            \n            # Update the model's parameters.\n            \n            if epoch >= EPOCH_START_TRAIN_ALL and epoch <= EPOCH_END_TRAIN_ALL:\n                optimizer.apply_gradients(zip(accumulated_grads, model.trainable_variables))\n        \n            optimizer_final.apply_gradients(zip(final_grads, final_variables))\n\n            \n        @tf.function\n        def train_step_1_epoch(data_iter, epoch):\n\n            for _ in tf.range(UPDATES_PER_EPOCH):  \n                strategy.experimental_run_v2(train_step_1_update, args=(next(data_iter), epoch))\n                \n                \n        @tf.function\n        def valid_step(data_iter):\n            \n            def valid_step_fn(images, labels):\n                \n                probabilities, pooling = model.train_call(images, training=False)\n                loss = tf.math.reduce_sum(loss_fn_sparse(labels, probabilities))\n                focal_loss = tf.math.reduce_sum(loss_fn_focal(tf.one_hot(labels, len(CLASSES)), probabilities))                \n                \n                loss_scaled = loss / UPDATE_SIZE\n                focal_loss_scaled = focal_loss / UPDATE_SIZE\n                \n                # update metrics\n                valid_accuracy.update_state(labels, probabilities)\n                \n                valid_loss.update_state(loss)\n                valid_focal_loss.update_state(focal_loss)\n\n            for _ in tf.range(VALID_BATCHES_PER_EPOCH):\n                strategy.experimental_run_v2(valid_step_fn, next(data_iter))                \n                \n    return train_step_1_epoch, valid_step","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training with gradient accumulation with even higer effective batch size + larger learning rate"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE_PER_REPLICA, BATCH_SIZE, BATCHES_PER_UPDATE, UPDATE_SIZE, UPDATES_PER_EPOCH, VALID_BATCHES_PER_EPOCH = set_batch_configuration(batch_size_per_replica=8, batches_per_update=64)\nmodel, loss_fn_sparse, loss_fn, loss_fn_focal, optimizer, train_accuracy, train_loss, valid_accuracy, valid_loss, train_hard_f1, train_hard_recall, train_hard_precision, train_soft_f1, train_soft_recall, train_soft_precision, train_soft_f1_loss, optimizer_final, embedding_dim, train_focal_loss, valid_focal_loss = set_model(learning_rate_scaling=16)\ntrain_step_1_epoch, valid_step = set_routines()\n\nfinal_variables = []\n\nfor layer in model.layers:\n    if layer.name in ['logit', 'linear']:\n        for variable in layer.variables:\n            final_variables.append(variable)\n\nprint(\"BATCH_SIZE_PER_REPLICA: {}\".format(BATCH_SIZE_PER_REPLICA))\nprint(\"BATCH_SIZE: {}\".format(BATCH_SIZE))\nprint(\"BATCHES_PER_UPDATE: {}\".format(BATCHES_PER_UPDATE))\nprint(\"UPDATE_SIZE: {}\".format(UPDATE_SIZE))\nprint(\"UPDATES_PER_EPOCH: {}\".format(UPDATES_PER_EPOCH))\nprint(\"VALID_BATCHES_PER_EPOCH: {}\".format(VALID_BATCHES_PER_EPOCH))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = get_training_dataset_with_oversample(batch_size=UPDATE_SIZE, shuffle_size=NUM_TRAINING_IMAGES_OVERSAMPLED, repeat_dataset=True, oversample=OVERSAMPLE, augumentation=AUGUMENTATION, drop_remainder=True)\ntrain_dist_ds = strategy.experimental_distribute_dataset(train_ds)\ntrain_data_iter = iter(train_dist_ds)\n\n# valid_ds = get_validation_dataset(batch_size=BATCH_SIZE, repeated=True)\n# valid_dist_ds = strategy.experimental_distribute_dataset(valid_ds)\n# valid_data_iter = iter(valid_dist_ds)\n\n# valid_ds_2 = get_validation_dataset(batch_size=BATCH_SIZE, ordered=True)\n# valid_images_ds = valid_ds_2.map(lambda image, label: image)\n# valid_labels_ds = valid_ds_2.map(lambda image, label: label).unbatch()\n# valid_labels = next(iter(valid_labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy().astype('U') # all in one batch\n# valid_labels = tf.convert_to_tensor(valid_labels, dtype=tf.int32)\n\ntest_ds = get_test_dataset(batch_size=BATCH_SIZE, ordered=True)\ntest_images_ds = test_ds.map(lambda image, idnum: image)\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\n\n# valid_steps = NUM_VALIDATION_IMAGES // BATCH_SIZE\n# if NUM_VALIDATION_IMAGES % BATCH_SIZE > 0:\n#     valid_steps += 1\n\ntest_steps = NUM_TEST_IMAGES // BATCH_SIZE\nif NUM_TEST_IMAGES % BATCH_SIZE > 0:\n    test_steps += 1\n\nfor epoch_idx in range(EPOCHS):\n    \n    s = datetime.datetime.now()\n    \n    epoch = tf.constant(epoch_idx + 1, dtype=tf.int32)\n    \n    train_step_1_epoch(train_data_iter, epoch)\n    \n    loss = train_loss.result() / (UPDATES_PER_EPOCH * UPDATE_SIZE)\n    acc = train_accuracy.result()\n    \n    focal_loss = train_focal_loss.result() / (UPDATES_PER_EPOCH * UPDATE_SIZE)\n    \n    hard_f1 = train_hard_f1.result() / (UPDATES_PER_EPOCH * strategy.num_replicas_in_sync)\n    hard_recall = train_hard_recall.result() / (UPDATES_PER_EPOCH * strategy.num_replicas_in_sync)\n    hard_precision = train_hard_precision.result() / (UPDATES_PER_EPOCH * strategy.num_replicas_in_sync)\n    \n    soft_f1 = train_soft_f1.result() / (UPDATES_PER_EPOCH * strategy.num_replicas_in_sync)\n    soft_recall = train_soft_recall.result() / (UPDATES_PER_EPOCH * strategy.num_replicas_in_sync)\n    soft_precision = train_soft_precision.result() / (UPDATES_PER_EPOCH * strategy.num_replicas_in_sync)\n    \n    soft_f1_loss = train_soft_f1_loss.result() / UPDATES_PER_EPOCH\n    \n    print(\"epoch: {}\".format(epoch_idx + 1))\n\n    print(\"train loss: {}\".format(loss))\n    print(\"train accuracy: {}\".format(acc))\n    \n    print(\"train focal loss: {}\".format(focal_loss))    \n    \n    print(\"train hard f1: {}\".format(hard_f1))\n    print(\"train hard recall: {}\".format(hard_recall))\n    print(\"train hard precision: {}\".format(hard_precision))    \n    \n    print(\"train soft f1: {}\".format(soft_f1))\n    print(\"train soft recall: {}\".format(soft_recall))\n    print(\"train soft precision: {}\".format(soft_precision))        \n    \n    print(\"train soft f1 loss: {}\".format(soft_f1_loss))\n    \n    train_loss.reset_states()\n    train_accuracy.reset_states()\n    \n    train_focal_loss.reset_states()\n    \n    train_hard_f1.reset_states()\n    train_hard_recall.reset_states()\n    train_hard_precision.reset_states()    \n    \n    train_soft_f1.reset_states()\n    train_soft_recall.reset_states()\n    train_soft_precision.reset_states()     \n    \n    train_soft_f1_loss.reset_states()\n\n    e = datetime.datetime.now()\n    print(\"elapsed: {}\".format((e-s).total_seconds()))\n    \n#     valid_step(valid_data_iter)\n    \n#     val_loss = valid_loss.result() / (VALID_BATCHES_PER_EPOCH * BATCH_SIZE)\n#     val_acc = valid_accuracy.result()    \n    \n#     val_focal_loss = valid_focal_loss.result() / (VALID_BATCHES_PER_EPOCH * BATCH_SIZE)\n\n#     print(\"valid loss: {}\".format(val_loss))\n#     print(\"valid accuracy: {}\".format(val_acc))\n    \n#     print(\"valid focal loss: {}\".format(val_focal_loss))\n     \n#     valid_loss.reset_states()\n#     valid_accuracy.reset_states()\n    \n#     valid_focal_loss.reset_states()\n\n#     valid_probs = model.predict(valid_images_ds, steps=valid_steps)\n#     valid_preds = np.argmax(valid_probs, axis=-1)\n\n#     valid_hard_f1, valid_hard_recall, valid_hard_precision = hard_f1_from_probs(valid_labels, valid_probs, len(CLASSES))\n    \n#     valid_hard_f1 = tf.reduce_mean(valid_hard_f1)\n#     valid_hard_recall = tf.reduce_mean(valid_hard_recall)\n#     valid_hard_precision = tf.reduce_mean(valid_hard_precision)    \n    \n#     valid_soft_f1, valid_soft_recall, valid_soft_precision = soft_f1_from_probs(valid_labels, valid_probs, len(CLASSES))    \n    \n#     valid_soft_f1 = tf.reduce_mean(valid_soft_f1)\n#     valid_soft_recall = tf.reduce_mean(valid_soft_recall)\n#     valid_soft_precision = tf.reduce_mean(valid_soft_precision)\n    \n#     valid_soft_f1_loss = 1 - valid_soft_f1\n    \n#     print(\"valid hard f1: {}\".format(valid_hard_f1))\n#     print(\"valid hard recall: {}\".format(valid_hard_recall))\n#     print(\"valid hard precision: {}\".format(valid_hard_precision))    \n    \n#     print(\"valid soft f1: {}\".format(valid_soft_f1))\n#     print(\"valid soft recall: {}\".format(valid_soft_recall))\n#     print(\"valid soft precision: {}\".format(valid_soft_precision))    \n    \n#     print(\"valid soft f1 loss: {}\".format(valid_soft_f1_loss))\n    \n    if (epoch_idx + 1) >= EPOCH_SAVING_START:\n        \n        ### model.save_weights(\"{}_epoch_{}.ckpt\".format(backend_name, epoch_idx + 1))\n\n        test_probs = model.predict(test_images_ds, steps=test_steps)\n        test_preds = np.argmax(test_probs, axis=-1)\n\n#         print('Generating valid_probs.txt file...')\n#         with open('valid_probs_epoch_{}.txt'.format(epoch_idx + 1), \"w\", encoding=\"UTF-8\") as fp:\n#             np.savetxt(fp, valid_probs, delimiter=',', fmt='%10.6f')\n\n        print('Generating test_probs.txt file...')\n        with open('test_probs_epoch_{}.txt'.format(epoch_idx + 1), \"w\", encoding=\"UTF-8\") as fp:\n            np.savetxt(fp, test_probs, delimiter=',', fmt='%10.6f')\n\n#         print('Generating valid_preds.csv file...')\n#         np.savetxt('valid_preds_epoch_{}.csv'.format(epoch_idx + 1), np.rec.fromarrays([valid_labels, valid_preds]), fmt=['%d', '%d'], delimiter=',', header='label,pred', comments='')\n\n        print('Generating submission.csv file...')\n        np.savetxt('submission_epoch_{}.csv'.format(epoch_idx + 1), np.rec.fromarrays([test_ids, test_preds]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')    \n    \n    print(\"-\" * 80)\n    \ndel optimizer\ndel model\ndel train_step_1_epoch\ngc.collect()\ntf.keras.backend.clear_session()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}