{"cells":[{"metadata":{},"cell_type":"markdown","source":"# About this kernel\n\nPretty much a fork of the very comprehensive [starter kernel](https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu) and the other starting kernels shared on Kaggle.\n\n## Updates\n\n* V4: Renamed and reoganized to run both locally and on Kaggle, fixed errors with image augmentation, initial training without image augmentation then train with image augmentation\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import math, re, os\nimport tensorflow as tf\nimport numpy as np\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets  # Cpmment this out when running locally\n#import efficientnet.tfkeras as efn\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\n\nprint(\"Tensorflow version \" + tf.__version__)\n\n# The Keras library provides support for neural networks and deep learning\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Input, Dense, Dropout, Activation, Lambda, Flatten, LSTM, SpatialDropout2D\nfrom tensorflow.keras.layers import Conv2D, Convolution2D, MaxPooling2D, Flatten, AveragePooling2D, GlobalAveragePooling2D\nfrom tensorflow.keras.optimizers import Adam, RMSprop\n#from tensorflow.keras.utils import np_utils\nfrom tensorflow.keras import utils\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint, Callback","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TPU or GPU detection"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Competition data access\nTPUs read data directly from Google Cloud Storage (GCS). This Kaggle utility will copy the dataset to a GCS bucket co-located with the TPU. If you have multiple datasets attached to the notebook, you can pass the name of a specific dataset to the get_gcs_path function. The name of the dataset is the name of the directory it is mounted in. Use `!ls /kaggle/input/` to list attached datasets."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data access for on Kaggle\nGCS_DS_PATH = KaggleDatasets().get_gcs_path()  # you can list the bucket with \"!gsutil ls $GCS_DS_PATH\"\n# Data access for on local machine\n#GCS_DS_PATH = \"data/\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nIMAGE_SIZE = [512, 512] # At this size, a GPU will run out of memory. Use the TPU.\n#IMAGE_SIZE = [224, 224] # For GPU training, please select 224 x 224 px image size or you will likely run out of RAM.\n\nEPOCHS = 15           # This is the training time. Can be set long if you use EarlyStopping, but be careful you too much TPU time\n#BATCH_SIZE = 32 * strategy.num_replicas_in_sync\n#BATCH_SIZE = 32       # Good for a single CPU or GPU\n\nBATCH_SIZE = 128     # 128 is Good for a TPU with multiple processeors\n\nprint (\"EPOCHS = \", EPOCHS)\nprint (\"BATCH_SIZE = \", BATCH_SIZE)\n\nGCS_PATH_SELECT = { # available image sizes\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec') # predictions on this dataset should be submitted for the competition\n\nCLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset functions"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Modify data_augment to change how training images are adjusted during training\n# See tf.image for documentation -- https://www.tensorflow.org/api_docs/python/tf/image\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    \n    image = tf.image.random_flip_left_right(image)\n#     image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_brightness(image, 0.1)\n    image = tf.image.random_contrast(image, 0.9, 1.0)\n#    image = tf.image.random_hue(image, 0.1)\n#    image = tf.image.random_contrast(image, 0.1)\n    image = tf.image.random_saturation(image, 0.9, 1.0)\n    \n#     image = tf.image.random_jpeg_quality(image, 85, 100)\n    width = IMAGE_SIZE[0]\n    large_width = math.floor(width * 1.2)  # increase images sizes by 10% before random crop\n    print (\"image width = \", width, \" resized to \", large_width)\n    image = tf.image.resize(image, [large_width, large_width])\n    image = tf.image.random_crop(image, [width, width, 3])\n    #image = tf.image.random_saturation(image, 0, 2)\n    image = tf.clip_by_value(image, clip_value_min=0., clip_value_max=1.)\n    return image, label   \n\n\n# ====== You should not have to edit any functions below here\n\ndef data_augment_flip_only(image, label):\n    image = tf.image.random_flip_left_right(image)\n    return image, label   \n\nAUTO = tf.data.experimental.AUTOTUNE\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\n# The class function is used by the balance_dataset to get the image id from each image\ndef class_func(image, id):\n    return id\n\n# Since there are many more images of some flowers than others, this tries to balance the number of each type of flower in the dataset\ndef balance_dataset(dataset):\n    NUM_CLASSES = len(CLASSES)\n    PROB = 1 / NUM_CLASSES\n    TARGET_DIST = [PROB] * NUM_CLASSES\n    print (\"balancing the dataset with rejection_resample with distribution = \",TARGET_DIST)\n    # TODO --- add code to calculate the number of images in each flower class and generate the initial_dist\n    #resampler = tf.data.experimental.rejection_resample(class_func, target_dist=TARGET_DIST, initial_dist=COUNT\n    resampler = tf.data.experimental.rejection_resample(class_func, target_dist=TARGET_DIST)\n    dataset = dataset.apply(resampler)\n    return dataset\n\ndef get_training_dataset(augment=False, balance=False):\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    if augment:\n        print (\"augmenting images in dataset\")\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    else:\n        dataset = dataset.map(data_augment_flip_only, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    if balance:\n        print(\"balancing the dataset\")\n        #dataset = balance_dataset(dataset) \n        NUM_CLASSES = len(CLASSES)\n        PROB = 1 / NUM_CLASSES\n        TARGET_DIST = [PROB] * NUM_CLASSES\n        # TODO --- add code to calculate the number of images in each flower class and generate the initial_dist\n        #resampler = tf.data.experimental.rejection_resample(class_func, target_dist=TARGET_DIST, initial_dist=COUNT\n        resampler = tf.data.experimental.rejection_resample(class_func, target_dist=TARGET_DIST)\n        #dataset = dataset.unbatch()\n        resample_ds  = dataset.apply(resampler)\n        #dataset = dataset.apply(resampler)\n        dataset = resample_ds.map(lambda extra_label, image_and_id: image_and_id)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_train_valid_datasets(augment=False, balance=False):\n    dataset = load_dataset(TRAINING_FILENAMES + VALIDATION_FILENAMES, labeled=True)\n    if augment:\n        print (\"augmenting images in dataset\")\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    else:\n        dataset = dataset.map(data_augment_flip_only, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    if balance:\n        print(\"balancing the dataset\")\n        #dataset = balance_dataset(dataset) \n        NUM_CLASSES = len(CLASSES)\n        PROB = 1 / NUM_CLASSES\n        TARGET_DIST = [PROB] * NUM_CLASSES\n        # TODO --- add code to calculate the number of images in each flower class and generate the initial_dist\n        #resampler = tf.data.experimental.rejection_resample(class_func, target_dist=TARGET_DIST, initial_dist=COUNT\n        resampler = tf.data.experimental.rejection_resample(class_func, target_dist=TARGET_DIST)\n        #dataset = dataset.unbatch()\n        resample_ds  = dataset.apply(resampler)\n        #dataset = dataset.apply(resampler)\n        dataset = resample_ds.map(lambda extra_label, image_and_id: image_and_id)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualization utilities\ndata -> pixels, nothing of much interest for the machine learning practitioner in this section."},{"metadata":{"trusted":true},"cell_type":"code","source":"# numpy and matplotlib defaults\nnp.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n    \ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n\n    \ndef display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()\n    \ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])\n\n# This is a callback to display the learning graphs during training\n# I don't know the original author of this method, but is is used frequently in Kaggle competitions and on GitHub\nclass PlotLearning(Callback):\n    def on_train_begin(self, logs={}):\n        self.i = 0\n        self.x = []\n        self.losses = []\n        self.val_losses = []\n        self.acc = []\n        self.val_acc = []\n        self.fig = plt.figure() \n        self.logs = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        self.logs.append(logs)\n        self.x.append(self.i)\n        self.losses.append(logs.get('loss'))\n        self.val_losses.append(logs.get('val_loss'))\n        self.acc.append(logs.get('sparse_categorical_accuracy'))\n        self.val_acc.append(logs.get('val_sparse_categorical_accuracy'))\n        self.i += 1\n        f, (ax1, ax2) = plt.subplots(1, 2, sharex=True)\n        #clear_output(wait=True)\n        ax1.set_yscale('log')\n        ax1.plot(self.x, self.losses, label=\"loss\")\n        ax1.plot(self.x, self.val_losses, label=\"val_loss\")\n        ax1.legend()\n        ax2.plot(self.x, self.acc, label=\"accuracy\")\n        ax2.plot(self.x, self.val_acc, label=\"validation accuracy\")\n        ax2.legend()\n        plt.show();\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset visualizations"},{"metadata":{"trusted":true},"cell_type":"code","source":"# data dump\nprint(\"Training data shapes:\")\nfor image, label in get_training_dataset().take(3):\n    print(image.numpy().shape, label.numpy().shape)\nprint(\"Training data label examples:\", label.numpy())\nprint(\"Validation data shapes:\")\nfor image, label in get_validation_dataset().take(3):\n    print(image.numpy().shape, label.numpy().shape)\nprint(\"Validation data label examples:\", label.numpy())\nprint(\"Test data shapes:\")\nfor image, idnum in get_test_dataset().take(3):\n    print(image.numpy().shape, idnum.numpy().shape)\nprint(\"Test data IDs:\", idnum.numpy().astype('U')) # U=unicode string","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Peek at training data without image augmentation\nNUM_IMAGES_TO_DISPLAY = BATCH_SIZE  # can only display images up to the batch size\ntraining_dataset = get_training_dataset()\ntraining_dataset = training_dataset.unbatch().batch(NUM_IMAGES_TO_DISPLAY)\ntrain_batch = iter(training_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run this cell again for next set of images\n# These images relfect the origina unaugmentated images\nprint (\"images without augmentation\")\ndisplay_batch_of_images(next(train_batch))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Peek at training data with image augmentation\nNUM_IMAGES_TO_DISPLAY = BATCH_SIZE  # can only display images up to the batch size\ntraining_dataset_augment = get_training_dataset(augment=True)\ntraining_dataset_augment = training_dataset_augment.unbatch().batch(NUM_IMAGES_TO_DISPLAY)\ntrain_batch_augment = iter(training_dataset_augment)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run this cell again for next set of images\n# These images relfect the image augmentation setting above\nprint (\"images with augmentation\")\ndisplay_batch_of_images(next(train_batch_augment))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run this cell again for next set of images\ndisplay_batch_of_images(next(train_batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run this cell again for next set of images\ndisplay_batch_of_images(next(train_batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# peer at test data\ntest_dataset = get_test_dataset()\ntest_dataset = test_dataset.unbatch().batch(20)\ntest_batch = iter(test_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run this cell again for next set of images\ndisplay_batch_of_images(next(test_batch))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model\nNot the best but it converges ..."},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n\n    INPUT_SIZE = [*IMAGE_SIZE, 3]\n    OUTPUT_SIZE = len(CLASSES)\n    print (\"INPUT_SIZE = \", INPUT_SIZE)\n    print (\"OUTPUT_SIZE = \", OUTPUT_SIZE)\n    \n    #pretrained_model = tf.keras.applications.DenseNet201(weights='imagenet', include_top=False ,input_shape=INPUT_SIZE)\n\n    # by default Xception expects images of size 299x299 pixels\n    pretrained_model = tf.keras.applications.Xception(weights='imagenet', include_top=False ,input_shape=INPUT_SIZE)\n\n    # VGG works with 224x224 size images\n    #pretrained_model = tf.keras.applications.VGG16(weights='imagenet', include_top=False ,input_shape=INPUT_SIZE)\n\n    pretrained_model.trainable = True      # False = transfer learning, True = fine-tuning\n \n    \n    model = tf.keras.Sequential([\n        pretrained_model,                                 # Include layers in pretrained model from above\n        tf.keras.layers.GlobalAveragePooling2D(),\n        #tf.keras.layers.Dense(1024, activation=\"relu\"),  # Can add additional layers here\n        #tf.keras.layers.Dense(200, activation=\"relu\"),  # Can add additional layers here\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n    ])\n    \n# Some sample weight optimizer settings\n#RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0)\noptimizer_RMSprop = RMSprop(lr=0.00001, epsilon=1e-08)\n#Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\n#optimizer_Adam = Adam(learning_rate=0.001) # default learning rate\noptimizer_Adam = Adam(learning_rate=0.0001)\n#optimizer_SGD = tf.keras.optimizers.SGD(lr=0.01, \n#                              decay=1e-6, \n#                              momentum=0.9, \n#                              nesterov=True)\n\nmodel.compile(\n#    optimizer=optimizer_Adam,\n    optimizer=optimizer_RMSprop,\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)\n\nprint (\"=== Pretrained Model =========================================================================\")\npretrained_model.summary()   # print layers in pretrained model\nprint (\"=== Final Model =========================================================================\")\nmodel.summary()              # print final model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # option to create your own model from scratch\n\n# with strategy.scope():\n  \n#     model = Sequential()\n#     model.add(Conv2D(64, kernel_size=(3,3), activation=\"relu\", input_shape=[*IMAGE_SIZE, 3]))\n#     model.add(Conv2D(64, kernel_size=(3,3), activation=\"relu\", padding='same'))\n#     model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))\n\n\n#     model.add(Conv2D(128, kernel_size=(3,3), activation=\"relu\", padding='same'))\n#     model.add(Conv2D(128, kernel_size=(3,3), activation=\"relu\", padding='same'))\n#     model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))\n\n#     model.add(Conv2D(256, kernel_size=(3,3), activation=\"relu\", padding='same'))\n#     model.add(Conv2D(256, kernel_size=(3,3), activation=\"relu\", padding='same'))\n#     model.add(Conv2D(256, kernel_size=(3,3), activation=\"relu\", padding='same'))\n#     model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))\n\n#     model.add(Conv2D(512, kernel_size=(3,3), activation=\"relu\", padding='same'))\n#     model.add(Conv2D(512, kernel_size=(3,3), activation=\"relu\", padding='same'))\n#     model.add(Conv2D(512, kernel_size=(3,3), activation=\"relu\", padding='same'))\n#     model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))\n\n#     #model.add(Conv2D(512, kernel_size=(3,3), activation=\"relu\", padding='same'))\n#     #model.add(Conv2D(512, kernel_size=(3,3), activation=\"relu\", padding='same'))\n#     #model.add(Conv2D(512, kernel_size=(3,3), activation=\"relu\", padding='same'))\n#     #model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))\n\n#     model.add(GlobalAveragePooling2D())\n#     model.add(Dense(1024, activation=\"relu\"))\n#     model.add(Dense(1024, activation=\"relu\"))\n#     model.add(Dense(numClasses, activation=\"softmax\"))\n        \n#     model.compile(\n#         optimizer='adam',\n#         loss = 'sparse_categorical_crossentropy',\n#         metrics=['sparse_categorical_accuracy']\n#     )\n#     model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# from keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint\n\n# Modify the patience to change how quickly the learning rate is reduced by the factor\nlearning_rate_reduction = ReduceLROnPlateau(monitor='loss', \n                                            patience=2, \n                                            verbose=2, \n                                            factor=0.5,                                            \n                                            min_lr=0.0000001)\n\n# Modify the patience to change how quickly the training is stopped once the loss is not dereasing\nearly_stops = EarlyStopping(monitor='loss', \n                            min_delta=0, \n                            patience=3, \n                            verbose=2, \n                            mode='auto')\n\n# Save the best models\ncheckpointer = ModelCheckpoint(filepath = 'FowersGPU1.{epoch:02d}-{accuracy:.6f}.hdf5',\n                               verbose=2,\n                               save_best_only=True, \n                               save_weights_only = True)\n\n# This is the learn rate function used in the original notebook. Works well also\ndef lrfn(epoch):\n    LR_START = 0.00001\n    #LR_MAX = 0.00005 * strategy.num_replicas_in_sync\n    LR_MAX = 0.00005\n    LR_MIN = 0.00001\n    LR_RAMPUP_EPOCHS = 5\n    LR_SUSTAIN_EPOCHS = 0\n    LR_EXP_DECAY = .8\n    \n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n\ndef freeze(model):\n    for layer in model.layers:\n        layer.trainable = False\n\ndef unfreeze(model):\n    for layer in model.layers:\n        layer.trainable = True\n\nlr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)\n# Learning rate schedule graph\nlrfn_rng = [i for i in range(25 if EPOCHS<25 else EPOCHS)]\nlrfn_y = [lrfn(x) for x in lrfn_rng]\na = plt.plot(lrfn_rng, lrfn_y)\n\n\n       \nplot_losses = PlotLearning()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print (\"BATCH_SIZE =  \", BATCH_SIZE)\nprint (\"STEPS_PER_EPOCH =  \", STEPS_PER_EPOCH)\nprint (\"EPOCHS =  \", EPOCHS)\nprint (\"initial training W/O image augmentation\")\n\nhistory = model.fit(get_train_valid_datasets(augment=False, balance=False),\n#history = model.fit(get_training_dataset(augment=False, balance=False),\n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    epochs=EPOCHS, \n                    callbacks=[learning_rate_reduction, early_stops, plot_losses],\n                    #callbacks=[early_stopping, reduce_lr, checkpoint, plot_losses],\n                    #callbacks=[lr_schedule],\n                    validation_data=get_validation_dataset()\n                   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_training_curves(history.history['loss'], history.history['val_loss'], 'loss', 211)\ndisplay_training_curves(history.history['sparse_categorical_accuracy'], history.history['val_sparse_categorical_accuracy'], 'accuracy', 212)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.optimizer.lr = 0.000005\n\nprint (\"STEPS_PER_EPOCH =  \", STEPS_PER_EPOCH)\nprint (\"EPOCHS =  \", EPOCHS)\nprint (\"initial training WITH image augmentation\")\n\nhistory = model.fit(get_train_valid_datasets(augment=True, balance=True), \n#history = model.fit(get_training_dataset(augment=True, balance=True), \n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    epochs=EPOCHS, \n                    callbacks=[learning_rate_reduction, early_stops, plot_losses],\n                    #callbacks=[early_stopping, reduce_lr, checkpoint, plot_losses],\n                    #callbacks=[lr_schedule],\n                    validation_data=get_validation_dataset()\n                   )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax_loss, ax_acc) = plt.subplots(1, 2, figsize=(15,5))\nax_loss.plot(history.epoch, history.history[\"loss\"], label=\"Train loss\")\nax_loss.plot(history.epoch, history.history[\"val_loss\"], label=\"Validation loss\")\nax_acc.plot(history.epoch, history.history[\"sparse_categorical_accuracy\"], label=\"Train accuracy\")\nax_acc.plot(history.epoch, history.history[\"val_sparse_categorical_accuracy\"], label=\"Validation accuracy\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Confusion matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"cmdataset = get_validation_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and labels, order matters.\nimages_ds = cmdataset.map(lambda image, label: image)\nlabels_ds = cmdataset.map(lambda image, label: label).unbatch()\ncm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy() # get everything as one batch\ncm_probabilities = model.predict(images_ds)\ncm_predictions = np.argmax(cm_probabilities, axis=-1)\nprint(\"Correct   labels: \", cm_correct_labels.shape, cm_correct_labels)\nprint(\"Predicted labels: \", cm_predictions.shape, cm_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wrongList = []\nfor correct,predict in zip(cm_correct_labels,cm_predictions):\n    if correct != predict:\n        #print(\"Correct flower: \", correct, \" Predicted flower: \", predict)\n        print(correct,\", \", CLASSES[correct],\", \", predict, \", \", CLASSES[predict] )\n        wrongList.append([correct,predict])\n#print (sorted(wrongList))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for img,correct,predict in zip(images_ds, labels_ds, cm_predictions):\n    if correct == predict:\n        print(\"Correct flower: \", correct, \" Predicted flower: \", predict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))\nscore = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\nprecision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\nrecall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\ncmat = (cmat.T / cmat.sum(axis=1)).T # normalized\ndisplay_confusion_matrix(cmat, score, precision, recall)\nprint('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\n\nprint('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\n!head submission.csv","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visual validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = get_validation_dataset()\ndataset = dataset.unbatch().batch(50)\nbatch = iter(dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run this cell again for next set of images\nimages, labels = next(batch)\nprobabilities = model.predict(images)\npredictions = np.argmax(probabilities, axis=-1)\ndisplay_batch_of_images((images, labels), predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}