{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"# Introduction\nThis notebook is created following the notebook presented in [Ryan Holbrook's tutorial](https://www.kaggle.com/ryanholbrook/create-your-first-submission) and will follow what he did. We will reorganize where some of the helper functions and imports are. Following the first iteration using the tutorial we will try to improve on the output. Having finished the computer image course we can apply additional techniques to hopefully improve the score. Improved the score by adding improved augmentations, early stopping conditions, checkpoints and training each individual layer."},{"metadata":{},"cell_type":"markdown","source":"# Step 1. Setup\nFollowing what is presented in the tutorial we will import all required packages here\n## Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"# this notebook: https://www.kaggle.com/atamazian/fc-ensemble-external-data-effnet-densenet#Models-and-training\n# has a very high score and uses this model\n\n!pip install -q efficientnet\nimport efficientnet.tfkeras as efn\n\nimport math\nimport re\nimport os\nimport random\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nfrom tensorflow.keras import optimizers\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\n\nfrom kaggle_datasets import KaggleDatasets\n\nprint(\"Tensorflow version \" + tf.__version__)\nprint(\"\\nImported packages\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define a random seed so that the notebook can be reproduced\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Accelerator Detecton"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Set Some Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = [512, 512]\nHEIGHT = 512\nWIDTH = 512\nGCS_DS_PATH = KaggleDatasets().get_gcs_path()\nGCS_PATH = GCS_DS_PATH + '/tfrecords-jpeg-512x512'\nAUTO = tf.data.experimental.AUTOTUNE\nEPOCHS = 30\nCHANNELS = 3\n\nmodel_path = f'model_{HEIGHT}x{WIDTH}.h5'\n\n# Define the batch size. This will be 16 with TPU off and 128 (=16*8) with TPU on\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\nWARMUP_LEARNING_RATE = 1e-4 * strategy.num_replicas_in_sync\nLEARNING_RATE = 3e-5 * strategy.num_replicas_in_sync\n\nNUM_CLASSES = 104\nES_PATIENCE = 5\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec') \n\nCLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper Functions"},{"metadata":{},"cell_type":"markdown","source":"### The following functions streamline loading in the dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    \"\"\"\n    Reads in a jpeg image file and converts casts it into a uint8 tensor.\n    Each cell in the tensor contains numerical information about a given\n    pixel in the original jpeg image. The code then resizes the tensor\n    to the appropraite size to be used by the TPU. If we want to decoe\n    other file types we need to modify the \"decode_jpeg\" portion of the\n    code to use other file extensions.\n    \"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    \"\"\"\n    Takes a classified file and decodes the image returning the label\n    \"\"\"\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    \"\"\"\n    Takes an unclassified file and decodes the image with the id associated\n    \"\"\"\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    \"\"\"\n    Read from TFRecords. For optimal performance, reading from multiple files at once and\n    disregarding data order. Order does not matter since we will be shuffling the data anyway.\n    \"\"\"\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### The following functions streamline reading in the training, validation and test datasets."},{"metadata":{"trusted":true},"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shift(image, height, h_shift, w_shift):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly shifted\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    height_shift = h_shift * tf.random.uniform([1],dtype='float32') \n    width_shift = w_shift * tf.random.uniform([1],dtype='float32') \n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n        \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shift_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef data_augment(image, label):\n    \"\"\"\n    This is the data augmentation function given in the tutorial\n    \n    Thanks to the dataset.prefetch(AUTO)\n    statement in the next function (below), this happens essentially\n    for free on TPU. Data pipeline code is executed on the \"CPU\"\n    part of the TPU while the TPU itself is computing gradients.\n    \"\"\"\n    image = tf.image.random_flip_left_right(image)\n    #image = tf.image.random_saturation(image, 0, 2)\n    return image, label   \n\ndef augment(image,label):\n    \"\"\"\n    data augmentation taken from \n    https://www.kaggle.com/dimitreoliveira/flower-with-tpus-advanced-augmentations#Model-(EfficientNet)\n    \"\"\"\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel = tf.random.uniform([], 0, 1.0, dtype=tf.float32)    \n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shift = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    \n    # Flips\n    if p_spatial >= .2:\n        image = tf.image.random_flip_left_right(image)\n        image = tf.image.random_flip_up_down(image)\n        \n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n    \n    if p_rotation >= .3: # Rotation\n        image = transform_rotation(image, height=HEIGHT, rotation=45.)\n    if p_shift >= .3: # Shift\n        image = transform_shift(image, height=HEIGHT, h_shift=15., w_shift=15.)\n    if p_shear >= .3: # Shear\n        image = transform_shear(image, height=HEIGHT, shear=20.)\n        \n    # Crops\n    if p_crop > .4:\n        crop_size = tf.random.uniform([], int(HEIGHT*.7), HEIGHT, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n    elif p_crop > .7:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.8)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.9)\n            \n    image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n        \n    # Pixel-level transforms\n    if p_pixel >= .2:\n        if p_pixel >= .8:\n            image = tf.image.random_saturation(image, lower=0, upper=2)\n        elif p_pixel >= .6:\n            image = tf.image.random_contrast(image, lower=.8, upper=2)\n        elif p_pixel >= .4:\n            image = tf.image.random_brightness(image, max_delta=.2)\n        else:\n            image = tf.image.adjust_gamma(image, gamma=.6)\n\n    return image, label\n\ndef get_training_dataset(aug='old'):\n    \"\"\"\n    Reads in the training dataset and shuffles it around\n    \"\"\"\n    if aug == 'new':\n        aug_arg = augment\n    else:\n        aug_arg = data_augment\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(aug_arg, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_training_dataset_subset(ordered=False):\n    \"\"\"\n    Reads in a subset of the training dataset\n    \"\"\"\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    \"\"\"\n    Reads in the validation dataset\n    \"\"\"\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    \"\"\"\n    Reads in the test dataset\n    \"\"\"\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef count_data_items(filenames):\n    \"\"\"\n    Counts the total number of items\n    \"\"\"\n    # the number of data items is written in the name of the .tfrec\n    # files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Functions Used to Show Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def batch_to_numpy_images_and_labels(data):\n    \"\"\"\n    Converts the images and labels to numpy\n    \"\"\"\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case,\n                                     # these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is\n    # the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    \"\"\"\n    Generates a title from the image label\n    \"\"\"\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    \"\"\"\n    Plots a single flower image\n    \"\"\"\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n    \ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"\n    Plots a batch of images\n    \n    This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square\n    # or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n\n\ndef display_training_curves(training, validation, title, subplot):\n    \"\"\"\n    Plots a collection of training curves based on the input\n    \"\"\"\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])\n    \ndef display_confusion_matrix(cmat, score, precision, recall):\n    \"\"\"\n    Generates a confusion matrix comparing how well the algorithm preforms and predicting\n    a given result\n    \"\"\"\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(input_shape, NUM_CLASSES):\n    pretrained_model = efn.EfficientNetB6(\n        weights='noisy-student',\n        include_top=False ,\n        input_shape=input_shape\n    )\n    pretrained_model.trainable = False\n    \n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n    ])\n    \n    return model\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Learning Rate Evaluation Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Learning Rate Schedule for Fine Tuning #\ndef exponential_schedule_with_warmup(epoch):\n    '''\n    Create a schedule with a learning rate that decreases exponentially after linearly increasing during a warmup period.\n    '''\n    \n    warmup_epochs = 3\n    hold_max_epochs = 0\n    lr_start = 1e-8\n    lr_max = LEARNING_RATE\n    lr_min = 1e-6\n    decay = 0.8\n        \n        \n    if epoch < warmup_epochs:\n        lr = (lr_max - lr_start) / warmup_epochs * epoch + lr_start\n    elif epoch < warmup_epochs + hold_max_epochs:\n        lr = lr_max\n    else:\n        lr = lr_max * (decay ** (epoch - warmup_epochs - hold_max_epochs))\n        if lr_min is not None:\n            lr = tf.math.maximum(lr_min, lr)\n            \n    return lr\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Step 2. Read in the Data"},{"metadata":{},"cell_type":"markdown","source":"### See how many training, validation and test images we have to work with"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train data\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\ntrain_dataset = get_training_dataset_subset(ordered=True)\ny_train = next(iter(train_dataset.unbatch().map(lambda image, label: label).batch(NUM_TRAINING_IMAGES))).numpy()\nprint(f'Number of training images {NUM_TRAINING_IMAGES}')\n\n# Validation data\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nvalid_dataset = get_validation_dataset(ordered=True)\ny_valid = next(iter(valid_dataset.unbatch().map(lambda image, label: label).batch(NUM_VALIDATION_IMAGES))).numpy()\nprint(f'Number of validation images {NUM_VALIDATION_IMAGES}')\n\n# Test data\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nprint(f'Number of test images {NUM_TEST_IMAGES}')\ntest_dataset = get_test_dataset(ordered=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Step 3. Exploring the Data\nLets just output a few of the images to ensure that the data is read in correctly, this acts like a sanity check."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Examining a batch of the training data\ntrain_iter = iter(train_dataset.unbatch().batch(20))\ntrain_batch = next(train_iter)\ndisplay_batch_of_images(train_batch)\n\n# Every time we run this code we expect a different batch of 20 flower images to appear","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Examining a batch of the valid data\nvalid_iter = iter(valid_dataset.unbatch().batch(20))\nvalid_batch = next(valid_iter)\ndisplay_batch_of_images(valid_batch)\n\n# Every time we run this code we expect a different batch of 20 flower images to appear","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Examining a batch of the test data\ntest_iter = iter(test_dataset.unbatch().batch(20))\ntest_batch = next(test_iter)\ndisplay_batch_of_images(test_batch)\n\n# Every time we run this code we expect a different batch of 20 flower images to appear","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_batch_of_images(next(iter(get_training_dataset().unbatch().batch(20))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_batch_of_images(next(iter(get_training_dataset(aug='new').unbatch().batch(20))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Part 4. Defining Models\nFollowing what was done in the tutorial we will use a neural network to generate a model to classify the images. The tutorial starts with **VGG16** as the model, so we will start with that one. I may add the results of different models afterwards. \n\nNotes:\n* We upped the number of Epochs to 30 which seems to improve results as our figure below implies that we arent overfitting\n* Models and Epochs tested with score:\n    * VGG16 model and Epoch = 12 - 0.04086\n    * Xception model and Epoch = 30 - 0.58815\n    * EfficientNETB7 and Epoch = 30 - 0.58855"},{"metadata":{"trusted":true},"cell_type":"code","source":"# strategy.scope splits the work amongst the TPU cores so when using TensorFlow with TPUs\n# it is important to properly use strategy.scope\n\n# Model the data\nwith strategy.scope():\n    model = create_model((None, None, CHANNELS), NUM_CLASSES)\n\n# Generate the model and output the summary of the results\nmodel.compile(\n    optimizer=optimizers.Adam(lr=WARMUP_LEARNING_RATE),\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy'],\n)\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nwarmup_history = model.fit(x=get_training_dataset(), \n                           steps_per_epoch=STEPS_PER_EPOCH, \n                           validation_data=get_validation_dataset(),\n                           epochs=3, \n                           verbose=2).history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Part 5. Training the Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate a learning rate schedule for the model to train\nlr_callback = tf.keras.callbacks.LearningRateScheduler(exponential_schedule_with_warmup, verbose=True)\n\nrng = [i for i in range(EPOCHS)]\ny = [exponential_schedule_with_warmup(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Now on to training the model using that learning rate,\n# with the number of epochs we want to use this might take\n# some time. Remember to check our losses to ensure we arent\n# over- or under-fitting.\n\nfor layer in model.layers:\n    layer.trainable = True\n    \ncheckpoint = ModelCheckpoint(model_path, monitor='val_lines', mode='min', save_best_only=True)\n\nes = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE,\n                  restore_best_weights=True, verbose=1)\n\nlr_callback = LearningRateScheduler(exponential_schedule_with_warmup, verbose=0)\n\ncallback_list = [checkpoint, es, lr_callback]\n\nmodel.compile(optimizer=optimizers.Adam(lr=LEARNING_RATE),\n              loss='sparse_categorical_crossentropy',\n              metrics=['sparse_categorical_accuracy'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(\n    get_training_dataset(),\n    validation_data=valid_dataset,\n    epochs=EPOCHS,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    callbacks=[lr_callback],\n    verbose=2\n).history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Now that we've trained the model, lets look at our loss and see if we've over or underfit the data\ndisplay_training_curves(\n    history['loss'],\n    history['val_loss'],\n    'loss',\n    211,\n)\ndisplay_training_curves(\n    history['sparse_categorical_accuracy'],\n    history['val_sparse_categorical_accuracy'],\n    'accuracy',\n    212,\n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The results look like they converge so things are looking good","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Part 6. Validate our Model\nWe do have a validation data set so comparing our model to it is a good idea. Just because the model works well on the training data doesn't mean the model would work when we expand the sample."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate the values used in a confusion matrix to visually see if our algorithm works well\n# The matrix has a lot of components so it will take some time to generate\n\ncmdataset = get_validation_dataset(ordered=True)\nimages_ds = cmdataset.map(lambda image, label: image)\nlabels_ds = cmdataset.map(lambda image, label: label).unbatch()\n\ncm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy()\ncm_probabilities = model.predict(images_ds)\ncm_predictions = np.argmax(cm_probabilities, axis=-1)\n\nlabels = range(len(CLASSES))\ncmat = confusion_matrix(\n    cm_correct_labels,\n    cm_predictions,\n    labels=labels,\n)\ncmat = (cmat.T / cmat.sum(axis=1)).T # normalize","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Generate the following stastical metrics, [Precision, Recall](https://en.wikipedia.org/wiki/Precision_and_recall) and [F-score](https://en.wikipedia.org/wiki/F-score)\n* Precision: \"How many selected items are relevant?\" The number of correctly identified positive results divided by the number of all positive results\n* Recall: \"How many relevant items are selected?\" The number of correctly identified positive results divided by the number of all samples that should've been identified as positive.\n* F-score: A measure of the accuracy, calculated using precision and recall.\n\nFor all 3 statistics, 1 is the maximum value and higher is better."},{"metadata":{"trusted":true},"cell_type":"code","source":"score = f1_score(\n    cm_correct_labels,\n    cm_predictions,\n    labels=labels,\n    average='macro',\n)\nprecision = precision_score(\n    cm_correct_labels,\n    cm_predictions,\n    labels=labels,\n    average='macro',\n)\nrecall = recall_score(\n    cm_correct_labels,\n    cm_predictions,\n    labels=labels,\n    average='macro',\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate the confusion matrix with F-score, precision and recall shown on the plot\ndisplay_confusion_matrix(cmat, score, precision, recall)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visually Comparing Results\nWhile the confusion matrix is helpful, having such a large number of categories makes it difficult to read. We can instead look at a small sample of our images to see if there are any glaring issues."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get the validation data\ndataset = get_validation_dataset()\ndataset = dataset.unbatch().batch(20)\nbatch = iter(dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compare our model predictions to the actual flower\nimages, labels = next(batch)\nprobabilities = model.predict(images)\npredictions = np.argmax(probabilities, axis=-1)\ndisplay_batch_of_images((images, labels), predictions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Part 7. Applying the Test Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True)\n\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate the submission\nprint('Generating submission.csv file...')\n\n# Get image ids from test set and convert to unicode\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\n# Write the submission file\nnp.savetxt(\n    'submission.csv',\n    np.rec.fromarrays([test_ids, predictions]),\n    fmt=['%s', '%d'],\n    delimiter=',',\n    header='id,label',\n    comments='',\n)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}