{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries, Data and TPU Configuration","metadata":{}},{"cell_type":"code","source":"# Import libraries\nimport os\nimport math, re\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport tensorflow as tf\nprint(tf.__version__)\n# Explore the structure of the data\nfor dirpath, dirnames, filenames in os.walk('/kaggle/input'):\n    print(f\"There are {len(dirnames)} directories and {len(filenames)} images in {dirpath}.\")","metadata":{"execution":{"iopub.status.busy":"2021-07-09T10:32:18.388847Z","iopub.execute_input":"2021-07-09T10:32:18.389254Z","iopub.status.idle":"2021-07-09T10:32:24.573154Z","shell.execute_reply.started":"2021-07-09T10:32:18.389168Z","shell.execute_reply":"2021-07-09T10:32:24.572147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T10:32:24.574713Z","iopub.execute_input":"2021-07-09T10:32:24.575004Z","iopub.status.idle":"2021-07-09T10:32:30.426993Z","shell.execute_reply.started":"2021-07-09T10:32:24.574974Z","shell.execute_reply":"2021-07-09T10:32:30.425863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Parameters and Data Utilities","metadata":{}},{"cell_type":"code","source":"# Instantiate some variables\nfrom kaggle_datasets import KaggleDatasets\nREPLICAS = strategy.num_replicas_in_sync\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nWARMUP_EPOCHS = 3\nWARMUP_LEARNING_RATE = 1e-4 * REPLICAS\nEPOCHS = 30\nLEARNING_RATE = 3e-5 * REPLICAS\nHEIGHT = 224; WIDTH = 224; CHANNELS = 3\nN_CLASSES = 104\n\n# Data access\nGCS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started') + f'/tfrecords-jpeg-{HEIGHT}x{WIDTH}'\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2021-07-09T10:32:30.42858Z","iopub.execute_input":"2021-07-09T10:32:30.428871Z","iopub.status.idle":"2021-07-09T10:32:31.075388Z","shell.execute_reply.started":"2021-07-09T10:32:30.428844Z","shell.execute_reply":"2021-07-09T10:32:31.074402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define Class Names\nCLASSES = [\n    'pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', \n    'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', \n    'globe thistle', 'snapdragon', \"colt's foot\", 'king protea', 'spear thistle', \n    'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', \n    'balloon flower', 'giant white arum lily', 'fire lily', 'pincushion flower', \n    'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', \n    'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william', \n    'carnation', 'garden phlox', 'love in the mist', 'cosmos',  'alpine sea holly', \n    'ruby-lipped cattleya', 'cape flower', 'great masterwort',  'siam tulip', \n    'lenten rose', 'barberton daisy', 'daffodil',  'sword lily', 'poinsettia', \n    'bolero deep blue',  'wallflower', 'marigold', 'buttercup', 'daisy', \n    'common dandelion', 'petunia', 'wild pansy', 'primula',  'sunflower', \n    'lilac hibiscus', 'bishop of llandaff', 'gaura',  'geranium', 'orange dahlia', \n    'pink-yellow dahlia', 'cautleya spicata',  'japanese anemone', \n    'black-eyed susan', 'silverbush', 'californian poppy',  'osteospermum', \n    'spring crocus', 'iris', 'windflower',  'tree poppy', 'gazania', 'azalea', \n    'water lily',  'rose', 'thorn apple', 'morning glory', 'passion flower',  \n    'lotus', 'toad lily', 'anthurium', 'frangipani',  'clematis', 'hibiscus', \n    'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', \n    'watercress',  'canna lily', 'hippeastrum ', 'bee balm', 'pink quill',  \n    'foxglove', 'bougainvillea', 'camellia', 'mallow',  'mexican petunia',  \n    'bromelia', 'blanket flower', 'trumpet creeper',  'blackberry lily', \n    'common tulip', 'wild rose']","metadata":{"execution":{"iopub.status.busy":"2021-07-09T10:32:31.07692Z","iopub.execute_input":"2021-07-09T10:32:31.077563Z","iopub.status.idle":"2021-07-09T10:32:31.087481Z","shell.execute_reply.started":"2021-07-09T10:32:31.077517Z","shell.execute_reply":"2021-07-09T10:32:31.085937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Datasets\n\nWe need functions for:\n* Decoding images\n* Reading labeled and unlabeled `tfrecord` files\n* Loading datasets\n* Applying data augmentation\n* Loading in the training, validation and test sets specifically","metadata":{}},{"cell_type":"code","source":"AUTO = tf.data.AUTOTUNE\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, dtype=tf.float32) / 255.\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_labeled_tfrecord(example):\n    format_ = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"class\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, format_)\n    image = decode_image(example[\"image\"])\n    label = tf.cast(example[\"class\"], dtype=tf.int32)\n    return image, label\n\ndef read_unlabeled_tfrecord(example):\n    format_ = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"id\": tf.io.FixedLenFeature([], tf.string) # No labels; IDs are provided\n    }\n    example = tf.io.parse_single_example(example, format_)\n    image = decode_image(example[\"image\"])\n    id_ = example[\"id\"]\n    return image, id_\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # Optimize speed\n    # Form the dataset\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # Reads in the files\n    dataset = dataset.with_options(ignore_order) # Uses data as soon as it's bought in \n    # Form the dataset that returns tuples of (image, label) or (image, id)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    return dataset\n\ndef data_augment(image, label):\n    \"\"\"Data Augmentation operations functionized\"\"\"\n    crop_sz = tf.random.uniform([], int(HEIGHT*0.7), HEIGHT, dtype=tf.int32)\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_saturation(image, lower=0, upper=2)    \n    image = tf.image.random_crop(image, size=[crop_sz, crop_sz, CHANNELS])\n    # Resize after applying all augmentation\n    image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n    return image, label\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # Make sure training doesn't abruptly stop\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE).prefetch(AUTO) # Parallelize and batch\n    return dataset\n\ndef get_training_dataset_preview(ordered=True):\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\nimport re\ndef count_data_items(filenames):\n    \"\"\"Counts the number of items in each .tfrec file, as specified by the name\"\"\"\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T12:12:16.043048Z","iopub.execute_input":"2021-07-09T12:12:16.043668Z","iopub.status.idle":"2021-07-09T12:12:16.065509Z","shell.execute_reply.started":"2021-07-09T12:12:16.043631Z","shell.execute_reply":"2021-07-09T12:12:16.064654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualizations\n\nDefine functions for visualizing some samples.","metadata":{}},{"cell_type":"code","source":"# Visualization utility functions\nnp.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n\ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n    \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n        \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-09T10:32:31.115988Z","iopub.execute_input":"2021-07-09T10:32:31.116395Z","iopub.status.idle":"2021-07-09T10:32:31.145191Z","shell.execute_reply.started":"2021-07-09T10:32:31.116356Z","shell.execute_reply":"2021-07-09T10:32:31.143873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And some functions to visualize model predictions for later.","metadata":{}},{"cell_type":"code","source":"def dataset_to_numpy_util(dataset, N):\n    dataset = dataset.unbatch().batch(N)\n    for images, labels in dataset:\n        numpy_images = images.numpy()\n        numpy_labels = labels.numpy()\n        break;  \n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    label = np.argmax(label, axis=-1)\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], str(correct), ', shoud be ' if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower_eval(image, title, subplot, red=False):\n    plt.subplot(subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    plt.title(title, fontsize=14, color='red' if red else 'black')\n    return subplot+1\n\ndef display_9_images_with_predictions(images, predictions, labels):\n    subplot=331\n    plt.figure(figsize=(13,13))\n    for i, image in enumerate(images):\n        title, correct = title_from_label_and_target(predictions[i], labels[i])\n        subplot = display_one_flower_eval(image, title, subplot, not correct)\n        if i >= 8:\n            break;\n              \n    plt.tight_layout()\n    plt.subplots_adjust(wspace=0.1, hspace=0.1)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-09T10:32:31.146874Z","iopub.execute_input":"2021-07-09T10:32:31.147283Z","iopub.status.idle":"2021-07-09T10:32:31.169394Z","shell.execute_reply.started":"2021-07-09T10:32:31.147241Z","shell.execute_reply":"2021-07-09T10:32:31.168162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"# Explore the Training set\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\ntrain_dataset = get_training_dataset_preview(ordered=True)\ny_train = next(iter(train_dataset.unbatch().map(lambda image, label: label).batch(NUM_TRAINING_IMAGES))).numpy()\nprint(f\"Number of Training Images: {NUM_TRAINING_IMAGES}\")\n\n# Explore the Validation set\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nval_dataset = get_validation_dataset(ordered=True)\ny_val = next(iter(val_dataset.unbatch().map(lambda image, label: label).batch(NUM_VALIDATION_IMAGES))).numpy()\nprint(f\"Number of Validation images: {NUM_VALIDATION_IMAGES}\")\n\n# Explore the Test data\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nprint(f\"Number of Test images: {NUM_TEST_IMAGES}\")\ntest_dataset = get_test_dataset(ordered=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T12:12:24.508718Z","iopub.execute_input":"2021-07-09T12:12:24.509071Z","iopub.status.idle":"2021-07-09T12:12:28.169471Z","shell.execute_reply.started":"2021-07-09T12:12:24.509039Z","shell.execute_reply":"2021-07-09T12:12:28.168404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is an approximate 80:20 ratio between the Training and Validation images.\n\n## Visualize samples","metadata":{}},{"cell_type":"code","source":"# Look at some samples from the Training set\ndisplay_batch_of_images(next(iter(train_dataset.unbatch().batch(20))))","metadata":{"execution":{"iopub.status.busy":"2021-07-09T10:32:37.394993Z","iopub.execute_input":"2021-07-09T10:32:37.395308Z","iopub.status.idle":"2021-07-09T10:32:39.300161Z","shell.execute_reply.started":"2021-07-09T10:32:37.395276Z","shell.execute_reply":"2021-07-09T10:32:39.29694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Some from the Validation set\ndisplay_batch_of_images(next(iter(val_dataset.unbatch().batch(20))))","metadata":{"execution":{"iopub.status.busy":"2021-07-09T10:32:39.301606Z","iopub.execute_input":"2021-07-09T10:32:39.301965Z","iopub.status.idle":"2021-07-09T10:32:41.000405Z","shell.execute_reply.started":"2021-07-09T10:32:39.301932Z","shell.execute_reply":"2021-07-09T10:32:40.999295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Label Distribution of Training and Validation Sets","metadata":{}},{"cell_type":"code","source":"# import seaborn as sns\n# train_agg = np.asarray([[label, (y_train == index).sum()] for index, label in enumerate(CLASSES)])\n# valid_agg = np.asarray([[label, (y_val == index).sum()] for index, label in enumerate(CLASSES)])\n\n# fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(24, 64))\n\n# ax1 = sns.barplot(x=train_agg[...,1], y=train_agg[...,0], order=CLASSES, ax=ax1)\n# ax1.set_title('Train', fontsize=30)\n# ax1.tick_params(labelsize=16)\n\n# ax2 = sns.barplot(x=valid_agg[...,1], y=valid_agg[...,0], order=CLASSES, ax=ax2)\n# ax2.set_title('Validation', fontsize=30)\n# ax2.tick_params(labelsize=16)\n\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-09T12:15:36.416709Z","iopub.execute_input":"2021-07-09T12:15:36.41717Z","iopub.status.idle":"2021-07-09T12:15:36.421362Z","shell.execute_reply.started":"2021-07-09T12:15:36.417083Z","shell.execute_reply":"2021-07-09T12:15:36.420262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model and Training\n\nWe will make use of a pretrained EfficientNetB6 that has over 666 layers with its own weights from ImageNet.\n\nThe base model will be initially frozen so that the randomized Dense Head is given some time to \"warmup\" and later, we can fine-tune the whole model at once.","metadata":{}},{"cell_type":"code","source":"# Use EfficientNetB6 as a backbone\nfrom tensorflow.keras import layers\n\nwith strategy.scope():\n    base_model = tf.keras.applications.EfficientNetB6(include_top=False)\n    base_model.trainable = False\n    \n    inputs = layers.Input((None, None, 3))\n    x = base_model(inputs)\n    x = layers.GlobalAveragePooling2D(name=\"pooling\")(x)\n    outputs = layers.Dense(N_CLASSES, activation=\"softmax\", name=\"output_layer\")(x)\n    \n    model0 = tf.keras.Model(inputs, outputs)\n\n# Compile the model\nmodel0.compile(optimizer=tf.keras.optimizers.Adam(),\n               loss=\"sparse_categorical_crossentropy\",\n               metrics=[\"sparse_categorical_accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2021-07-09T10:53:26.584831Z","iopub.execute_input":"2021-07-09T10:53:26.585219Z","iopub.status.idle":"2021-07-09T10:53:55.118831Z","shell.execute_reply.started":"2021-07-09T10:53:26.585188Z","shell.execute_reply":"2021-07-09T10:53:55.117744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the summary\nmodel0.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-09T10:47:10.954424Z","iopub.execute_input":"2021-07-09T10:47:10.954741Z","iopub.status.idle":"2021-07-09T10:47:11.004642Z","shell.execute_reply.started":"2021-07-09T10:47:10.954711Z","shell.execute_reply":"2021-07-09T10:47:11.003485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the model\nfrom tensorflow.keras.utils import plot_model\nplot_model(model0)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T10:47:48.797177Z","iopub.execute_input":"2021-07-09T10:47:48.797727Z","iopub.status.idle":"2021-07-09T10:47:49.282201Z","shell.execute_reply.started":"2021-07-09T10:47:48.797662Z","shell.execute_reply":"2021-07-09T10:47:49.280936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nWARMUP_EPOCHS = 5;\nwarmup_history = model0.fit(x=get_training_dataset(),\n                          steps_per_epoch=STEPS_PER_EPOCH,\n                          validation_data=get_validation_dataset(),\n                          epochs=WARMUP_EPOCHS,\n                          verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T10:53:55.120258Z","iopub.execute_input":"2021-07-09T10:53:55.120541Z","iopub.status.idle":"2021-07-09T10:55:23.905466Z","shell.execute_reply.started":"2021-07-09T10:53:55.120514Z","shell.execute_reply":"2021-07-09T10:55:23.904519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a function to plot the loss curves\ndef plot_loss_curves(history):\n    acc = history.history[\"sparse_categorical_accuracy\"]\n    val_acc = history.history[\"val_sparse_categorical_accuracy\"]\n    loss = history.history[\"loss\"]\n    val_loss = history.history[\"val_loss\"]\n    epochs = range(len(val_loss))\n    \n    \n    plt.figure();\n    plt.plot(epochs, acc, label=\"Training Accuracy\")\n    plt.plot(epochs, val_acc, label=\"Validation Accuracy\")\n    plt.legend()\n    plt.xlabel(\"Epochs\")\n    plt.title(\"Accuracy\")\n    \n    plt.figure();\n    plt.plot(epochs, loss, label=\"Training Loss\")\n    plt.plot(epochs, val_loss, label=\"Validation Loss\")\n    plt.legend()\n    plt.xlabel(\"Epochs\")\n    plt.title(\"Loss\")","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:07:13.517883Z","iopub.execute_input":"2021-07-09T11:07:13.518277Z","iopub.status.idle":"2021-07-09T11:07:13.526447Z","shell.execute_reply.started":"2021-07-09T11:07:13.51824Z","shell.execute_reply":"2021-07-09T11:07:13.525302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_loss_curves(warmup_history)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:07:14.892147Z","iopub.execute_input":"2021-07-09T11:07:14.892791Z","iopub.status.idle":"2021-07-09T11:07:15.297071Z","shell.execute_reply.started":"2021-07-09T11:07:14.892753Z","shell.execute_reply":"2021-07-09T11:07:15.296305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the trainable state \nprint(\"Before Unfreezing\\n\")\nfor layer in model0.layers:\n    print(layer.name, layer.trainable)\nprint(\"After Unfreezing\\n\")\nbase_model.trainable = True\nfor layer in model0.layers:\n    print(layer.name, layer.trainable)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:11:30.743456Z","iopub.execute_input":"2021-07-09T11:11:30.743862Z","iopub.status.idle":"2021-07-09T11:11:30.777466Z","shell.execute_reply.started":"2021-07-09T11:11:30.743829Z","shell.execute_reply":"2021-07-09T11:11:30.776611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# How many layers inside EfficientNet are trainable?\ncount = 0\nfor layer in model0.layers[1].layers:\n    count += 1\nprint(f\"There are {count} layers trainable in EfficientNetB6\")","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:12:29.334033Z","iopub.execute_input":"2021-07-09T11:12:29.334579Z","iopub.status.idle":"2021-07-09T11:12:29.341266Z","shell.execute_reply.started":"2021-07-09T11:12:29.334528Z","shell.execute_reply":"2021-07-09T11:12:29.340115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Setup a Learning Rate scehdule\nLR_START = 1e-7\nLR_MIN = 1e-6\nLR_MAX = LEARNING_RATE\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = 0.8\nEPOCHS = 60\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_MIN) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\nx = list(range(EPOCHS))\ny = [lrfn(i) for i in x]\n\nsns.set_style(\"whitegrid\")\nplt.figure();\nplt.plot(x, y)\nprint(f\"\"\"\nTotal epochs: {EPOCHS} \nSteps per epoch: {NUM_TRAINING_IMAGES//BATCH_SIZE}\nLearning Rate Schedule: {y[0]:.3g} to {max(y):.3g} to {y[-1]:.3g}\n\"\"\")","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:23:05.112588Z","iopub.execute_input":"2021-07-09T11:23:05.113002Z","iopub.status.idle":"2021-07-09T11:23:05.342423Z","shell.execute_reply.started":"2021-07-09T11:23:05.112964Z","shell.execute_reply":"2021-07-09T11:23:05.341519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make callbacks to prevent overfitting and schedule LR\nearly_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min',\n                                                 patience=3, \n                                                 restore_best_weights=True,\n                                                 verbose=1)\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=0)\n# Recompile the model\nmodel0.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=tf.keras.optimizers.Adam(lr=LEARNING_RATE),\n              metrics=[\"sparse_categorical_accuracy\"])\nmodel0.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:34:38.010521Z","iopub.execute_input":"2021-07-09T11:34:38.011099Z","iopub.status.idle":"2021-07-09T11:34:38.128877Z","shell.execute_reply.started":"2021-07-09T11:34:38.011063Z","shell.execute_reply":"2021-07-09T11:34:38.127746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit the model\nhistory = model0.fit(x=get_training_dataset(),\n                    steps_per_epoch=STEPS_PER_EPOCH,\n                    validation_data=get_validation_dataset(),\n                    epochs=EPOCHS,\n                    callbacks=[lr_callback, early_stopping])","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:35:33.757083Z","iopub.execute_input":"2021-07-09T11:35:33.757487Z","iopub.status.idle":"2021-07-09T11:49:15.443651Z","shell.execute_reply.started":"2021-07-09T11:35:33.757454Z","shell.execute_reply":"2021-07-09T11:49:15.442659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Evaluation\n\nNote that with the ImageNet weights, the model barely got a 6% accuracy on the Validation Set. After fine-tuning however, the accuracies jumped to much higher levels since we allowed the pretrained model to adapt to new patterns in the data that were not there in ImageNet. ","metadata":{}},{"cell_type":"code","source":"# Check the loss curves for overfitting\nplot_loss_curves(history)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:56:46.836722Z","iopub.execute_input":"2021-07-09T11:56:46.837081Z","iopub.status.idle":"2021-07-09T11:56:47.360669Z","shell.execute_reply.started":"2021-07-09T11:56:46.837051Z","shell.execute_reply":"2021-07-09T11:56:47.359523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"While the model's performance is not remarkable in plain numbers, it is great to see that the loss curves are very close to one another. This means there are rather low chances that our model began to overfit.","metadata":{}},{"cell_type":"code","source":"# Check the performance across each of the classes in the Validation set\nfrom sklearn.metrics import classification_report\nx = val_dataset.map(lambda image, label: image)\nmodel_probs = model0.predict(x)\nmodel_preds = tf.argmax(model_probs, axis=-1)\nprint(classification_report(y_val, model_preds, target_names=CLASSES))","metadata":{"execution":{"iopub.status.busy":"2021-07-09T12:01:13.318805Z","iopub.execute_input":"2021-07-09T12:01:13.319189Z","iopub.status.idle":"2021-07-09T12:01:29.369981Z","shell.execute_reply.started":"2021-07-09T12:01:13.319157Z","shell.execute_reply":"2021-07-09T12:01:29.368732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's visualize some predictions!\nx_train_samp, y_train_samp = dataset_to_numpy_util(train_dataset, 9)\ntrain_samp_preds = model0.predict(x_train_samp, batch_size=9)\ndisplay_9_images_with_predictions(x_train_samp, train_samp_preds, y_train_samp)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T12:02:13.29098Z","iopub.execute_input":"2021-07-09T12:02:13.291348Z","iopub.status.idle":"2021-07-09T12:02:21.735149Z","shell.execute_reply.started":"2021-07-09T12:02:13.291316Z","shell.execute_reply":"2021-07-09T12:02:21.731933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# FIN","metadata":{}},{"cell_type":"code","source":"# Submit\ntest_dataset = get_test_dataset(ordered=True)\nx_test = test_dataset.map(lambda image, idnum: image)\ntest_preds = model0.predict(x_test)\ntest_preds = np.argmax(test_preds, axis=-1)\n\ntest_ids_ds = test_dataset.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\nsubmission = pd.DataFrame(test_ids, columns=['id'])\nsubmission['label'] = test_preds\nsubmission.to_csv('submission.csv', index=False)\ndisplay(submission.head(10))","metadata":{"execution":{"iopub.status.busy":"2021-07-09T12:15:45.768504Z","iopub.execute_input":"2021-07-09T12:15:45.769042Z","iopub.status.idle":"2021-07-09T12:15:56.513642Z","shell.execute_reply.started":"2021-07-09T12:15:45.769007Z","shell.execute_reply":"2021-07-09T12:15:56.512678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}