{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Machine Learning Final Project Notebook with DenseNet 201\n## *Petals to the Metal: Flower Classification on TPU*\n### *By Xuanzhi Huang, Rahul Paul*"},{"metadata":{},"cell_type":"markdown","source":"## Step 1: Some pre-setting"},{"metadata":{},"cell_type":"markdown","source":"* ### Package preliminary"},{"metadata":{},"cell_type":"markdown","source":"#### Import all the packages we need."},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# These are some basic packages\nimport random, re, math, os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n\n\n# These are for data processing\nimport tensorflow_addons as tfa\nfrom kaggle_datasets import KaggleDatasets\n\n\n# These are for model training\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nfrom tensorflow.keras.applications import DenseNet201\nimport tensorflow.keras.backend as K\nfrom sklearn.model_selection import KFold\n\n\n# These are performance metrics\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\n\n\n# These are for class weights. Not used\nimport datetime\nimport tqdm\nimport json\nfrom collections import Counter\nimport gc\n\n\n# These are for model visualization. Not used\nfrom tensorflow.keras.utils import plot_model\nfrom IPython.display import Image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Detect the hardware and tell the appropriate distribution strategy"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n# Make the system tune the number of threads for us\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Configuration for image size, training epoch, number of folds, batch size, and random seed"},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = [512, 512]\nEPOCHS = 16\nSEED = 100\nFOLDS = 3\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Set the data access"},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')\nGCS_PATH_SELECT = { \n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Add more mixed precision and/or XLA (refer to Chris Deotte's notebook)\n[Rotation Augmentation GPU/TPU - [0.96+]](https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Add more mixed precision and/or XLA to allow the TPU memory to handle larger batch sizes \n# and can speed up the training process\nMIXED_PRECISION = False\nXLA_ACCELERATE = False\n\nif MIXED_PRECISION:\n    if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')\n    else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n    mixed_precision.set_policy(policy)\n    print('Mixed precision enabled')\n\nif XLA_ACCELERATE:\n    tf.config.optimizer.set_jit(True)\n    print('Accelerated Linear Algebra enabled')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Show all the classes we have"},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose'] ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Step 2: Set some visualization functions"},{"metadata":{},"cell_type":"markdown","source":"* ### Set training and validation curve function to show the changes in loss and accuracy (not used)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_train_valid_curves(training, validation, title, subplot):\n    \n    if subplot % 10 == 1:\n        plt.subplots(figsize = (15,15), facecolor = '#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    ax.set_xlabel('epoch')\n    ax.legend(['training', 'validation.'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Set a function to plot confusion matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_confusion_matrix(cmat, score, precision, recall):\n    \n    plt.figure(figsize = (15, 15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap = 'Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict = {'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha = \"left\", rotation_mode = \"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict = {'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation = 45, ha = \"right\", rotation_mode = \"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict = {'fontsize': 18, 'horizontalalignment': 'right', 'verticalalignment': 'top', 'color': '#804040'})\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Construct functions to show the beautiful flowers (refer to Dimitre Oliveira) (not used)\n[Flower with TPUs - Advanced augmentations](https://www.kaggle.com/dimitreoliveira/flower-with-tpus-advanced-augmentations)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object:\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (for test data)\n    return numpy_images, numpy_labels\n\n\ndef title_from_label_and_target_(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\n\ndef display_one_flower(image, title, subplot, red = False, titlesize = 16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize = int(titlesize) if not red else int(titlesize / 1.2), color = 'red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2] + 1)\n\n\ndef display_batch_of_images(databatch, predictions = None):\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n    rows = int(math.sqrt(len(images)))\n    cols = len(images) // rows\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot = (rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize = (FIGSIZE, FIGSIZE / cols*rows))\n    else:\n        plt.figure(figsize = (FIGSIZE / rows * cols,FIGSIZE))\n    \n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target_(predictions[i], label)\n        dynamic_titlesize = FIGSIZE * SPACING / max(rows,cols) * 40 + 3\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize = dynamic_titlesize)\n    \n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace = 0, hspace = 0)\n    else:\n        plt.subplots_adjust(wspace = SPACING, hspace = SPACING)\n    plt.show()\n    \n\n# Visualize predictions. Images with labels telling whether prediction is true will be shown.\ndef dataset_to_numpy_util(dataset, N):\n    dataset = dataset.unbatch().batch(N)\n    for images, labels in dataset:\n        numpy_images = images.numpy()\n        numpy_labels = labels.numpy()\n        break;  \n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    label = np.argmax(label, axis = -1)\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], str(correct), ', should be ' if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower_eval(image, title, subplot, red = False):\n    plt.subplot(subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    plt.title(title, fontsize = 14, color = 'red' if red else 'black')\n    return subplot + 1\n\ndef display_9_images_with_predictions(images, predictions, labels):\n    subplot = 331\n    plt.figure(figsize = (13,13))\n    for i, image in enumerate(images):\n        title, correct = title_from_label_and_target(predictions[i], labels[i])\n        subplot = display_one_flower_eval(image, title, subplot, not correct)\n        if i >= 8:\n            break;\n    plt.tight_layout()\n    plt.subplots_adjust(wspace = 0.1, hspace = 0.1)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Step 3: Set functions to gain training set, validation set, and test set"},{"metadata":{},"cell_type":"markdown","source":"* ### Decode images and standardize them"},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    \n    image = tf.image.decode_jpeg(image_data, channels = 3)\n    image = tf.cast(image, tf.float32) / 255.0  \n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    \n    return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Set a function to read labeled tfrec files (i.e. training & validation set)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    \n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"class\": tf.io.FixedLenFeature([], tf.int64),\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    \n    return image, label\n\n\n\n# This is for data visualization. Not used\ndef read_labeled_id_tfrecord(example):\n    \n    LABELED_ID_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"class\": tf.io.FixedLenFeature([], tf.int64),\n        \"id\": tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, LABELED_ID_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    idnum =  example['id']\n    \n    return image, label, idnum","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Set a function to read unlabeled tfrec files (i.e. test set)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_unlabeled_tfrecord(example):\n    \n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"id\": tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    \n    return image, idnum","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Load image data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# For best performance, read from multiple tfrec files at once\n# Disregard data's order, since data will be shuffled\ndef load_dataset(filenames, labeled = True, ordered = False):\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False  # Disable order to increase running speed\n    # Automatically interleaves reading\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)\n    # Use data in the shuffled order\n    dataset = dataset.with_options(ignore_order)\n    # Returns a dataset of (image, label) pairs if labeled = True (i.e. training & validation set)\n    # or (image, id) pairs if labeld = False (i.e. test set)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Data augmentation"},{"metadata":{},"cell_type":"markdown","source":"#### Since the quality of images in test set are similar to images in training and validation set, to build a better model for test images, there is no need to do too much image augmentation. We think resizing, cropping, and flipping are adequate. However, for better generalization, it is reasonable to take all the augmentation listed below (including those set as comments) into account."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Randomly make some changes to the images and return the new images and labels\ndef data_augment(image, label):\n        \n    # Set seed for data augmentation\n    seed = 100\n    \n    # Randomly resize and then crop images\n    image = tf.image.resize(image, [800, 800])\n    image = tf.image.random_crop(image, [512, 512, 3], seed = seed)\n\n    # Randomly reset brightness of images\n    # image = tf.image.random_brightness(image, 0.6, seed = seed)\n    \n    # Randomly reset saturation of images\n    # image = tf.image.random_saturation(image, 3, 5, seed = seed)\n        \n    # Randomly reset contrast of images\n    # image = tf.image.random_contrast(image, 0.3, 0.5, seed = seed)\n\n    # Blur images\n    # image = tfa.image.mean_filter2d(image, filter_shape = 10)\n    \n    # Randomly flip images\n    image = tf.image.random_flip_left_right(image, seed = seed)\n    image = tf.image.random_flip_up_down(image, seed = seed)\n    \n    return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Gain training set"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset(dataset, do_aug = True):\n    \n    if do_aug: dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)  # prefetch next batch while training (autotune prefetch buffer size)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Gain validation set"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_dataset(dataset):\n    \n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Gain test set"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(ordered = False):\n    \n    dataset = load_dataset(TEST_FILENAMES, labeled = False, ordered = ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Count the number of images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\nNUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES) * (FOLDS - 1.) / FOLDS)\nNUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES) * (1. / FOLDS))\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Show example augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"row = 3\ncol = 4\nall_elements = get_training_dataset(load_dataset(TRAINING_FILENAMES),do_aug = False).unbatch()\none_element = tf.data.Dataset.from_tensors(next(iter(all_elements)))\n# Map the images to the data augmentation function for image processing\naugmented_element = one_element.repeat().map(data_augment).batch(row * col)\naugmented_element = augmented_element.prefetch(AUTO)\n\nfor (img, label) in augmented_element:\n    plt.figure(figsize = (15, int(15 * row / col)))\n    for j in range(row * col):\n        plt.subplot(row, col, j + 1)\n        plt.axis('off')\n        plt.imshow(img[j, ])\n    plt.show()\n    break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Step 4: Build the model and make prediction"},{"metadata":{},"cell_type":"markdown","source":"* ### Customize learning rate scheduler and visualize it"},{"metadata":{"trusted":true},"cell_type":"code","source":"def lrfn(epoch):\n    \n    LR_START = 0.00001\n    LR_MAX = 0.00005 * strategy.num_replicas_in_sync\n    LR_MIN = 0.00001\n    LR_RAMPUP_EPOCHS = 5\n    LR_SUSTAIN_EPOCHS = 0\n    LR_EXP_DECAY = .8\n    \n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY ** (epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n\n# Visualization changes in learning rate\nrng = [i for i in range(25 if EPOCHS < 25 else EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Build the model and load it into TPU, make prediction (refer to Chris Deotte)\n[Rotation Augmentation GPU/TPU - [0.96+]](https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96/notebook)"},{"metadata":{},"cell_type":"markdown","source":"#### Set all the layers trainable and conduct cross validation for training and prediction. If set some layers untrainable, the accuracy grows too slow, so we don't do that."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    \n    with strategy.scope():\n\n        rnet = DenseNet201(\n            input_shape = (512, 512, 3),\n            weights = 'imagenet',  # Use the preset parameters of ImageNet\n            include_top = False\n        )\n\n        rnet.trainable = True\n        #for i in range(len(rnet.layers) - 2, len(rnet.layers)):\n        #    rnet.layers[i].trainable = True\n\n        model = tf.keras.Sequential([\n            rnet,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            #tf.keras.layers.Dense(64, activation = 'relu', dtype = 'float32'),\n            #tf.keras.layers.Dropout(0.5),\n            #tf.keras.layers.Dense(32, activation = 'relu'),\n            #tf.keras.layers.Dropout(0.5),\n            tf.keras.layers.Dense(len(CLASSES), activation = 'softmax', dtype = 'float32')\n        ])\n\n        model.compile(\n            optimizer = 'adam',\n            # For multiclassification, we can use cross entropy or sparse cross entropy as our loss function \n            # These two cross entropy are the same in essence, but they are applied in different scenarios\n            # If our target is one-hot encoded, it is better to use cross entropy\n            # If our target is an integer, sparse cross entropy is a better choice, and this is our case\n            loss = 'sparse_categorical_crossentropy', \n            metrics = ['sparse_categorical_accuracy']\n        )\n\n        model.summary()\n        # Save the model\n        return model\n        \n        \ndef train_cross_validate(folds = 5):\n    \n    histories = []\n    models = []\n    early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 3)\n    kfold = KFold(folds, shuffle = True, random_state = SEED)\n    for f, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n        print(); print('#' * 25)\n        print('### FOLD',f + 1)\n        print('#' * 25)\n        train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)\n        val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)\n        model = get_model()\n        history = model.fit(\n            get_training_dataset(train_dataset), \n            steps_per_epoch = STEPS_PER_EPOCH,\n            epochs = EPOCHS,\n            callbacks = [lr_callback],\n            validation_data = get_validation_dataset(val_dataset),\n            verbose = 2\n        )\n        models.append(model)\n        histories.append(history)\n    \n    return histories, models\n\n\ndef train_and_predict(folds = 5):\n    \n    test_ds = get_test_dataset(ordered = True)  # Since we are splitting the dataset and iterating separately on images and ids, order matters.\n    test_images_ds = test_ds.map(lambda image, idnum: image)\n    histories, models = train_cross_validate(folds = folds)\n    # Get the mean probability of the folds models and the predicted labels\n    probabilities = np.average([models[i].predict(test_images_ds) for i in range(folds)], axis = 0)\n    predictions = np.argmax(probabilities, axis = -1)\n    test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n    test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n    np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt = ['%s', '%d'], delimiter = ',', header = 'id,label', comments = '')\n\n    return histories, models\n    \n# run train and predict\nhistories, models = train_and_predict(folds = FOLDS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ### Check model's performance on validation set"},{"metadata":{},"cell_type":"markdown","source":"#### Draw the confusion matrix, compute F1 score, precision, and recall"},{"metadata":{"trusted":true},"cell_type":"code","source":"all_labels = []; all_prob = []; all_pred = []\nkfold = KFold(FOLDS, shuffle = True, random_state = SEED)\nfor j, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n    print('Inferring fold', j + 1, 'validation images...')\n    VAL_FILES = list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES'])\n    NUM_VALIDATION_IMAGES = count_data_items(VAL_FILES)\n    cmdataset = get_validation_dataset(load_dataset(VAL_FILES, labeled = True, ordered = True))\n    images_ds = cmdataset.map(lambda image, label: image)\n    labels_ds = cmdataset.map(lambda image, label: label).unbatch()\n    all_labels.append(next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy()) # get everything as one batch\n    prob = models[j].predict(images_ds)\n    all_prob.append(prob)\n    all_pred.append(np.argmax(prob, axis = -1))\ncm_correct_labels = np.concatenate(all_labels)\ncm_probabilities = np.concatenate(all_prob)\ncm_predictions = np.concatenate(all_pred)\n\ncmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))\nscore = f1_score(cm_correct_labels, cm_predictions, labels = range(len(CLASSES)), average = 'macro')\nprecision = precision_score(cm_correct_labels, cm_predictions, labels = range(len(CLASSES)), average = 'macro')\nrecall = recall_score(cm_correct_labels, cm_predictions, labels = range(len(CLASSES)), average = 'macro')\ndisplay_confusion_matrix(cmat, score, precision, recall)\nprint('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall)); print()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}