{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras as keras\nimport tensorflow_addons as tfa\nimport numpy as np\n\nfrom tensorflow.data.experimental import AUTOTUNE\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\n\nimport os\nimport datetime\nimport random\nimport math, re, os\n\nfrom glob import glob\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\n\n%load_ext tensorboard\n\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TPU or GPU detection","metadata":{}},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path() \n\ntry: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError: # detect GPUs\n    strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    #strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE = [512, 512] \n\nEPOCHS = 50\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\nGCS_PATH_SELECT = { # available image sizes\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\n\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec') \n\nCLASSES = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', 'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', 'globe thistle', 'snapdragon', \"colt's foot\", 'king protea', 'spear thistle', 'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower', 'giant white arum lily', 'fire lily', 'pincushion flower', 'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william', 'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip', 'lenten rose', 'barberton daisy', 'daffodil', 'sword lily', 'poinsettia', 'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion', 'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia', 'cautleya spicata', 'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy', 'gazania', 'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily', 'anthurium', 'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily', 'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower', 'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose']                                                                                                                                            # 100 - 102\n\nnumber_of_classes = len(CLASSES)\nlearning_rate = .1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization utilities","metadata":{}},{"cell_type":"code","source":"# numpy and matplotlib defaults\nnp.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n    \ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n\ndef display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()\n    \ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Datasets","metadata":{}},{"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)  # image format uint8 [0,255]\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    #image = tf.image.random_saturation(image, 0, 2)\n    return image, label   \n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALIDATION_STEPS = -(-NUM_VALIDATION_IMAGES // BATCH_SIZE) # The \"-(-//)\" trick rounds up instead of down :-)\nTEST_STEPS = -(-NUM_TEST_IMAGES // BATCH_SIZE)             # The \"-(-//)\" trick rounds up instead of down :-)\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset visualizations","metadata":{}},{"cell_type":"code","source":"# Peek at training data\ntraining_dataset = get_training_dataset()\ntraining_dataset = training_dataset.unbatch().batch(20)\ntrain_batch = iter(training_dataset)\ndisplay_batch_of_images(next(train_batch))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class layers:\n    class ResidualUnit(keras.layers.Layer):\n        def __init__(self, filters, strides, activation='relu', **kwargs):\n            super().__init__(**kwargs)\n\n            self.activation = keras.activations.get(activation)\n            self.layers = [\n                keras.layers.Conv2D(filters, 3, strides, padding='same', use_bias=False),\n                keras.layers.BatchNormalization(),\n                self.activation,\n                keras.layers.Conv2D(filters, 3, 1, padding='same', use_bias=False),\n                keras.layers.BatchNormalization(),\n            ]\n\n            if strides == 1:\n                self.skip = []\n            else:\n                self.skip = [\n                    keras.layers.Conv2D(filters, 1, strides, padding='same', use_bias=False),\n                    keras.layers.BatchNormalization(),\n                ]\n\n\n        def __call__(self, inputs):\n            Z = Z_skip = inputs\n            for layer in self.layers:\n                Z = layer(Z)\n            for layer in self.skip:\n                Z_skip = layer(Z_skip)\n            return self.activation(Z + Z_skip)\n        \n        \n\n        \n    class DeepResidualUnit(keras.layers.Layer):\n        def __init__(self, filters, strides, activation='relu', **kwargs):\n            super().__init__(**kwargs)\n\n            self.activation = keras.activations.get(activation)\n            self.layers = [\n                keras.layers.Conv2D(filters // 4, 1, strides, padding='same', use_bias=False),\n                keras.layers.BatchNormalization(),\n                self.activation,\n                keras.layers.Conv2D(filters // 4, 3, 1, padding='same', use_bias=False),\n                keras.layers.BatchNormalization(),\n                self.activation,\n                keras.layers.Conv2D(filters, 1, 1, padding='same', use_bias=False),\n                keras.layers.BatchNormalization(),\n            ]\n\n            if strides == 1:\n                self.skip = []\n            else:\n                self.skip = [\n                    keras.layers.Conv2D(filters, 1, strides, padding='same', use_bias=False),\n                    keras.layers.BatchNormalization(),\n                ]\n\n\n        def __call__(self, inputs):\n            Z = Z_skip = inputs\n            for layer in self.layers:\n                Z = layer(Z)\n            for layer in self.skip:\n                Z_skip = layer(Z_skip)\n            return self.activation(Z + Z_skip)\n\n        \n        \n        \n        \n        \n        \n        \ndef ResNet34():\n    model = keras.models.Sequential()\n    model.add(keras.layers.Conv2D(64, 7, strides=2, padding='same', input_shape=(None, None, 3)))\n    model.add(keras.layers.BatchNormalization())\n    model.add(keras.layers.Activation('relu'))\n    model.add(keras.layers.MaxPool2D(pool_size=3, strides=2, padding=\"same\"))\n\n    filters = [64] * 3 + [128] * 4 + [256] * 6 + [512] * 3\n\n    prev_filters = filters[0]\n    for filters in filters:\n        strides = 1 if filters == prev_filters else 2\n        model.add(layers.ResidualUnit(filters, strides))\n        prev_filters = filters\n\n    model.add(keras.layers.GlobalAveragePooling2D())\n    model.add(keras.layers.Flatten())\n    model.add(keras.layers.Dense(number_of_classes, 'softmax'))\n\n    return model\n\n\ndef ResNet152():\n    model = keras.models.Sequential()\n    model.add(keras.layers.Conv2D(64, 7, strides=2, padding='same', input_shape=(None, None, 3)))\n    model.add(keras.layers.BatchNormalization())\n    model.add(keras.layers.Activation('relu'))\n    \n    model.add(keras.layers.Conv2D(256, 1, strides=1, padding='same', input_shape=(None, None, 3)))\n    model.add(keras.layers.BatchNormalization())\n    model.add(keras.layers.Activation('relu'))\n    \n    model.add(keras.layers.MaxPool2D(pool_size=3, strides=2, padding=\"same\"))\n\n    filters = [256] * 3 + [512] * 8 + [1024] * 36 + [2048] * 3\n\n    prev_filters = filters[0]\n    for filters in filters:\n        strides = 1 if filters == prev_filters else 2\n        model.add(layers.DeepResidualUnit(filters, strides))\n        prev_filters = filters\n\n    model.add(keras.layers.GlobalAveragePooling2D())\n    model.add(keras.layers.Flatten())\n    model.add(keras.layers.Dense(number_of_classes, 'softmax'))\n\n    return model\n\n\n\n\n\n\ndef compile_model(model):\n    model.compile(optimizer=keras.optimizers.SGD(learning_rate=learning_rate,\n                                                 momentum=.9,\n                                                 nesterov=True),\n                  loss='sparse_categorical_crossentropy',\n                  metrics=[keras.metrics.SparseCategoricalAccuracy(name='accuracy'),\n                           keras.metrics.SparseTopKCategoricalAccuracy(k=5, name='top-5_accuracy')],\n                  steps_per_execution=16)\n\n    return model\n\n\n\nwith strategy.scope():\n    model = ResNet34()\n#     model = ResNet152()\n    model = compile_model(model)\n    print(f'number of parameters: {model.count_params()}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"log_dir = 'logs/' + datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S')\n\ntensor_board = keras.callbacks.TensorBoard(log_dir='../output/logs')\nlr_schedule = tf.keras.callbacks.ReduceLROnPlateau(factor=0.1, patience=8, verbose=1)\n\nhistory = model.fit(get_training_dataset(), \n          validation_data=get_validation_dataset(),\n          validation_steps=VALIDATION_STEPS,\n          epochs=EPOCHS, \n          steps_per_epoch=STEPS_PER_EPOCH,\n          callbacks=[lr_schedule],\n    )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_training_curves(history.history['accuracy'], history.history['val_accuracy'], 'accuracy', 212)\ndisplay_training_curves(history.history['top-5_accuracy'], history.history['val_top-5_accuracy'], 'top-5 accuracy', 211)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Confusion matrix","metadata":{}},{"cell_type":"code","source":"cmdataset = get_validation_dataset(ordered=True) \nimages_ds = cmdataset.map(lambda image, label: image)\nlabels_ds = cmdataset.map(lambda image, label: label).unbatch()\ncm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy() \ncm_probabilities = model.predict(images_ds, steps=VALIDATION_STEPS)\ncm_predictions = np.argmax(cm_probabilities, axis=-1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))\nscore = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\nprecision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\nrecall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\ncmat = (cmat.T / cmat.sum(axis=1)).T \ndisplay_confusion_matrix(cmat, score, precision, recall)\nprint('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions Visual validation","metadata":{}},{"cell_type":"code","source":"dataset = get_validation_dataset()\ndataset = dataset.unbatch().batch(20)\nbatch = iter(dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, labels = next(batch)\nprobabilities = model.predict(tf.cast(images, tf.float32))\npredictions = np.argmax(probabilities, axis=-1)\ndisplay_batch_of_images((images, labels), predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}