{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Training with TPU for [House of Blocks Competition](http://https://www.kaggle.com/c/applications-of-deep-learning-wustl-fall-2020/overview)\n\nThis notebook:\n- loads previously created [TFRecords](http://https://www.kaggle.com/jesseallardice/tfrecords-for-adl-wustl-fall-2020) for the applications-of-deep-learning-wustl-fall-2020\n- Set a Learning Rate Schedule for the training.\n- Trains a Xception model to >99% validation accuracy.\n- Performs inferance on the test data.\n\n\nNotebook for creating the TFRecords:\nhttps://www.kaggle.com/jesseallardice/creating-tfrecords\n\nUseful resources:\nhttps://www.kaggle.com/docs/tpu\n\nMost of the code is adapted from:\n- Building the TFRecords https://www.kaggle.com/cdeotte/how-to-create-tfrecords\n- Model and training https://www.kaggle.com/mgornergoogle/five-flowers-with-keras-and-xception-on-tpu"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import math, re, os\nimport cv2\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nfrom tensorflow.keras.callbacks import EarlyStopping\n\nprint(f\"Tensor Flow version: {tf.__version__}\")\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"# Check for TPU or GPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"try: # detect TPUs\n#     tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept ValueError: # detect GPU\n    strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    # strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    # strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nprint(f\"Number of Accelerators: {strategy.num_replicas_in_sync}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Get the GCS bucket"},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path('tfrecords-for-adl-wustl-fall-2020')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Check the contents of the bucket."},{"metadata":{"trusted":true},"cell_type":"code","source":"!gsutil ls $GCS_PATH","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Select the Image Size and set Learning Rate Schedule"},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 24\nIMAGE_SIZE = [192,192]\n\nBLOCKS_TRAIN_DATASETS = { # avialable image sizes\n    192: GCS_PATH + '/Train/192x192/*.tfrec',\n    331: GCS_PATH + '/Train/331x331/*.tfrec',\n}\nCLASSES = [0,1]\nassert IMAGE_SIZE[0] == IMAGE_SIZE[1], \"only square images are supported\"\nassert IMAGE_SIZE[0] in BLOCKS_TRAIN_DATASETS, \"this image size is not supported\"\n\n# learning rate schedule for TPU, GPU and CPU.\n# using a LR ramp up because fine-tuning a pre-trained model.\n# startin with a high LR would break the pre-trained weights.\n\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync # this is 8 on TPU v3-8, it is 1 on CPU and GPU\nLR_START = 0.00001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng,y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Visualisation Ulities"},{"metadata":{"trusted":true},"cell_type":"code","source":"def dataset_to_numpy_util(dataset, N):\n    dataset = dataset.unbatch().batch(N)\n    for images, labels in dataset:\n        numpy_images = images.numpy()\n        numpy_labels = labels.numpy()\n        break;  \n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(\n        CLASSES[int(label)], \n        'OK' if correct else 'NO', \n        u\"\\u2192\" if not correct else '',\n        CLASSES[correct_label] if not correct else ''\n    ), correct\n\ndef display_one_flower(image, title, subplot, red=False):\n    plt.subplot(subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    plt.title(title, fontsize=16, color='red' if red else 'black')\n    return subplot+1\n  \ndef display_9_images_from_dataset(dataset):\n    subplot=331\n    plt.figure(figsize=(13,13))\n    images, labels = dataset_to_numpy_util(dataset, 9)\n    for i, image in enumerate(images):\n        title = CLASSES[labels[i]]\n        subplot = display_one_flower(image, title, subplot)\n        if i >= 8:\n            break;\n              \n    #plt.tight_layout()\n    plt.subplots_adjust(wspace=0.1, hspace=0.1)\n    plt.show()  \n\ndef display_9_images_with_predictions(images, predictions, labels):\n    subplot=331\n    plt.figure(figsize=(13,13))\n    for i, image in enumerate(images):\n        title, correct = title_from_label_and_target(predictions[i], labels[i])\n        subplot = display_one_flower(image, title, subplot, not correct)\n        if i >= 8:\n            break;\n              \n    #plt.tight_layout()\n    plt.subplots_adjust(wspace=0.1, hspace=0.1)\n    plt.show()\n    \ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        #plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read images and labels from TFRecords"},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\ngcs_pattern = BLOCKS_TRAIN_DATASETS[IMAGE_SIZE[0]]\nvalidation_split = 0.20\nfilenames = tf.io.gfile.glob(gcs_pattern)\nsplit = len(filenames) - int(len(filenames) * validation_split)\nTRAINING_FILENAMES = filenames[:split]\nVALIDATION_FILENAMES = filenames[split:]\nTRAIN_STEPS = count_data_items(TRAINING_FILENAMES) // BATCH_SIZE\nprint(\"TRAINING IAGES; \", count_data_items(TRAINING_FILENAMES), \", STEPS PER EPOCH: \", TRAIN_STEPS)\nprint(\"VALIDATION IMAGES: \", count_data_items(VALIDATION_FILENAMES))\n\ndef read_tfrecord(example):\n    features = {\n        \"image\": tf.io.FixedLenFeature([],tf.string),\n        \"id\": tf.io.FixedLenFeature([], tf.int64),\n        \"filename\": tf.io.FixedLenFeature([],tf.string),\n        \"stable\": tf.io.FixedLenFeature([], tf.int64),\n    }\n    example = tf.io.parse_single_example(example, features)\n    image = tf.image.decode_jpeg(example[\"image\"], channels=3)\n    image = tf.cast(image,tf.float32) / 255.0 # convert image to floats in [0, 1] range\n    target = tf.cast(example[\"stable\"],tf.int32)\n    return image, target\n\ndef force_image_sizes(dataset, image_size):\n    # explicit size need for TPU\n    reshape_images = lambda image, label: (tf.reshape(image, [*image_size, 3]), label)\n    dataset = dataset.map(reshape_images, num_parallel_calls=AUTO)\n    return dataset\n\ndef load_dataset(filenames):\n    # read from TFRecords. For optimal performance, reading from multiplie files at once\n    # and disregarding data order. Order does not matter since we will suffle the data away.\n    \n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False\n    \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTO)\n    dataset = force_image_sizes(dataset, IMAGE_SIZE)\n    return dataset\n\ndef data_augment(image, target):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    # image = tf.image.random_saturation(image, 0, 2)\n    # random brightness/exposure?\n    return image, target \n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset():\n    dataset = load_dataset(VALIDATION_FILENAMES)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training and Validation Datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"training_dataset = get_training_dataset()\nvalidation_dataset = get_validation_dataset()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Display a set of the training images from the TFRecords."},{"metadata":{"trusted":true},"cell_type":"code","source":"display_9_images_from_dataset(validation_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model():\n    #pretrained_model = tf.keras.applications.MobileNetV2(input_shape=[*IMAGE_SIZE, 3], include_top=False)\n    pretrained_model = tf.keras.applications.Xception(input_shape=[*IMAGE_SIZE, 3], include_top=False)\n    #pretrained_model = tf.keras.applications.VGG16(weights='imagenet', include_top=False ,input_shape=[*IMAGE_SIZE, 3])\n    #pretrained_model = tf.keras.applications.ResNet50(weights='imagenet', include_top=False, input_shape=[*IMAGE_SIZE, 3])\n    #pretrained_model = tf.keras.applications.MobileNet(weights='imagenet', include_top=False, input_shape=[*IMAGE_SIZE, 3])\n    # EfficientNet can be loaded through efficientnet.tfkeras library (https://github.com/qubvel/efficientnet)\n    #pretrained_model = efficientnet.tfkeras.EfficientNetB0(weights='imagenet', include_top=False)\n    \n    pretrained_model.trainable = True\n\n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        #tf.keras.layers.Flatten(),\n        tf.keras.layers.Dense(1, activation='sigmoid')\n    ])\n\n    model.compile(\n        optimizer='adam',\n        loss = 'binary_crossentropy',\n        metrics=['accuracy']\n    )\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Check model summary."},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = create_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Set early stopping criteria and train model."},{"metadata":{"trusted":true},"cell_type":"code","source":"es = EarlyStopping(monitor='val_loss', min_delta=1e-4, patience=10, verbose=1, mode='auto',\n        restore_best_weights=True)\n\ncallback_list = [es, lr_callback]\n\nhistory = model.fit(training_dataset, validation_data=validation_dataset,\n                    steps_per_epoch=TRAIN_STEPS, epochs=EPOCHS, callbacks=callback_list)\n\nfinal_accuracy = history.history[\"val_accuracy\"][-1:]\nprint(\"FINAL ACCURACY MEAN-1: \", np.mean(final_accuracy))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_training_curves(history.history['accuracy'][0:], history.history['val_accuracy'][0:], 'accuracy', 211)\ndisplay_training_curves(history.history['loss'][0:], history.history['val_loss'][0:], 'loss', 212)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # a couple of images to test predictions too\n# some_flowers, some_labels = dataset_to_numpy_util(validation_dataset, 160)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # randomize the input so that you can execute multiple times to change results\n# permutation = np.random.permutation(8*20)\n# some_flowers, some_labels = (some_flowers[permutation], some_labels[permutation])\n\n# predictions = model.predict(some_flowers, batch_size=16)\n# evaluations = model.evaluate(some_flowers, some_labels, batch_size=16)\n  \n# print(predictions.tolist())\n# print('[val_loss, val_acc]', evaluations)\n\n# display_9_images_with_predictions(some_flowers, predictions, some_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # TPUs need this extra setting to save to local disk, otherwise, they can only save models to GCS (Google Cloud Storage).\n# # The setting instructs Tensorflow to retrieve all parameters from the TPU then do the saving from the local VM, not the TPU.\n# save_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\n# model.save('./model', options=save_locally) # saving in Tensorflow's \"saved model\" format","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Save the model.\n\nThe above cell (using experiement_io_device) didnt work for me, however the simple model.save() seems to work fine."},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('model.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make predictions on Test dataset"},{"metadata":{},"cell_type":"markdown","source":"Set the path for the test dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"BLOCKS_TEST_DATASETS = { # avialable image sizes\n    192: GCS_PATH + '/Test/192x192/*.tfrec',\n    331: GCS_PATH + '/Test/331x331/*.tfrec',\n}\nCLASSES = [0,1]\nassert IMAGE_SIZE[0] == IMAGE_SIZE[1], \"only square images are supported\"\nassert IMAGE_SIZE[0] in BLOCKS_TEST_DATASETS, \"this image size is not supported\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gcs_pattern = BLOCKS_TEST_DATASETS[IMAGE_SIZE[0]]\n\n\nTEST_FILENAMES = tf.io.gfile.glob(gcs_pattern)\nprint(\"TEST IMAGES; \", count_data_items(TEST_FILENAMES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Helper function to read the test datset. Its important to keep the order for the test data."},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_test_tfrecord(example):\n    features = {\n        \"image\": tf.io.FixedLenFeature([],tf.string),\n        \"id\": tf.io.FixedLenFeature([], tf.int64),\n        \"filename\": tf.io.FixedLenFeature([],tf.string),\n    }\n    example = tf.io.parse_single_example(example, features)\n    image = tf.image.decode_jpeg(example[\"image\"], channels=3)\n    image = tf.cast(image,tf.float32) / 255.0 # convert image to floats in [0, 1] range\n    return image\n\ndef force_test_image_sizes(dataset, image_size):\n    # explicit size need for TPU\n    reshape_images = lambda image: tf.reshape(image, [*image_size, 3])\n    dataset = dataset.map(reshape_images, num_parallel_calls=AUTO)\n    return dataset\n\ndef load_test_dataset(filenames):\n    # read from TFRecords. For optimal performance, reading from multiplie files at once\n    # and disregarding data order. Order does not matter since we will suffle the data away.\n    \n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = True # want in order\n    \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_test_tfrecord, num_parallel_calls=AUTO)\n    dataset = force_test_image_sizes(dataset, IMAGE_SIZE)\n    return dataset\n\ndef get_test_dataset():\n    dataset = load_test_dataset(TEST_FILENAMES)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Make predictions on the test dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = get_test_dataset()\ny_pred = model.predict(test_dataset, batch_size=16)\ny_pred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Helper functions to read the id for the test data. Its important to keep the order for the test data."},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_dataset_to_numpy_util(dataset): #, N):\n    dataset = dataset #.unbatch() #.batch(N)\n    for IDs in dataset:\n        numpy_IDs = IDs.numpy()\n        break;  \n    return numpy_IDs\n\ndef read_test_tfrecord_id(example):\n    features = {\n        \"image\": tf.io.FixedLenFeature([],tf.string),\n        \"id\": tf.io.FixedLenFeature([], tf.int64),\n        \"filename\": tf.io.FixedLenFeature([],tf.string),\n    }\n    example = tf.io.parse_single_example(example, features)\n    ID = example[\"id\"]\n    return ID\n\ndef load_test_id(filenames):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = True # want in order\n    \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_test_tfrecord_id, num_parallel_calls=AUTO)\n    return dataset\n    \ndef get_test_id():\n    dataset = load_test_id(TEST_FILENAMES)\n    return dataset\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_id_dataset = get_test_id()\ntest_id = test_dataset_to_numpy_util(test_id_dataset)\nx = [IDs.numpy() for IDs in test_id_dataset]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Read the test meta-data file and set the submission file to have the same ordering."},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = \"/kaggle/input/applications-of-deep-learning-wustl-fall-2020/final-kaggle-data/\"\n\nPATH_TEST = os.path.join(PATH, \"test.csv\")\n\ndf_test = pd.read_csv(PATH_TEST)\n\ndf_test.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submit = pd.DataFrame({\"id\":np.array(x).flatten(), \"stable\":y_pred.flatten()})\ndf_submit = df_test.merge(df_submit, how='inner')\ndf_submit.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submit.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submit.to_csv(\"/kaggle/working/submit.csv\",index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}