{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<div style=\"font-size:20px\"><p>\nTensor Processing Units (TPUs) are Google’s custom-developed application-specific integrated circuits (ASICs) used to accelerate machine learning workloads previous algorithm take took weeks to train on GPUs can even be trained in hours using TPUs.</p><p> We can use TPUs from cloud services like google cloud or for free from Kaggle or google colab In this tutorial we will see how can we use TPU on TFRecords. TFRecords is a file format optimized for TensorFlow that is used for storing a sequence of binary records. It is very useful for large dataset since only data that is required is loaded in batches</p>\n</div>","metadata":{}},{"cell_type":"markdown","source":"# Importing Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nimport keras\nimport keras.layers as L","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:46:04.089710Z","iopub.execute_input":"2021-07-26T22:46:04.090213Z","iopub.status.idle":"2021-07-26T22:46:04.096721Z","shell.execute_reply.started":"2021-07-26T22:46:04.090175Z","shell.execute_reply":"2021-07-26T22:46:04.095508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(\"Device:\", tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint(\"Number of replicas:\", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:46:04.222460Z","iopub.execute_input":"2021-07-26T22:46:04.222856Z","iopub.status.idle":"2021-07-26T22:46:09.971070Z","shell.execute_reply.started":"2021-07-26T22:46:04.222821Z","shell.execute_reply":"2021-07-26T22:46:09.969986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To use TPU, we must load our data from GCS to that we need GCS path of our dataset which we can get using KaggleDatasets","metadata":{}},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:46:09.972835Z","iopub.execute_input":"2021-07-26T22:46:09.973171Z","iopub.status.idle":"2021-07-26T22:46:09.981785Z","shell.execute_reply.started":"2021-07-26T22:46:09.973137Z","shell.execute_reply":"2021-07-26T22:46:09.980215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nGCS_DS_Path = KaggleDatasets().get_gcs_path('tpu-getting-started')\nprint(GCS_DS_Path)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:46:09.984220Z","iopub.execute_input":"2021-07-26T22:46:09.984563Z","iopub.status.idle":"2021-07-26T22:46:10.419719Z","shell.execute_reply.started":"2021-07-26T22:46:09.984532Z","shell.execute_reply":"2021-07-26T22:46:10.418435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = [224,224]\nGCS_PATH = GCS_DS_Path + '/tfrecords-jpeg-224x224'","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:46:10.422218Z","iopub.execute_input":"2021-07-26T22:46:10.422715Z","iopub.status.idle":"2021-07-26T22:46:10.427896Z","shell.execute_reply.started":"2021-07-26T22:46:10.422667Z","shell.execute_reply":"2021-07-26T22:46:10.426643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_file = tf.io.gfile.glob(GCS_PATH+'/train/*.tfrec') \ntest_file = tf.io.gfile.glob(GCS_PATH+'/test/*.tfrec')\nvalid_file = tf.io.gfile.glob(GCS_PATH+'/val/*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:46:10.430335Z","iopub.execute_input":"2021-07-26T22:46:10.430984Z","iopub.status.idle":"2021-07-26T22:46:10.666860Z","shell.execute_reply.started":"2021-07-26T22:46:10.430934Z","shell.execute_reply":"2021-07-26T22:46:10.665382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing Data","metadata":{}},{"cell_type":"markdown","source":"To make TFRecords dataset we will use TFRecordDataset Class available in tf.data module","metadata":{}},{"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0 \n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:46:10.671287Z","iopub.execute_input":"2021-07-26T22:46:10.671703Z","iopub.status.idle":"2021-07-26T22:46:10.685235Z","shell.execute_reply.started":"2021-07-26T22:46:10.671666Z","shell.execute_reply":"2021-07-26T22:46:10.684049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef get_training_dataset():\n    dataset = load_dataset(training_file, labeled=True)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(valid_file, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(test_file, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:46:10.686653Z","iopub.execute_input":"2021-07-26T22:46:10.686989Z","iopub.status.idle":"2021-07-26T22:46:10.705942Z","shell.execute_reply.started":"2021-07-26T22:46:10.686957Z","shell.execute_reply":"2021-07-26T22:46:10.704404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\nds_train = get_training_dataset()\nds_valid = get_validation_dataset()\nds_test = get_test_dataset()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:46:10.708821Z","iopub.execute_input":"2021-07-26T22:46:10.709161Z","iopub.status.idle":"2021-07-26T22:46:10.803813Z","shell.execute_reply.started":"2021-07-26T22:46:10.709122Z","shell.execute_reply":"2021-07-26T22:46:10.802862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_iter = iter(ds_train.unbatch().batch(20))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:46:10.805899Z","iopub.execute_input":"2021-07-26T22:46:10.806723Z","iopub.status.idle":"2021-07-26T22:46:10.824480Z","shell.execute_reply.started":"2021-07-26T22:46:10.806673Z","shell.execute_reply":"2021-07-26T22:46:10.823215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_batch = next(ds_iter)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:46:10.826063Z","iopub.execute_input":"2021-07-26T22:46:10.826787Z","iopub.status.idle":"2021-07-26T22:46:11.450962Z","shell.execute_reply.started":"2021-07-26T22:46:10.826732Z","shell.execute_reply":"2021-07-26T22:46:11.449903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1,20):\n    plt.subplot(4,5,i)\n    plt.imshow(one_batch[0][i],aspect='auto')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:46:11.454987Z","iopub.execute_input":"2021-07-26T22:46:11.455393Z","iopub.status.idle":"2021-07-26T22:46:13.460248Z","shell.execute_reply.started":"2021-07-26T22:46:11.455357Z","shell.execute_reply":"2021-07-26T22:46:13.459169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"markdown","source":"<p style=\"font-size:18px\">VGG16 is a convolutional neural network model proposed by K. Simonyan and A. Zisserman from the University of Oxford in the paper “Very Deep Convolutional Networks for Large-Scale Image Recognition”. The model achieves 92.7% top-5 test accuracy in ImageNet, which is a dataset of over 14 million images belonging to 1000 classes.</p>","metadata":{}},{"cell_type":"markdown","source":"<img src=\"https://miro.medium.com/max/2268/1*CrjJwSX9S7f759dK2EtGJQ.png\">","metadata":{}},{"cell_type":"markdown","source":"Helper Function to make ConvBlock","metadata":{}},{"cell_type":"code","source":"def convblock(filter_size,is_block2=False):\n    model.add(L.Conv2D(filter_size,kernel_size=(3,3),padding='same',activation='relu'))\n    model.add(L.Conv2D(filter_size,kernel_size=(3,3),padding='same',activation='relu'))\n    if is_block2:\n        model.add(L.Conv2D(filter_size,kernel_size=(3,3),padding='same',activation='relu'))\n    model.add(L.MaxPool2D(pool_size=(2,2),strides=(2,2),padding='same'))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:46:13.461729Z","iopub.execute_input":"2021-07-26T22:46:13.462311Z","iopub.status.idle":"2021-07-26T22:46:13.471090Z","shell.execute_reply.started":"2021-07-26T22:46:13.462267Z","shell.execute_reply":"2021-07-26T22:46:13.469993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = keras.utils.get_file('vgg16_weights','https://storage.googleapis.com/tensorflow/keras-applications/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels.h5')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:46:13.472356Z","iopub.execute_input":"2021-07-26T22:46:13.472796Z","iopub.status.idle":"2021-07-26T22:46:13.484633Z","shell.execute_reply.started":"2021-07-26T22:46:13.472751Z","shell.execute_reply":"2021-07-26T22:46:13.483805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model = keras.Sequential()\n    model.add(L.InputLayer(input_shape=(224,224,3)))\n    convblock(64)\n    \n    convblock(128)\n    \n    convblock(256,is_block2=True)\n\n    convblock(512,is_block2=True)\n    \n    convblock(512,is_block2=True)\n    model.add(L.Flatten())\n    model.add(L.Dense(4096,activation='relu'))\n    model.add(L.Dense(4096,activation='relu'))\n    model.add(L.Dense(1000,activation='relu'))\n    model.load_weights(weights)\n    for Layers in model.layers:\n        Layers.trainable = False\n    model.add(L.Dense(104, activation='softmax')) # since our dataset have 104 classes","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:07:14.110484Z","iopub.execute_input":"2021-07-26T23:07:14.111280Z","iopub.status.idle":"2021-07-26T23:07:24.208613Z","shell.execute_reply.started":"2021-07-26T23:07:14.111184Z","shell.execute_reply":"2021-07-26T23:07:24.207723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:07:30.992489Z","iopub.execute_input":"2021-07-26T23:07:30.993057Z","iopub.status.idle":"2021-07-26T23:07:31.009367Z","shell.execute_reply.started":"2021-07-26T23:07:30.993009Z","shell.execute_reply":"2021-07-26T23:07:31.005987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    optimizer=keras.optimizers.Adam(learning_rate=0.001),\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy'],\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:07:34.652436Z","iopub.execute_input":"2021-07-26T23:07:34.652822Z","iopub.status.idle":"2021-07-26T23:07:34.686712Z","shell.execute_reply.started":"2021-07-26T23:07:34.652790Z","shell.execute_reply":"2021-07-26T23:07:34.685675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_TRAINING_IMAGES = 12753\nNUM_TEST_IMAGES = 7382\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:07:34.913804Z","iopub.execute_input":"2021-07-26T23:07:34.914159Z","iopub.status.idle":"2021-07-26T23:07:34.919181Z","shell.execute_reply.started":"2021-07-26T23:07:34.914128Z","shell.execute_reply":"2021-07-26T23:07:34.918020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    ds_train,\n    validation_data=ds_valid,\n    epochs=50,steps_per_epoch=STEPS_PER_EPOCH\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:07:35.239236Z","iopub.execute_input":"2021-07-26T23:07:35.239620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True)\n\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T22:28:44.143134Z","iopub.status.idle":"2021-07-26T22:28:44.143704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Generating submission.csv file...')\n\n# Get image ids from test set and convert to unicode\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\n# Write the submission file\nnp.savetxt(\n    'submission.csv',\n    np.rec.fromarrays([test_ids, predictions]),\n    fmt=['%s', '%d'],\n    delimiter=',',\n    header='id,label',\n    comments='',\n)\n\n# Look at the first few predictions\n!head submission.csv","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Refrences\n1. )https://www.kaggle.com/ryanholbrook/create-your-first-submission\n2. )https://neurohive.io/en/popular-networks/vgg16/\n3. )https://cloud.google.com/tpu\n4. )https://en.wikipedia.org/wiki/Tensor_Processing_Unit\n5. )https://medium.com/mostly-ai/tensorflow-records-what-they-are-and-how-to-use-them-c46bc4bbb564 ","metadata":{}}]}