{"cells":[{"metadata":{},"cell_type":"markdown","source":"# How To Create TFRecords\nIn this notebook i ging to show you how i've created my own TFrecords for cassava leaf [competition](https://www.kaggle.com/c/cassava-leaf-disease-classification/data), i did this because i want to learn more about tfrecords and TPU processing. This notebook contain, how to create TFRecords and how to upload them to google cloud storage."},{"metadata":{},"cell_type":"markdown","source":"> # Load Meta Data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# LOAD LIBRARIES\nfrom sklearn.model_selection import StratifiedKFold\nimport numpy as np, pandas as pd, os\nimport matplotlib.pyplot as plt, cv2\nimport tensorflow as tf, re, math\nimport pandas_profiling as pp\nfrom PIL import Image\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport plotly.express as px\nimport plotly.figure_factory as ff\n\ndef plot_distribution(df_table):\n    temp = df_table.copy()\n    temp[\"count\"] = 1\n    temp = temp[[\"label\", \"count\"]].groupby([\"label\"]).sum()\n    class_count = temp.reset_index()\n    fig = px.bar(class_count, x=\"label\", y=\"count\", title=\"Count per leaf desease\",text=class_count['count'])\n    fig.update_traces(texttemplate='%{text:.2s}',textposition='outside')\n    fig.update_layout(template= \"plotly_dark\" , \n                      xaxis = dict(title = \"Cassava Leaf Desease\"))\n    fig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# PATHS TO IMAGES\nPATH = '../input/cassava-leaf-disease-merged/train/'\nIMGS = os.listdir(PATH)\nprint('There are {} train images '.format(len(IMGS)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# LOAD TRAIN META DATA\ndf = pd.read_csv('../input/cassava-leaf-disease-merged/merged.csv')\ndf.rename({'image_id':'image_name'},axis=1,inplace=True)\ndf.tail()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Connect to GCS\n\nYou have to create a projecto on google cloud platform, only pay for storage\n* https://cloud.google.com/storage/docs/\n\nThen you need to activate the Add-ons menu buttom above \"Google cloud services\""},{"metadata":{"trusted":true},"cell_type":"code","source":"ID_PROJECT = 'spring-paratext-297605'\nfrom google.cloud import storage\nstorage_client = storage.Client(project=ID_PROJECT)\n\ndef create_bucket(dataset_name):\n    \"\"\"Creates a new bucket. https://cloud.google.com/storage/docs/ \"\"\"\n    bucket = storage_client.create_bucket(dataset_name)\n    print('Bucket {} created'.format(bucket.name))\n\ndef upload_blob(bucket_name, source_file_name, destination_blob_name):\n    \"\"\"Uploads a file to the bucket. https://cloud.google.com/storage/docs/ \"\"\"\n    bucket = storage_client.get_bucket(bucket_name)\n    blob = bucket.blob(destination_blob_name)\n    blob.upload_from_filename(source_file_name)\n    print('File {} uploaded to {}.'.format(\n        source_file_name,\n        destination_blob_name))\n    \ndef list_blobs(bucket_name):\n    \"\"\"Lists all the blobs in the bucket. https://cloud.google.com/storage/docs/\"\"\"\n    blobs = storage_client.list_blobs(bucket_name)\n    for blob in blobs:\n        print(blob.name)\n        \ndef download_to_kaggle(bucket_name,destination_directory,file_name):\n    \"\"\"Takes the data from your GCS Bucket and puts it into the working directory of your Kaggle notebook\"\"\"\n    os.makedirs(destination_directory, exist_ok = True)\n    full_file_path = os.path.join(destination_directory, file_name)\n    blobs = storage_client.list_blobs(bucket_name)\n    for blob in blobs:\n        blob.download_to_filename(full_file_path)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create a New Bucket\nOnly if you want to. Or you only need to call your current bucket"},{"metadata":{"trusted":true},"cell_type":"code","source":"BUCKET_NAME = 'cassava_tfrecords_merge_kfold'         \ntry:\n    create_bucket(BUCKET_NAME)   \nexcept:\n    print(\"Error: truying to create the bucket {}\".format(BUCKET_NAME))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Write TFRecords - Train\nAll the code below comes from TensorFlow's docs [here][1]\n\n[1]: https://www.tensorflow.org/tutorials/load_data/tfrecord"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _float_feature_array(value):## This is for storage arrays\n  \"\"\"Returns a float_list from array\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def serialize_example(feature0, feature1, feature2, feature3,):\n  feature = {\n      'image': _bytes_feature(feature0),\n      'image_name': _bytes_feature(feature1),\n      'label': _float_feature(feature2),\n      'hotencode': _float_feature_array(feature3),\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\nenc = OneHotEncoder(handle_unknown='ignore')\nenc.fit(df[[\"label\"]])\n#enc.transform([[3]]).toarray()[0].tobytes()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Sorting images in IMGS for Stratify kfold"},{"metadata":{"trusted":true},"cell_type":"code","source":"n_splits = 50\nskf = StratifiedKFold(n_splits=n_splits)\nsort_list = []\nfor ind_k, (index_train, index_val) in enumerate(skf.split(df[\"image_name\"], df[\"label\"])):\n    sort_list = sort_list + index_val.tolist()\n    #plot_distribution(df.iloc[index_val])\ndf2 = df.iloc[sort_list].reset_index()\nIMGS =df2.image_name.tolist()\n\nplot_distribution(df.iloc[index_val])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true,"collapsed":true},"cell_type":"code","source":"SIZE = len(IMGS)//n_splits \nCT = len(IMGS)//SIZE + int(len(IMGS)%SIZE!=0)\nfor j in range(CT):\n    print(); print('Writing TFRecord {} of {}...'.format(j,CT))\n    CT2 = min(SIZE,len(IMGS)-j*SIZE)\n    tfrecord_name = 'train{:.0f}-{:.0f}.tfrec'.format(j,CT2)\n    source_tfrecord = \"./\" + tfrecord_name\n    with tf.io.TFRecordWriter(tfrecord_name) as writer:\n        for k in range(CT2):\n            img = cv2.imread(PATH+IMGS[SIZE*j+k])\n            #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # Fix incorrect colors\n            img = cv2.imencode('.jpeg', img,(cv2.IMWRITE_JPEG_QUALITY, 100 ) )[1].tobytes() #\n            name = IMGS[SIZE*j+k]\n            row = df.loc[df.image_name==name]\n            example = serialize_example(\n                                        img, \n                                        str.encode(name),\n                                        row.label.values[0],\n                                        enc.transform([[row.label.values[0]]]).toarray()[0],\n            )\n            writer.write(example)\n            if k%100==0: print(k,', ',end='')\n        #upload_blob(BUCKET_NAME, source_tfrecord, tfrecord_name)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Maybe you want to upload all tfrecord after they'll be ready"},{"metadata":{"trusted":true,"collapsed":true,"_kg_hide-output":true},"cell_type":"code","source":"import glob\nfilenames = glob.glob(\"./*.tfrec\")\ndestination = [filename.split(\"/\")[-1] for filename in filenames]\n\nfor source, destination in zip(filenames, destination):\n    upload_blob(BUCKET_NAME, source, destination)\nprint('Data inside of',BUCKET_NAME,':')\n#list_blobs(BUCKET_NAME)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### see all tfrecord created"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"! ls -l","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Verify TFRecords\nWe will verify the TFRecords we just made by using code from the Flower Comp starter notebook [here][1] to display the TFRecords below.\n\n[1]: https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu"},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\nIMAGE_SIZE = [456, 456]\n\nCLASSES = 5\nbatch_size = 32\nAUG_BATCH = batch_size\npath_ = './*.tfrec'\nvalidation_split = 0.50\nfilenames = tf.io.gfile.glob(path_)\nsplit = len(filenames) - int(len(filenames) * validation_split)\ntrain_fns = filenames[:split]\nvalidation_fns = filenames[split:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def parse_tfrecord(example):\n  features = {\n    \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n    \"label\": tf.io.VarLenFeature(tf.float32), #tf.io.FixedLenFeature([], tf.int64),  # shape [] means scalar\n    \"hotencode\": tf.io.VarLenFeature(tf.float32),\n  }\n  example = tf.io.parse_single_example(example, features)\n  decoded = tf.io.decode_jpeg(example['image'], channels=3)\n  #img_channel_swap = decoded[..., ::-1]\n  #img_channel_swap_1 = tf.reverse(decoded, axis=[-1])\n  normalized = tf.cast(decoded, tf.float32) / 255.0 # convert each 0-255 value to floats in [0, 1] range\n  image_tensor = tf.image.resize( normalized, [600,800], method = \"bilinear\", preserve_aspect_ratio=True)\n  one_hot_class = tf.reshape(tf.sparse.to_dense(example['hotencode']), [CLASSES])\n  return normalized, one_hot_class\n\ndef load_dataset(filenames_):\n  # Read from TFRecords. For optimal performance, we interleave reads from multiple files.\n  records = tf.data.TFRecordDataset(filenames_, num_parallel_reads=AUTO)\n  return records.map(parse_tfrecord, num_parallel_calls=AUTO)\n\ndef get_val_dataset(filenames_val):\n    dataset = load_dataset(filenames_val)\n    \n    def catch_image_size(image, one_hot_class):\n        image_tensor = tf.image.resize( image, [*IMAGE_SIZE], method = \"bilinear\")\n        return image_tensor, one_hot_class\n    \n    return dataset.map(catch_image_size,num_parallel_calls=AUTO).batch(batch_size).prefetch(AUTO)\n        \ndef get_training_dataset(filenames_train):\n  dataset = load_dataset(filenames_train)\n  # Create some additional training images by randomly flipping and\n  # increasing/decreasing the saturation of images in the training set. \n  def data_augment(image, one_hot_class):\n    modified = tf.image.random_crop(image,size = [*IMAGE_SIZE, 3])    \n    modified = tf.image.random_flip_left_right(modified)\n    modified = tf.image.random_saturation(modified, 0, 2)\n    modified = tf.image.random_brightness(modified, 0.1)\n    modified = tf.image.random_flip_up_down(modified)\n    modified = tf.image.random_jpeg_quality(modified, 80 ,100)\n    return modified, one_hot_class\n  augmented = dataset.map(data_augment, num_parallel_calls=AUTO)\n\n  # Prefetch the next batch while training (autotune prefetch buffer size).\n  return augmented.repeat().shuffle(2048).batch(batch_size).prefetch(AUTO) \n\ntraining_dataset = get_training_dataset(train_fns)\nvalidation_dataset = get_val_dataset(validation_fns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSES_ = ['CBB', 'CBSD', 'CGM', 'CMD', 'Healthy']\n\ndef display_one_leaf(image, title, subplot, color):\n  plt.subplot(subplot)\n  plt.axis('off')\n  plt.imshow(image)\n  plt.title(title, fontsize=16, color=color)\n  \n# If model is provided, use it to generate predictions.\ndef display_nine_leafs(images, titles, title_colors=None):\n  subplot = 331\n  plt.figure(figsize=(13,13))\n  for i in range(9):\n    color = 'black' if title_colors is None else title_colors[i]\n    display_one_leaf(images[i], titles[i], 331+i, color)\n  plt.tight_layout()\n  plt.subplots_adjust(wspace=0.1, hspace=0.1)\n  plt.show()\n\ndef get_dataset_iterator(dataset, n_examples):\n  return dataset.unbatch().batch(n_examples).as_numpy_iterator()\n\ntraining_viz_iterator = get_dataset_iterator(training_dataset, 9)\nval_viz_iterator = get_dataset_iterator(validation_dataset, 9)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Viz"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Re-run this cell to show a new batch of images\nimages, classes = next(training_viz_iterator)\nclass_idxs = np.argmax(classes, axis=-1) # transform from one-hot array to class number\nlabels = [CLASSES_[idx] for idx in class_idxs]\ndisplay_nine_leafs(images, labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test Viz"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Re-run this cell to show a new batch of images\nimages, classes = next(val_viz_iterator)\nclass_idxs = np.argmax(classes, axis=-1) # transform from one-hot array to class number\nlabels = [CLASSES_[idx] for idx in class_idxs]\ndisplay_nine_leafs(images, labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}