{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import cv2, math, os, sys\nfrom tqdm import tqdm_notebook as tqdm\nimport zipfile\nimport io\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport warnings\nimport tensorflow as tf\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE # used in tf.data.Dataset API\nwarnings.filterwarnings(\"ignore\")\nPATH_IMAGE = '/kaggle/input/testimagesgraph/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir /kaggle/working/test-tfrecords","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_OUTPUT ='/kaggle/working/test-tfrecords/' # prefix for output file names\nSHARDS = 12\nTARGET_SIZE = [64, 64]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_and_crop_image(image, label):\n  w = tf.shape(image)[0]\n  h = tf.shape(image)[1]\n  tw = TARGET_SIZE[1]\n  th = TARGET_SIZE[0]\n  resize_crit = (w * th) / (h * tw)\n  image = tf.cond(resize_crit < 1,\n                  lambda: tf.image.resize(image, [w*tw/w, h*tw/w]), # if true\n                  lambda: tf.image.resize(image, [w*th/h, h*th/h])  # if false\n                 )\n  nw = tf.shape(image)[0]\n  nh = tf.shape(image)[1]\n  image = tf.image.crop_to_bounding_box(image, (nw - tw) // 2, (nh - th) // 2, tw, th)\n  return image, label\n  \n\ndef decode_jpeg_and_label(filename):\n  bits = tf.io.read_file(filename)\n  image = tf.image.decode_jpeg(bits)\n  # parse flower name from containing directory\n  label = tf.strings.split(tf.expand_dims(filename, axis=-1), sep='/')\n  label = label.values[-1]\n  return image, label\n\nfilenames = tf.data.Dataset.list_files(PATH_IMAGE + '*.png', seed=35155) # This also shuffles the images\ndataset1 = filenames.map(decode_jpeg_and_label, num_parallel_calls=AUTO)\ndataset1 = dataset1.map(resize_and_crop_image, num_parallel_calls=AUTO) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_9_images_from_dataset(dataset):\n  plt.figure(figsize=(13,13))\n  subplot=331\n  for i, (image, label) in enumerate(dataset):\n    img = image.numpy()\n    img = cv2.resize(img,(64,64))\n    plt.subplot(subplot)\n    plt.axis('off')\n    plt.imshow(img.astype('float32'))\n    plt.title(label.numpy().decode(\"utf-8\"), fontsize=16)\n    subplot += 1\n    if i==2:\n        break\n  plt.tight_layout()\n  plt.subplots_adjust(wspace=0.1, hspace=0.1)\n  plt.show()\ndisplay_9_images_from_dataset(dataset1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def recompress_image(image, label):\n  height = tf.shape(image)[0]\n  width = tf.shape(image)[1]\n  image = tf.cast(image, tf.uint8)\n  image = tf.image.encode_jpeg(image, optimize_size=True, chroma_downsampling=False)\n  return image, label, height, width\ndataset3 = dataset1.map(recompress_image, num_parallel_calls=AUTO)\ndataset3 = dataset3.batch(12) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytestring_feature(list_of_bytestrings):\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=list_of_bytestrings))\n\ndef _int_feature(list_of_ints): # int64\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=list_of_ints))\n\ndef _float_feature(list_of_floats): # float32\n  return tf.train.Feature(float_list=tf.train.FloatList(value=list_of_floats))\n  \n\ndef to_tfrecord(tfrec_filewriter, img_bytes, label):  \n  feature = {\n      \"image\": _bytestring_feature([img_bytes]), \n      \"label\":         _bytestring_feature([label])\n  }\n  return tf.train.Example(features=tf.train.Features(feature=feature))\n  \nprint(\"Writing TFRecords\")\nfor shard, (image, label, height, width) in enumerate(dataset3):\n  # batch size used as shard size here\n  shard_size = image.numpy().shape[0]\n  # good practice to have the number of records in the filename\n  filename = GCS_OUTPUT + \"test-{:02d}-{}.tfrec\".format(shard, shard_size)\n  \n  with tf.io.TFRecordWriter(filename) as out_file:\n        for i in range(shard_size):\n            example = to_tfrecord(out_file,\n                            image.numpy()[i], # re-compressed image: already a byte string\n                            label.numpy()[i])\n            out_file.write(example.SerializeToString())\n        print(\"Wrote file {} containing {} records\".format(filename, shard_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\nshutil.make_archive('test-frecords', 'zip', '/kaggle/working/test-tfrecords')\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}