{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import cv2, math, os, sys\nfrom tqdm import tqdm_notebook as tqdm\nimport zipfile\nimport io\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport warnings\nimport tensorflow as tf\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE # used in tf.data.Dataset API\nwarnings.filterwarnings(\"ignore\")\n\nPATH_IMAGE = '/kaggle/input/imagesgraph/'\ntrain_df = pd.read_csv('/kaggle/input/bengaliai-cv19/train.csv')\n\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir /kaggle/working/tfrecords","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_OUTPUT ='/kaggle/working/tfrecords/' # prefix for output file names\nSHARDS = 128\nTARGET_SIZE = [64, 64]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_and_crop_image(image, label):\n  w = tf.shape(image)[0]\n  h = tf.shape(image)[1]\n  tw = TARGET_SIZE[1]\n  th = TARGET_SIZE[0]\n  resize_crit = (w * th) / (h * tw)\n  image = tf.cond(resize_crit < 1,\n                  lambda: tf.image.resize(image, [w*tw/w, h*tw/w]), # if true\n                  lambda: tf.image.resize(image, [w*th/h, h*th/h])  # if false\n                 )\n  nw = tf.shape(image)[0]\n  nh = tf.shape(image)[1]\n  image = tf.image.crop_to_bounding_box(image, (nw - tw) // 2, (nh - th) // 2, tw, th)\n  return image, label\n  \n\ndef decode_jpeg_and_label(filename):\n  bits = tf.io.read_file(filename)\n  image = tf.image.decode_jpeg(bits)\n  label = tf.strings.split(tf.expand_dims(filename, axis=-1), sep='/')\n  label = label.values[-1]\n  return image, label\n\nfilenames = tf.data.Dataset.list_files(PATH_IMAGE + '*.png', seed=35155) # This also shuffles the images\ndataset1 = filenames.map(decode_jpeg_and_label, num_parallel_calls=AUTO)\ndataset1 = dataset1.map(resize_and_crop_image, num_parallel_calls=AUTO)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_9_images_from_dataset(dataset):\n  plt.figure(figsize=(13,13))\n  subplot=331\n  for i, (image, label) in enumerate(dataset):\n    img = image.numpy()\n    img = cv2.resize(img,(64,64))\n    plt.subplot(subplot)\n    plt.axis('off')\n    plt.imshow(img.astype('float32'))\n    plt.title(label.numpy().decode(\"utf-8\"), fontsize=16)\n    subplot += 1\n    if i==2:\n        break\n  plt.tight_layout()\n  plt.subplots_adjust(wspace=0.1, hspace=0.1)\n  plt.show()\ndisplay_9_images_from_dataset(dataset1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def recompress_image(image, label):\n  height = tf.shape(image)[0]\n  width = tf.shape(image)[1]\n  image = tf.cast(image, tf.uint8)\n  image = tf.image.encode_jpeg(image, optimize_size=True, chroma_downsampling=False)\n  return image, label, height, width\ndataset3 = dataset1.map(recompress_image, num_parallel_calls=AUTO)\ndataset3 = dataset3.batch(128) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# FOR EXAMPLE I GET 10 BATCH, BUT IF YOU WANH CONVERT ALL IMAGES YOU NEED DELETE THIS CALL\ndataset3 = dataset3.take(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"head_root_hot_classes =  [x for x in range(168)]\nhead_vowel_hot_classes =  [x  for x in range(11)]\nhead_consonant_hot_classes = [x  for x in range(7)]\ndef get_hot_class(CLASSES, label):\n    class_num = np.argmax(np.array(CLASSES)==label)\n    return np.eye(len(CLASSES))[class_num]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytestring_feature(list_of_bytestrings):\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=list_of_bytestrings))\n\ndef _int_feature(list_of_ints): # int64\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=list_of_ints))\n\ndef _float_feature(list_of_floats): # float32\n  return tf.train.Feature(float_list=tf.train.FloatList(value=list_of_floats))\n  \n\ndef to_tfrecord(tfrec_filewriter, img_bytes, label, height, width ,grapheme_root,vowel_diacritic,consonant_diacritic):  \n  head_root_hot =  get_hot_class(head_root_hot_classes, grapheme_root)\n  head_vowel_hot =  get_hot_class(head_vowel_hot_classes, vowel_diacritic)\n  head_consonant_hot =  get_hot_class(head_consonant_hot_classes, consonant_diacritic)\n\n  feature = {\n      \"image\": _bytestring_feature([img_bytes]), \n      \"grapheme_root\": _int_feature([grapheme_root]),       \n      \"vowel_diacritic\": _int_feature([vowel_diacritic]),       \n      \"consonant_diacritic\": _int_feature([consonant_diacritic]),  \n\n      \"label\":         _bytestring_feature([label]),         \n      \"size\":          _int_feature([height, width]),     \n      \"head_root_hot\": _float_feature(head_root_hot.tolist()),\n      \"head_vowel_hot\": _float_feature(head_vowel_hot.tolist()),\n      \"head_consonant_hot\": _float_feature(head_consonant_hot.tolist()),\n  }\n  return tf.train.Example(features=tf.train.Features(feature=feature))\n  \nprint(\"Writing TFRecords\")\nfor shard, (image, label, height, width) in enumerate(dataset3):\n  shard_size = image.numpy().shape[0]\n  filename = GCS_OUTPUT + \"{:02d}-{}.tfrec\".format(shard, shard_size)\n  \n  with tf.io.TFRecordWriter(filename) as out_file:\n    for i in range(shard_size):\n        image_name = label.numpy()[i].decode(\"utf-8\").split('.')[0]\n        grapheme_root = train_df.loc[train_df.image_id == image_name]['grapheme_root'].values[0]\n        vowel_diacritic = train_df.loc[train_df.image_id == image_name]['vowel_diacritic'].values[0]\n        consonant_diacritic = train_df.loc[train_df.image_id == image_name]['consonant_diacritic'].values[0]\n        example = to_tfrecord(out_file,\n                            image.numpy()[i], # re-compressed image: already a byte string\n                            label.numpy()[i],\n                            height.numpy()[i],\n                            width.numpy()[i],\n                            grapheme_root,\n                            vowel_diacritic,\n                            consonant_diacritic)\n        out_file.write(example.SerializeToString())\n    print(\"Wrote file {} containing {} records\".format(filename, shard_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_tfrecord(example):\n    features = {\n      \"image\": tf.io.FixedLenFeature([], tf.string), \n      \"grapheme_root\": tf.io.FixedLenFeature([], tf.int64),       \n      \"vowel_diacritic\": tf.io.FixedLenFeature([], tf.int64),       \n      \"consonant_diacritic\": tf.io.FixedLenFeature([], tf.int64),  \n\n      \"label\":         tf.io.FixedLenFeature([], tf.string),         \n      \"size\":          tf.io.FixedLenFeature([2], tf.int64),     \n      \"head_root_hot\": tf.io.VarLenFeature(tf.float32),\n      \"head_vowel_hot\": tf.io.VarLenFeature(tf.float32),\n      \"head_consonant_hot\": tf.io.VarLenFeature(tf.float32),\n    }\n\n    example = tf.io.parse_single_example(example, features)\n\n    image = tf.image.decode_image(example['image'], channels=3)\n    image = tf.cast(image, tf.float32)/255.0 \n    \n    grapheme_root = example['grapheme_root']\n    vowel_diacritic = example['vowel_diacritic']\n    consonant_diacritic = example['consonant_diacritic']\n     \n    head_root_hot = tf.sparse.to_dense(example['head_root_hot'])\n    head_vowel_hot = tf.sparse.to_dense(example['head_vowel_hot'])\n    head_consonant_hot = tf.sparse.to_dense(example['head_consonant_hot'])\n    \n    head_root_hot = tf.reshape(head_root_hot, [168])\n    head_vowel_hot = tf.reshape(head_vowel_hot, [11])\n    head_consonant_hot = tf.reshape(head_consonant_hot, [7])\n    \n    label  = example['label']\n    height = example['size'][0]\n    width  = example['size'][1]\n    return image,  {\"head_root\": head_root_hot, \"head_vowel\": head_vowel_hot, \"head_consonant\": head_consonant_hot}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\nshutil.make_archive('gztfrecords', 'zip', '/kaggle/working/tfrecords')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}