{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, json, time, sys, math\nimport numpy as np\nimport pandas as pd \nfrom matplotlib import pyplot as plt\nfrom PIL import Image\n\nif 'google.colab' in sys.modules:\n    %tensorflow_version 2.x\nimport tensorflow as tf\n\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE \n\nstart_time = time.time()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### create data.csv\n[Link for understand data and create data.csv](https://www.kaggle.com/seraphwedd18/herbarium-consolidating-the-details)"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/data-for-datatse-herbarium/data.csv')\ndf1 = df.copy()\ndf1['file_name'] = df['file_name'].map(lambda x: x.split('/')[-1])\ndf1.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_PATTERN = '/kaggle/input/herbarium-2020-fgvc7/nybg2020/train/images/*/*/*.jpg'\nTEST_PATTERN = '/kaggle/input/herbarium-2020-fgvc7/nybg2020/test/images/*/*/*.jpg'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shard_size = 32","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_from_dataset(dataset):\n    plt.figure(figsize=(13,13))\n    subplot=331\n    for i, (image, label) in enumerate(dataset):\n        plt.subplot(subplot)\n        plt.axis('off')\n        plt.imshow(image.numpy().astype(np.uint8))\n        plt.title(label.numpy().decode(\"utf-8\"), fontsize=16)\n        subplot += 1\n        if i==8:\n            break\n    plt.tight_layout()\n    plt.subplots_adjust(wspace=0.1, hspace=0.1)\n    plt.show()\n    \ndef decode_jpeg_and_label(filename):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits)\n    label = tf.strings.split(tf.expand_dims(filename, axis=-1), sep='/')\n    label = label.values[-1]\n    return image, label\n\ndef resize_and_crop_image(image, label):\n    w = tf.shape(image)[0]\n    h = tf.shape(image)[1]\n    tw = w//5\n    th = h//5\n    resize_crit = (w * th) / (h * tw)\n    image = tf.cond(resize_crit < 1,\n                    lambda: tf.image.resize(image, [w*tw/w, h*tw/w]), # if true\n                    lambda: tf.image.resize(image, [w*th/h, h*th/h])  # if false\n                   )\n    nw = tf.shape(image)[0]\n    nh = tf.shape(image)[1]\n    image = tf.image.crop_to_bounding_box(image, (nw - tw) // 2, (nh - th) // 2, tw, th)\n    return image, label\n\ndef recompress_image(image, label):\n    height = tf.shape(image)[0]\n    width = tf.shape(image)[1]\n    image = tf.cast(image, tf.uint8)\n    image = tf.image.encode_jpeg(image, optimize_size=True, chroma_downsampling=False)\n    return image, label, height, width","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filenames = tf.data.Dataset.list_files(TRAIN_PATTERN) \ndataset   = filenames.map(decode_jpeg_and_label, num_parallel_calls=AUTO)\ndataset   = dataset.map(resize_and_crop_image, num_parallel_calls=AUTO) \ndataset   = dataset.map(recompress_image, num_parallel_calls=AUTO)\ndataset   = dataset.batch(shard_size)\ndataset   = dataset.prefetch(AUTO)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir tfrecords","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef _bytestring_feature(list_of_bytestrings):\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=list_of_bytestrings))\n\ndef _int_feature(list_of_ints): # int64\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=list_of_ints))\n\ndef _float_feature(list_of_floats): # float32\n    return tf.train.Feature(float_list=tf.train.FloatList(value=list_of_floats))\n  \n\ndef to_tfrecord(tfrec_filewriter, img_bytes, label, family, genus, category_id, width, height):\n    one_hot_family = np.eye(310)[family]\n    one_hot_genus = np.eye(3678)[genus]\n    one_hot_category_id = np.eye(32094)[category_id]\n\n    feature = {\n      \"image\": _bytestring_feature([img_bytes]),\n      \"label\":  _bytestring_feature([label]),\n        \n      \"family\": _int_feature([family]),\n      \"genus\": _int_feature([genus]),\n      \"category_id\": _int_feature([category_id]),\n        \n      \"one_hot_family\": _float_feature(one_hot_family.tolist()),\n      \"one_hot_genus\": _float_feature(one_hot_genus.tolist()),\n      \"one_hot_category_id\": _float_feature(one_hot_category_id.tolist()),\n        \n      \"size\":  _int_feature([width, height])\n    }\n    return tf.train.Example(features=tf.train.Features(feature=feature))\n\nprint(\"Writing TFRecords\")\nstoped = 0\nfor shard, (image, label, height, width) in enumerate(dataset):\n    if stoped > 0:\n        break\n    stoped +=1\n    shard_size = image.numpy().shape[0]\n    filename   = './tfrecords/' + \"{:02d}-{}.tfrec\".format(shard, shard_size)\n  \n    with tf.io.TFRecordWriter(filename) as out_file:\n        for i in range(shard_size):\n            lbl         = label.numpy()[i]\n            family      = df1.loc[df1.file_name == lbl.decode('utf-8')]['family'].values[0]\n            genus       = df1.loc[df1.file_name == lbl.decode('utf-8')]['genus'].values[0]\n            category_id = df1.loc[df1.file_name == lbl.decode('utf-8')]['category_id'].values[0]\n            \n            example = to_tfrecord(out_file,\n                            image.numpy()[i],\n                            lbl,\n                            family, \n                            genus, \n                            category_id,\n                            height.numpy()[i],\n                            width.numpy()[i])\n            out_file.write(example.SerializeToString())\n        print(\"Wrote file {} containing {} records\".format(filename, shard_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_tfrecord(example):\n    features = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        \"label\": tf.io.FixedLenFeature([], tf.string),\n        \n        \"family\": tf.io.FixedLenFeature([], tf.int64), \n        \"genus\": tf.io.FixedLenFeature([], tf.int64), \n        \"category_id\": tf.io.FixedLenFeature([], tf.int64), \n        \n        \"one_hot_family\": tf.io.VarLenFeature(tf.float32) ,\n        \"one_hot_genus\": tf.io.VarLenFeature(tf.float32) ,\n        \"one_hot_category_id\": tf.io.VarLenFeature(tf.float32) ,\n\n        \"size\": tf.io.FixedLenFeature([2], tf.int64) \n    }\n    example = tf.io.parse_single_example(example, features)\n    width = example['size'][0]\n    height  = example['size'][1]\n    image = tf.image.decode_jpeg(example['image'], channels=3)\n    image = tf.reshape(image, [width,height, 3])\n    \n    label = example['label']\n    \n  \n    return image, label\n\noption_no_order = tf.data.Options()\noption_no_order.experimental_deterministic = False\n\nfilenames = tf.io.gfile.glob('/kaggle/working/tfrecords/' + \"*.tfrec\")\ndataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\ndataset = dataset.with_options(option_no_order)\ndataset = dataset.map(read_tfrecord, num_parallel_calls=AUTO)\ndataset = dataset.shuffle(300)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_from_dataset(dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"end_time = time.time()\ntotal = end_time - start_time\nh = total//3600\nm = (total%3600)//60\ns = total%60\nprint(\"Total time spent: %i hours, %i minutes, and %i seconds\" %(h, m, s))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}