{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Instead of reading all image files in train_images folder, I decided to read all images from .tfrec datasets. "},{"metadata":{},"cell_type":"markdown","source":"This made time reduction! \n\nHere's some update info\n* added some functions to convert byte format image data to numpy uint8 \n* optimised conversion functions to reduce additional latency\n"},{"metadata":{},"cell_type":"markdown","source":"Here's a runtime calculation result:\n\n1. read all the .tfrec files, map, and convert into numpy array\n    * CPU times: user 1min 30s, sys: 2.37 s, total: 1min 32s\n    * Wall time: 1min 31s\n\n\n2. read each image file in train folder using v2.imread\n    * CPU times: user 2min 47s, sys: 8.81 s, total: 2min 56s\n    * Wall time: 5min 25s\n    \nComparing the 2 wall times, still 85.4% time reduction!!"},{"metadata":{},"cell_type":"markdown","source":"What can you do with tfrec file?\n* you can read different data types associated with a same object at once.\n* (image data numpy array, image name string, and disease class int using only TFRecordDataset())  \n* If you use a conventional method, you need to use separate functions to read different data types.\n* (read jpeg image using imread(), image name string, and disease class int using pandas.read(), etc...)\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# read .tfrec data\n\nimport cv2\nimport numpy as np # linear algebra\nimport tensorflow as tf\nimport os\nimport matplotlib.pyplot as plt\nimport pandas as pd\n\n# Create a dictionary describing the features.\ntrain_feature_description = {\n    'image': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'image_name': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'target': tf.io.FixedLenFeature([], tf.int64, default_value=0),\n}\n\ndef _parse_image_function(example_proto):\n    return tf.io.parse_single_example(example_proto, train_feature_description)\n\ndef preprocess_image(image):\n    image = tf.io.decode_image(image, channels=3)\n    # Cassava image data size\n#     image = tf.image.resize(image, [600, 800])\n    return image\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nBASE_DIR = \"../input/cassava-leaf-disease-classification/\"\ntfrec_dir = \"train_tfrecords/\"\ntfimage_set = []\n\nfor tfName in os.listdir(os.path.join(BASE_DIR, tfrec_dir))[:]:\n    train_image_dataset = tf.data.TFRecordDataset(BASE_DIR+tfrec_dir+tfName)\n    \n    # when you want to check the dataset structure\n#     for raw_record in train_image_dataset.take(1):\n#         example = tf.train.Example()\n#         example.ParseFromString(raw_record.numpy())\n\n    train_images = train_image_dataset.map(_parse_image_function)    \n    \n    counter=0\n    for image_features in train_images:\n        counter += 1\n        image_raw = preprocess_image(image_features['image'])\n        image_raw_int = image_raw.numpy()\n\n        # this is when you want to display the first image from each .tfrec file\n#         if(counter==1):\n#             print(image_raw_int.dtype,image_raw_int.shape,image_raw_int.min(),image_raw_int.max())\n#             plt.imshow(image_raw_int)\n#             plt.show()\n            \n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# conventional: \n# read each image file in train_images folder\n# read associated info for each image file \n\nfor image_name in os.listdir(os.path.join(BASE_DIR, \"train_images\"))[:]:\n    image = cv2.imread(os.path.join(BASE_DIR, \"train_images\", image_name))\n\ndf_train = pd.read_csv(os.path.join(BASE_DIR, \"train.csv\"))\n    ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}