{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Description"},{"metadata":{},"cell_type":"markdown","source":"### Generates following TFRecords:\n\n* Original 800x600 resized to 300x300\n* Original 800x600 resized to 512x512\n* Original 800x600 center cropped to 600x600\n* Original 800x600 center cropped to 600x600 resized to 300x300\n* Original 800x600 center cropped to 600x600 resized to 512x512"},{"metadata":{},"cell_type":"markdown","source":"# Setup"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import re, math, os, cv2\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_FOLDER = '../input/cassava-leaf-disease-classification'\nIMAGES_FOLDER = os.path.join(INPUT_FOLDER, 'train_images')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utilities"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Preprocessing\n\ndef center_crop(image):\n    h, w = image.shape[0], image.shape[1]\n\n    if h > w:\n        image = tf.image.crop_to_bounding_box(image, (h - w) // 2, 0, w, w)\n    else:\n        image = tf.image.crop_to_bounding_box(image, 0, (w - h) // 2, h, h)\n\n    return image.numpy().astype('uint8')\n\n\n# Create TF Records\n\ndef _bytes_feature(value):\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    \n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef _int64_feature(value):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n\ndef serialize_example(image, target, image_name):\n    feature = {\n        'image': _bytes_feature(image),\n        'target': _int64_feature(target),\n        'image_name': _bytes_feature(image_name),\n    }\n    \n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    \n    return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generate records"},{"metadata":{},"cell_type":"markdown","source":"### Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"IMGS = os.listdir(IMAGES_FOLDER)\nIMG_QUALITY = 100\n\nprint(f'Image samples: {len(IMGS)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CONFIGURATIONS = [\n    {\n        'transform': lambda img: img,\n        'output_folder': '800x600_original'\n    },\n#     {\n#         'transform': lambda img: cv2.resize(img, (300, 300)),\n#         'output_folder': '800x600_resized_300x300'\n#     },\n#     {\n#         'transform': lambda img: cv2.resize(img, (512, 512)),\n#         'output_folder': '800x600_resized_512x512'\n#     },\n#     {\n#         'transform': lambda img: cv2.resize(center_crop(img), (600, 600)),\n#         'output_folder': '800x600_cropped_600x600'\n#     },\n#     {\n#         'transform': lambda img: cv2.resize(center_crop(img), (300, 300)),\n#         'output_folder': '800x600_cropped_600x600_resized_300x300'\n#     },\n#     {\n#         'transform': lambda img: cv2.resize(center_crop(img), (512, 512)),\n#         'output_folder': '800x600_cropped_600x600_resized'\n#     },\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv(os.path.join(INPUT_FOLDER, 'train.csv'))\ndata['label'] = data['label']\n\nprint(data.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filtered_data = data[~data['image_id'].isin(['1562043567.jpg', '911861181.jpg'])]\nfiltered_data = filtered_data.reset_index(drop=True)\n\nprint(filtered_data.shape)\nfiltered_data.groupby('label').size()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Split samples into files"},{"metadata":{"trusted":true},"cell_type":"code","source":"RECORDS_SPLIT = {\n    0: 4,\n    1: 8,\n    2: 9,\n    3: 49,\n    4: 9\n}\n\ngrouped_data = list(filtered_data.groupby('label'))\ngrouped_files = []\n\nfor (g_id, group) in grouped_data:\n    n, j = RECORDS_SPLIT[g_id], 0\n    \n    group = group.reset_index(drop=True)\n    group['file'] = 0\n    \n    n = int(np.ceil(len(group)/n))\n    \n    for i in range(0, len(group), n):\n        group.iloc[i:i+n, -1] = j\n        j += 1\n        \n    grouped_files.append(group)\n    \ngrouped_files = pd.concat(grouped_files)\n\ngrouped_files.groupby(['label', 'file']).size()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# RECORDS_SPLIT = {\n#     0: 1087,\n#     1: 1096,\n#     2: 1193,\n#     3: 1013,\n#     4: 1288\n# }\n\n# grouped_data = list(filtered_data.groupby('label'))\n# grouped_files = []\n\n# for (g_id, group) in grouped_data:\n#     n, j = RECORDS_SPLIT[g_id], 0\n#     group = group.reset_index(drop=True)\n#     group['file'] = 0\n#     for i in range(0, len(group), n):\n#         group.iloc[i:i+n, -1] = j\n#         j += 1\n        \n#     grouped_files.append(group)\n    \n# grouped_files = pd.concat(grouped_files)\n\n# grouped_files.groupby(['label', 'file']).size()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Write TFRecords"},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_data = list(grouped_files.groupby(['label', 'file']))\nn_records = len(grouped_data) * len(CONFIGURATIONS) \nj = 1\n\nfor (lbl, fid), samples in grouped_data:\n    fname = 'train_2020_%i_%.2i-%i.tfrec' % (lbl, fid, len(samples))\n    \n    for cfg in CONFIGURATIONS:\n        print('Writing TFRecord %i of %i...' % (j, n_records))\n        os.makedirs(cfg['output_folder'], exist_ok=True)\n        \n        with tf.io.TFRecordWriter(os.path.join(cfg['output_folder'], fname)) as writer:\n            for row in samples.itertuples():\n                label = row.label\n                image_name = row.image_id\n                img_path = os.path.join(IMAGES_FOLDER, image_name)\n\n                img = cv2.imread(img_path)\n                img = cfg['transform'](img)\n                img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, IMG_QUALITY))[1].tostring()\n\n                example = serialize_example(img, label, str.encode(image_name))\n\n                writer.write(example)\n                \n        j += 1","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}