{"cells":[{"metadata":{},"cell_type":"markdown","source":"Some basic ideas are credited to [DimitreOliveira](https://www.kaggle.com/dimitreoliveira) for this [notebook](https://www.kaggle.com/dimitreoliveira/cassava-leaf-disease-stratified-tfrecords-256x256), please upvote it also if you like."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\nimport os\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import StratifiedKFold, KFold\nfrom tqdm.autonotebook import tqdm\nimport random","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"ROOT_DIR = '../input/cassava-leaf-disease-classification'\nHEIGHT, WIDTH = 512, 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nSEED = 0\nN_TRAINFILES = 50\nN_FOLDS = 5\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(os.path.join(ROOT_DIR, 'train.csv'))\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Label distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(x='label', data=train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Oversample"},{"metadata":{"trusted":true},"cell_type":"code","source":"# The following functions can be used to convert a value to a type compatible\n# with tf.train.Example.\n\ndef _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_example(image, image_name, target):\n    feature = {\n        'image': _bytes_feature(image),\n        'image_name': _bytes_feature(image_name),\n        'target': _int64_feature(target)\n    }\n    # Create a Features message using tf.train.Example.\n\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Write to tfrecord files"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create file for training\nskf = StratifiedKFold(n_splits=N_TRAINFILES, shuffle=True, random_state=SEED)\nidx_os = [] \nfor fold, (train_idx, test_idx) in tqdm(enumerate(skf.split(train_df, train_df['label']))):\n    filename = 'ld_train{:02d}-{}.tfrec'.format(fold, len(test_idx))\n#     idx_os.append(test_idx)\n    with tf.io.TFRecordWriter(filename) as writer:\n        for item in test_idx:\n            image_name = train_df.loc[item]['image_id']\n            image = cv2.imread(os.path.join(ROOT_DIR, 'train_images', image_name))\n            image = cv2.resize(image, (HEIGHT, WIDTH))\n            image = cv2.imencode('.jpg', image, (cv2.IMWRITE_JPEG_QUALITY, 100))[1].tostring()\n            target = train_df.loc[item]['label']\n            example = serialize_example(image, image_name.encode(), target)\n            writer.write(example)\n    fold_df = train_df.loc[test_idx]\n    for c in [0, 1, 2, 4]:\n        class_imgnames = fold_df[fold_df['label']==c]['image_id'].values\n        filename = 'class{}_{:02d}-{}.tfrec'.format(c, fold, len(class_imgnames))\n        with tf.io.TFRecordWriter(filename) as writer:\n            for image_name in class_imgnames:\n                image = cv2.imread(os.path.join(ROOT_DIR, 'train_images', image_name))\n                image = cv2.resize(image, (HEIGHT, WIDTH))\n                image = cv2.imencode('.jpg', image, (cv2.IMWRITE_JPEG_QUALITY, 100))[1].tostring()\n                target = c\n                example = serialize_example(image, image_name.encode(), target)\n                writer.write(example)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}