{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport os\nimport shutil\nfrom sklearn.model_selection import train_test_split,StratifiedKFold\nimport gc ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ndf_images = df['image_id']\ndf_labels = df['label']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5,shuffle=True)\n\nFOLDS_LIST=[]\na = 0\nfor train_index,test_index in skf.split(df_images,df_labels):\n    #print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n    df_images_train, df_images_test = df_images[train_index], df_images[test_index]\n    df_labels_train, df_labels_test = df_labels[train_index], df_labels[test_index]\n    df_train = pd.concat([df_images_train,df_labels_train],axis=1)\n    df_test = pd.concat([df_images_test,df_labels_test],axis=1)\n    \n    df_test.to_csv('fold_'+str(a)+'.csv')\n    a+=1\n    \n    FOLDS_LIST.append(df_test)\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nIMG_SIZE = 299\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Image generator\n\nMake a provision such that images are provided and written to TFrecords without too much RAM consumption. "},{"metadata":{"trusted":true},"cell_type":"code","source":"#@tf.function\ndef read_img(image_name):\n    filepath = '../input/cassava-leaf-disease-classification/train_images/'+image_name\n    image = tf.io.decode_jpeg(tf.io.read_file(filepath))\n    \n    return image\n\n\n#@tf.function\ndef image_generator(image_name):\n    img = read_img(image_name)\n    img = augment_img_randomly(img)\n    return img\n\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"#@tf.function\ndef get_augment_list():\n    return np.array(list(map(lambda x:x<1,np.random.randint(2, size=6))),dtype='bool')\n\n\n@tf.function\ndef resize_image(image):\n    return tf.cast(tf.image.resize(image,[IMG_SIZE,IMG_SIZE]),tf.float16)\n\n    \n    \n@tf.function\ndef augment_img_randomly(img):\n    '''\n    Augmentaions to be used: (use stateless versions of these)\n    \n    Random hue (0.2)\n    Random brightness (0.3)\n    Random saturation (0.7,1.3)\n    Random contrast  (0.8,1.2)\n    ''' \n    augment_list = get_augment_list()\n    image = resize_image(img)\n     #(32,512,512,3)\n    \n    if augment_list[0]:\n        image = tf.image.random_saturation(image,0.7,1.3)\n    if augment_list[1]:\n        image = tf.image.random_contrast(image,0.8,1.2)\n    if augment_list[2]:\n        image = tf.image.random_brightness(image,0.3)\n    if augment_list[3]:\n        image = tf.image.random_hue(image,0.2)\n    if augment_list[4]:\n        image = tf.image.random_flip_left_right(image)\n    if augment_list[5]:\n        image = tf.image.random_flip_up_down(image)\n    \n    \n    \n    image = tf.math.divide(image,255)\n    del augment_list,img\n    gc.collect()\n        \n    return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Writing TFRecords\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n\ndef generate_example(image_name,fold_df):\n    img = image_generator(image_name)\n    label = fold_df[fold_df['image_id']==image_name]['label']\n    \n    feature={\n        'image':_bytes_feature(img.numpy().tobytes()),\n        'target':_int64_feature(int(label)),\n        'image_id':_bytes_feature(bytes(image_name,encoding='utf8'))\n    }\n    #del img,label\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"So, we have coded for the following things:\n    1. Take image \n    2. Augment\n    3. Make a example protobuf\n\nNow we need to take all images in a FOLDS_LIST[i] and put them in a single TFRec. Same for its validation. \nIn this way, we will get 5 folds and all will have different images. Note that validation images will be augmented, and we\nwill be using TTA\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(FOLDS_LIST)):\n    record_file = 'fold_'+str(i)+'.tfrecords'\n    \n    print('Writing ',record_file)\n    \n    image_names = list(FOLDS_LIST[i]['image_id'])\n    \n    fold_df = FOLDS_LIST[i]\n    \n    a=1\n    num_files = len(list(image_names))\n    \n    with tf.io.TFRecordWriter(record_file) as writer:\n      for k in image_names:\n        \n        print('Writing image ',a,' of ',num_files)\n        proto_example = generate_example(k,fold_df)\n        writer.write(proto_example)\n        del proto_example\n        gc.collect()\n        a+=1\n    del writer\n    gc.collect()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# record_file = 'images.tfrecords'\n# with tf.io.TFRecordWriter(record_file) as writer:\n#   for filename, label in image_labels.items():\n#     image_string = open(filename, 'rb').read()\n#     tf_example = image_example(image_string, label)\n#     writer.write(tf_example.SerializeToString())","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}