{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Plant Pathology TFRecords Maker\n\nVersion 0: Implement all functionality except MixUp   \n- Image size 224   \n- Random hue,sat,contrast,brightness,flips   \n- Feature dict :\n    ```\n    {\n        'image' : _bytes_feature(img.numpy().tobytes()),  # float32 images\n        'target': _float_arr_feature(labels),\n        'image_name': _bytes_feature(bytes(image_name,encoding='utf8'))\n    }\n    ```\n\nVersion 1:\n- Image size 512\n- Random hue,sat,contrast,brightness,flips\n- Feature dict :\n    ```\n    {\n        'image' : _bytes_feature(img.numpy().tobytes()), #uint8 images\n        'target': _float_arr_feature(labels),\n        'image_name': _bytes_feature(bytes(image_name,encoding='utf8'))\n    }\n    ```\n","metadata":{}},{"cell_type":"markdown","source":"### Import necessary packages","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport os\nimport shutil\nfrom sklearn.model_selection import train_test_split,StratifiedKFold\nimport gc\nimport cv2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get main training csv","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('../input/plant-pathology-2021-fgvc8/train.csv')\ndf_images = df['image']\ndf_labels = df['labels']\n","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Multiple entries one-hot function","metadata":{}},{"cell_type":"code","source":"def custom_one_hot(label_str):\n    all_labels = ['healthy', 'scab', 'frog_eye_leaf_spot', 'rust', 'powdery_mildew','complex']\n    this_labels = label_str.split()\n    \n    retarr = np.zeros((6,),dtype=np.float32)\n    for i in this_labels:\n        retarr[all_labels.index(i)] = 1.0\n    return retarr\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# all_labels = ['healthy']\n# for i in range(len(df_labels)):\n#     label = df_labels[i]\n#     if len(label.split())!=1:\n#         labels = label.split()\n#         for j in labels:\n#             if j not in all_labels:\n#                 all_labels.append(j)\n\n\n\n# #all_labels = ['healthy', 'scab', 'frog_eye_leaf_spot',  'rust', 'powdery_mildew']\n\n# for i in range(len(df_labels)):\n#     df_labels[i] = custom_one_hot(df_labels[i])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Declaring image size","metadata":{}},{"cell_type":"code","source":"IMG_SIZE = 512","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make 6 Stratified K Folds","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=6,shuffle=True)\n\nFOLDS_LIST=[]\na = 0\nfor train_index,test_index in skf.split(df_images,df_labels):\n    #print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n    df_images_train, df_images_test = df_images[train_index], df_images[test_index]\n    df_labels_train, df_labels_test = df_labels[train_index], df_labels[test_index]\n    df_train = pd.concat([df_images_train,df_labels_train],axis=1)\n    df_test = pd.concat([df_images_test,df_labels_test],axis=1)\n    \n    df_test.to_csv('fold_'+str(a)+'.csv')\n    a+=1\n    \n    FOLDS_LIST.append(df_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Declaring TFRecords utility functions","metadata":{}},{"cell_type":"code","source":"def _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int32_list=tf.train.Int64List(value=[value]))\n\ndef _float_arr_feature(arr):\n    return tf.train.Feature(float_list=tf.train.FloatList(value=arr))","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Augmentation functions","metadata":{}},{"cell_type":"code","source":"def read_img(image_name):\n    filepath = '../input/plant-pathology-2021-fgvc8/train_images/'+image_name\n    image = tf.io.decode_jpeg(tf.io.read_file(filepath))\n    \n    return image\n\ndef get_augment_list():\n    return np.array(list(map(lambda x:x<1,np.random.randint(2, size=6))),dtype='bool')\n\n\n@tf.function\ndef resize_image(image):\n    return tf.cast(tf.image.resize(image,[IMG_SIZE,IMG_SIZE]),tf.float32)\n\n    \n    \n@tf.function\ndef augment_img_randomly(img):\n    '''\n    Augmentaions to be used: (use stateless versions of these)\n    \n    Random hue (0.2)\n    Random brightness (0.3)\n    Random saturation (0.7,1.3)\n    Random contrast  (0.8,1.2)\n    ''' \n    augment_list = get_augment_list()\n    image = resize_image(img)\n     #(32,512,512,3)\n    \n    if augment_list[0]:\n        image = tf.image.random_saturation(image,0.7,1.3)\n    if augment_list[1]:\n        image = tf.image.random_contrast(image,0.8,1.2)\n    if augment_list[2]:\n        image = tf.image.random_brightness(image,0.3)\n    if augment_list[3]:\n        image = tf.image.random_hue(image,0.2)\n    if augment_list[4]:\n        image = tf.image.random_flip_left_right(image)\n    if augment_list[5]:\n        image = tf.image.random_flip_up_down(image)\n    \n    \n    \n    image = tf.cast(image,tf.uint8)\n    del augment_list,img\n    gc.collect()\n        \n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make example","metadata":{}},{"cell_type":"code","source":"\ndef example_generator(fold_num,image_name):\n    img = read_img(image_name)\n    img = augment_img_randomly(img)\n    fold_df = FOLDS_LIST[fold_num]\n    labels = fold_df[fold_df['image']==image_name].values[0,1]\n    labels = custom_one_hot(labels)\n    feature = {\n        'image' :  _bytes_feature(img.numpy().tobytes()),\n        'target' : _float_arr_feature(labels),\n        'image_name' : _bytes_feature(bytes(image_name,encoding='utf8'))\n    }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fold_df = FOLDS_LIST[3]\n# print(fold_df.head())\n# image_names = fold_df['image']\n# for image_name in image_names:\n#     labels = fold_df[fold_df['image']==image_name].values\n#     #print(custom_one_hot(labels[0,1]))\n#     #print(example_generator(3,labels[0,0]))\n#     break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Write TFRecords","metadata":{}},{"cell_type":"code","source":"for i in range(len(FOLDS_LIST)):\n    record_file = 'fold_'+str(i)+'.tfrecords'\n    \n    print('Writing ',record_file)\n    \n    image_names = list(FOLDS_LIST[i]['image'])\n    \n    fold_df = FOLDS_LIST[i]\n    \n    a=1\n    num_files = len(list(image_names))\n    \n    with tf.io.TFRecordWriter(record_file) as writer:\n      for k in image_names:\n        \n        print('Writing image ',a,' of ',num_files)\n        proto_example = example_generator(i,k)\n        writer.write(proto_example)\n        del proto_example\n        gc.collect()\n        a+=1\n    del writer\n    gc.collect()\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}