{"cells":[{"metadata":{},"cell_type":"markdown","source":"I used to have albumentation(excellent library btw) to perform the data augmentation. <br />\nBut in this comptition, since our data is .tfrec format, so it will remain in tensor during the data pipeline. <br />\nAnd since I have noticed a lots of people have questions about data augmentation in this competition. <br />\nSo I had implemented some simple data augmentation methods in pure tensorflow, which can be used in both Eager mode and Graph mode. <br />\n<br />\nInclude :  <br />\n\n**1. Gaussian blur** <br />\n**2. Random block out** <br />\n**3. Random rotate** <br />\n**4. Random scale** <br />\n\nPlease enjoy it :) <br />\nImage rotate refer to : https://stackoverflow.com/questions/34801342/tensorflow-how-to-rotate-an-image-for-data-augmentation"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\n\nprint('Tensorflow version : {}'.format(tf.__version__))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path('flower-classification-with-tpus')\nGCS_PATH = os.path.join(GCS_DS_PATH, 'tfrecords-jpeg-512x512')\nTEST_FNS = tf.io.gfile.glob(os.path.join(GCS_PATH, 'test/*.tfrec'))\nIMG_DIM = (512, 512)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AugParams = {\n    'scale_factor':0.5,\n    'scale_prob':0.5,\n    'rot_range':90,\n    'rot_prob':0.5,\n    'blockout_sl':0.1,\n    'blockout_sh':0.2,\n    'blockout_rl':0.4,\n    'blockout_prob':0.5,\n    'blur_ksize':3,\n    'blur_sigma':1,\n    'blur_prob':0.5\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_rotate(image, angle):\n\n    if len(image.get_shape().as_list()) != 3:\n        raise ValueError('`image_rotate` only support image with 3 dimension(h, w, c)`')\n\n    angle = tf.cast(angle, tf.float32)\n    h, w, c = IMG_DIM[0], IMG_DIM[1], 3\n    cy, cx = h//2, w//2\n\n    ys = tf.range(h)\n    xs = tf.range(w)\n\n    ys_vec = tf.tile(ys, [w])\n    xs_vec = tf.reshape( tf.tile(xs, [h]), [h,w] )\n    xs_vec = tf.reshape( tf.transpose(xs_vec, [1,0]), [-1])\n\n    ys_vec_centered, xs_vec_centered = ys_vec - cy, xs_vec - cx\n    new_coord_centered = tf.cast(tf.stack([ys_vec_centered, xs_vec_centered]), tf.float32)\n\n    inv_rot_mat = tf.reshape( tf.dynamic_stitch([0,1,2,3], [tf.cos(angle), tf.sin(angle), -tf.sin(angle), tf.cos(angle)]), [2,2])\n    old_coord_centered = tf.matmul(inv_rot_mat, new_coord_centered)\n\n    old_ys_vec_centered, old_xs_vec_centered = old_coord_centered[0,:], old_coord_centered[1,:]\n    old_ys_vec = tf.cast( tf.round(old_ys_vec_centered+cy), tf.int32)\n    old_xs_vec = tf.cast( tf.round(old_xs_vec_centered+cx), tf.int32)\n\n    outside_ind = tf.logical_or( tf.logical_or(old_ys_vec > h-1 , old_ys_vec < 0), tf.logical_or(old_xs_vec > w-1 , old_xs_vec<0))\n\n    old_ys_vec = tf.boolean_mask(old_ys_vec, tf.logical_not(outside_ind))\n    old_xs_vec = tf.boolean_mask(old_xs_vec, tf.logical_not(outside_ind))\n\n    ys_vec = tf.boolean_mask(ys_vec, tf.logical_not(outside_ind))\n    xs_vec = tf.boolean_mask(xs_vec, tf.logical_not(outside_ind))\n\n    old_coord = tf.cast(tf.transpose(tf.stack([old_ys_vec, old_xs_vec]), [1,0]), tf.int32)\n    new_coord = tf.cast(tf.transpose(tf.stack([ys_vec, xs_vec]), [1,0]), tf.int64)\n\n    channel_vals = tf.split(image, c, axis=-1)\n    rotated_channel_vals = list()\n    for channel_val in channel_vals:\n        rotated_channel_val = tf.gather_nd(channel_val, old_coord)\n\n        sparse_rotated_channel_val = tf.SparseTensor(new_coord, tf.squeeze(rotated_channel_val,axis=-1), [h, w])\n        rotated_channel_vals.append(tf.sparse.to_dense(sparse_rotated_channel_val, default_value=0, validate_indices=False))\n\n    rotated_image = tf.transpose(tf.stack(rotated_channel_vals), [1, 2, 0])\n    return rotated_image\n    \ndef random_blockout(img, sl=0.1, sh=0.2, rl=0.4):\n\n    h, w, c = IMG_DIM[0], IMG_DIM[1], 3\n    origin_area = tf.cast(h*w, tf.float32)\n\n    e_size_l = tf.cast(tf.round(tf.sqrt(origin_area * sl * rl)), tf.int32)\n    e_size_h = tf.cast(tf.round(tf.sqrt(origin_area * sh / rl)), tf.int32)\n\n    e_height_h = tf.minimum(e_size_h, h)\n    e_width_h = tf.minimum(e_size_h, w)\n\n    erase_height = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_height_h, dtype=tf.int32)\n    erase_width = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_width_h, dtype=tf.int32)\n\n    erase_area = tf.zeros(shape=[erase_height, erase_width, c])\n    erase_area = tf.cast(erase_area, tf.uint8)\n\n    pad_h = h - erase_height\n    pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)\n    pad_bottom = pad_h - pad_top\n\n    pad_w = w - erase_width\n    pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)\n    pad_right = pad_w - pad_left\n\n    erase_mask = tf.pad([erase_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)\n    erase_mask = tf.squeeze(erase_mask, axis=0)\n    erased_img = tf.multiply(tf.cast(img,tf.float32), tf.cast(erase_mask, tf.float32))\n\n    return tf.cast(erased_img, img.dtype)\n    \ndef zoom_out(x, scale_factor):\n\n    resize_x = tf.random.uniform(shape=[], minval=tf.cast(IMG_DIM[1]//(1/scale_factor), tf.int32), maxval=IMG_DIM[1], dtype=tf.int32)\n    resize_y = tf.random.uniform(shape=[], minval=tf.cast(IMG_DIM[0]//(1/scale_factor), tf.int32), maxval=IMG_DIM[0], dtype=tf.int32)\n    top_pad = (IMG_DIM[0] - resize_y) // 2\n    bottom_pad = IMG_DIM[0] - resize_y - top_pad\n    left_pad = (IMG_DIM[1] - resize_x ) // 2\n    right_pad = IMG_DIM[1] - resize_x - left_pad\n        \n    x = tf.image.resize(x, (resize_y, resize_x))\n    x = tf.pad([x], [[0,0], [top_pad, bottom_pad], [left_pad, right_pad], [0,0]])\n    x = tf.image.resize(x, IMG_DIM)\n    return tf.squeeze(x, axis=0)\n    \ndef zoom_in(x, scale_factor):\n\n    scales = list(np.arange(0.5, 1.0, 0.05))\n    boxes = np.zeros((len(scales),4))\n            \n    for i, scale in enumerate(scales):\n        x_min = y_min = 0.5 - (0.5*scale)\n        x_max = y_max = 0.5 + (0.5*scale)\n        boxes[i] = [x_min, y_min, x_max, y_max]\n        \n    def random_crop(x):\n        crop = tf.image.crop_and_resize([x], boxes=boxes, box_indices=np.zeros(len(boxes)), crop_size=IMG_DIM)\n        return crop[tf.random.uniform(shape=[], minval=0, maxval=len(scales), dtype=tf.int32)]\n        \n    return random_crop(x)\n\ndef gaussian_blur(img, ksize=5, sigma=1):\n    \n    def gaussian_kernel(size=3, sigma=1):\n\n        x_range = tf.range(-(size-1)//2, (size-1)//2 + 1, 1)\n        y_range = tf.range((size-1)//2, -(size-1)//2 - 1, -1)\n\n        xs, ys = tf.meshgrid(x_range, y_range)\n        kernel = tf.exp(-(xs**2 + ys**2)/(2*(sigma**2))) / (2*np.pi*(sigma**2))\n        return tf.cast( kernel / tf.reduce_sum(kernel), tf.float32)\n    \n    kernel = gaussian_kernel(ksize, sigma)\n    kernel = tf.expand_dims(tf.expand_dims(kernel, axis=-1), axis=-1)\n    \n    r, g, b = tf.split(img, [1,1,1], axis=-1)\n    r_blur = tf.nn.conv2d([r], kernel, [1,1,1,1], 'SAME')\n    g_blur = tf.nn.conv2d([g], kernel, [1,1,1,1], 'SAME')\n    b_blur = tf.nn.conv2d([b], kernel, [1,1,1,1], 'SAME')\n\n    blur_image = tf.concat([r_blur, g_blur, b_blur], axis=-1)\n    return tf.squeeze(blur_image, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def augmentation(image):\n    image = tf.cast(image, tf.float32)\n    #Gaussian blur\n    if tf.random.uniform(shape=[], minval=0.0, maxval=1.0) > AugParams['blur_prob']:\n        image = gaussian_blur(image, AugParams['blur_ksize'], AugParams['blur_sigma'])\n    \n    #Random block out\n    if tf.random.uniform(shape=[], minval=0.0, maxval=1.0) > AugParams['blockout_prob']:\n        image = random_blockout(image, AugParams['blockout_sl'], AugParams['blockout_sh'], AugParams['blockout_rl'])\n        \n    #Random scale\n    if tf.random.uniform(shape=[], minval=0.0, maxval=1.0) > AugParams['scale_prob']:\n        if tf.random.uniform(shape=[], minval=0.0, maxval=1.0) > 0.5:\n            image = zoom_in(image, AugParams['scale_factor'])\n        else:\n            image = zoom_out(image, AugParams['scale_factor'])\n    #Random rotate\n    if tf.random.uniform(shape=[], minval=0.0, maxval=1.0) > AugParams['rot_prob']:\n        angle = tf.random.uniform(shape=[], minval=-AugParams['rot_range'], maxval=AugParams['rot_range'], dtype=tf.int32)\n        image = image_rotate(image,angle)\n    \n    return tf.cast(image, tf.uint8)\n    \ndef decoded_example(example):\n    example = tf.io.parse_single_example(example, { \"image\" : tf.io.FixedLenFeature([], tf.string) })\n    bits = example['image']\n    image = tf.image.decode_jpeg(bits)\n    return image\n\nds = tf.data.TFRecordDataset(TEST_FNS)\nds = ds.map(decoded_example)\nds = ds.map(augmentation)\nds = ds.batch(25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images = next(iter(ds))\nimages = images.numpy()\nplt.figure(figsize=(24,24))\ncol = 5\nrow = 5\nfor idx, image in enumerate(images):\n    plt.subplot(row, col, idx+1)\n    plt.imshow(image)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}