{"cells":[{"metadata":{},"cell_type":"markdown","source":"## This kernel is just to see how the basic augmentation will effect the images.\n### <font color='red'>If you find this kernel helpful please upvote 😊. Also dont forget to upvote the mentioned kernel below.</font>\n\n________________________________________________\n\nTodo - See rare classes and see how to augment those\n_______________________________________________\n\nSome of the functions are taken from this kernel <br>\nhttps://www.kaggle.com/dimitreoliveira/flower-classification-with-tpus-eda-and-baseline\n\nRotation augmentation from this kernel. <br>\nhttps://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96/comments\n\nfour_point_transform <br>\nhttps://www.pyimagesearch.com/2014/08/25/4-point-opencv-getperspective-transform-example/"},{"metadata":{},"cell_type":"markdown","source":"## Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math, re, os\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nimport numpy as np\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nprint(\"Tensorflow version \" + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Configurations"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"HEIGHT = 512\nWIDTH = 512\nCHANNELS = 3\nN_CLASSES = 104\nSHOW_LIMIT = 10\nseed = 27\n\nGCS_PATH = KaggleDatasets().get_gcs_path() + '/tfrecords-jpeg-%sx%s' % (HEIGHT, WIDTH)\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')\n\nCLASSES = [\n    'pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', \n    'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', \n    'globe thistle', 'snapdragon', \"colt's foot\", 'king protea', 'spear thistle', \n    'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', \n    'balloon flower', 'giant white arum lily', 'fire lily', 'pincushion flower', \n    'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', \n    'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william', \n    'carnation', 'garden phlox', 'love in the mist', 'cosmos',  'alpine sea holly', \n    'ruby-lipped cattleya', 'cape flower', 'great masterwort',  'siam tulip', \n    'lenten rose', 'barberton daisy', 'daffodil',  'sword lily', 'poinsettia', \n    'bolero deep blue',  'wallflower', 'marigold', 'buttercup', 'daisy', \n    'common dandelion', 'petunia', 'wild pansy', 'primula',  'sunflower', \n    'lilac hibiscus', 'bishop of llandaff', 'gaura',  'geranium', 'orange dahlia', \n    'pink-yellow dahlia', 'cautleya spicata',  'japanese anemone', 'black-eyed susan', \n    'silverbush', 'californian poppy',  'osteospermum', 'spring crocus', 'iris', \n    'windflower',  'tree poppy', 'gazania', 'azalea', 'water lily',  'rose', \n    'thorn apple', 'morning glory', 'passion flower',  'lotus', 'toad lily', \n    'anthurium', 'frangipani',  'clematis', 'hibiscus', 'columbine', 'desert-rose', \n    'tree mallow', 'magnolia', 'cyclamen ', 'watercress',  'canna lily', \n    'hippeastrum ', 'bee balm', 'pink quill',  'foxglove', 'bougainvillea', \n    'camellia', 'mallow',  'mexican petunia',  'bromelia', 'blanket flower', \n    'trumpet creeper',  'blackberry lily', 'common tulip', 'wild rose']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    \n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    # Not showing labels so no need of if condition\n    #if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n    numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def showimage(image1,image2,title1= 'Before augmentation',title2 ='After augmentation'):\n    plt.figure(figsize=(15, 15))\n    plt.subplot(121)\n    plt.imshow(image1)\n    plt.title(title1)\n    \n    plt.subplot(122)\n    plt.imshow(image2)\n    plt.title(title2)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Datasets utility functions\nAUTO = tf.data.experimental.AUTOTUNE # instructs the API to read from multiple files if available.\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef data_augment(image, label):\n    image = tf.image.random_flip_left_right(image, seed=seed)\n    image = tf.image.random_flip_up_down(image, seed=seed)\n    image = tf.image.random_saturation(image, lower=0, upper=2, seed=seed)\n    image = tf.image.random_contrast(image, lower=.8, upper=2, seed=seed)\n    image = tf.image.random_brightness(image, max_delta=.2, seed=seed)\n    image = tf.image.random_crop(image, size=[int(HEIGHT*.8), int(WIDTH*.8), CHANNELS], seed=seed)\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Augmentations"},{"metadata":{},"cell_type":"markdown","source":"### 1. Rotation at angle and shear from https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96/comments"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef transform(image,label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = HEIGHT\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 15. * tf.random.normal([1],dtype='float32')\n    shr = 5. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 16. * tf.random.normal([1],dtype='float32') \n    w_shift = 16. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3]),label\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=True)\ndataset_aug = dataset.map(transform, num_parallel_calls=AUTO)\nfor i,(data, data_aug) in enumerate(zip(dataset,dataset_aug)):\n    image1, label1 = batch_to_numpy_images_and_labels(data)\n    image2, label2 = batch_to_numpy_images_and_labels(data_aug)\n    showimage(image1,image2)\n    if i == SHOW_LIMIT:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('random_flip_left_right')\ndef random_flip_left_right(image, label):\n    image = tf.image.random_flip_left_right(image, seed=seed)\n    return image, label\n\ndataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=True)\ndataset_aug = dataset.map(random_flip_left_right, num_parallel_calls=AUTO)\nfor i,(data, data_aug) in enumerate(zip(dataset,dataset_aug)):\n    image1, label1 = batch_to_numpy_images_and_labels(data)\n    image2, label2 = batch_to_numpy_images_and_labels(data_aug)\n    showimage(image1,image2)\n    if i == SHOW_LIMIT:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('random_flip_up_down')\ndef random_flip_up_down(image, label):\n    image = tf.image.random_flip_up_down(image, seed=seed)\n    return image, label\n\n\ndataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=True)\ndataset_aug = dataset.map(random_flip_up_down, num_parallel_calls=AUTO)\nfor i,(data, data_aug) in enumerate(zip(dataset,dataset_aug)):\n    image1, label1 = batch_to_numpy_images_and_labels(data)\n    image2, label2 = batch_to_numpy_images_and_labels(data_aug)\n    showimage(image1,image2)\n    if i == SHOW_LIMIT:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('random_saturation')\ndef random_saturation(image, label):\n    image = tf.image.random_saturation(image, lower=0, upper=2, seed=seed)\n    return image, label\n\ndataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=True)\ndataset_aug = dataset.map(random_saturation, num_parallel_calls=AUTO)\nfor i,(data, data_aug) in enumerate(zip(dataset,dataset_aug)):\n    image1, label1 = batch_to_numpy_images_and_labels(data)\n    image2, label2 = batch_to_numpy_images_and_labels(data_aug)\n    showimage(image1,image2)\n    if i == SHOW_LIMIT:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('random_crop')\ndef random_crop(image, label):\n    image = tf.image.random_crop(image, size=[int(HEIGHT*.8), int(WIDTH*.8), CHANNELS], seed=seed)\n    return image, label\n\ndataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=True)\ndataset_aug = dataset.map(random_crop, num_parallel_calls=AUTO)\nfor i,(data, data_aug) in enumerate(zip(dataset,dataset_aug)):\n    image1, label1 = batch_to_numpy_images_and_labels(data)\n    image2, label2 = batch_to_numpy_images_and_labels(data_aug)\n    showimage(image1,image2)\n    if i == SHOW_LIMIT:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('random_contrast')\ndef random_contrast(image, label):\n    image = tf.image.random_contrast(image, lower=.8, upper=2, seed=seed)\n    return image, label\n\ndataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=True)\ndataset_aug = dataset.map(random_contrast, num_parallel_calls=AUTO)\nfor i,(data, data_aug) in enumerate(zip(dataset,dataset_aug)):\n    image1, label1 = batch_to_numpy_images_and_labels(data)\n    image2, label2 = batch_to_numpy_images_and_labels(data_aug)\n    showimage(image1,image2)\n    if i == SHOW_LIMIT:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('random_brightness')\ndef random_brightness(image, label):\n    image = tf.image.random_brightness(image, max_delta=.2, seed=seed)\n    return image, label\n\ndataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=True)\ndataset_aug = dataset.map(random_brightness, num_parallel_calls=AUTO)\nfor i,(data, data_aug) in enumerate(zip(dataset,dataset_aug)):\n    image1, label1 = batch_to_numpy_images_and_labels(data)\n    image2, label2 = batch_to_numpy_images_and_labels(data_aug)\n    showimage(image1,image2)\n    if i == SHOW_LIMIT:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('random_brightness')\ndef random_brightness(image, label):\n    image = tf.image.random_brightness(image, max_delta=.2, seed=seed)\n    return image, label\n\ndataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=True)\ndataset_aug = dataset.map(random_brightness, num_parallel_calls=AUTO)\nfor i,(data, data_aug) in enumerate(zip(dataset,dataset_aug)):\n    image1, label1 = batch_to_numpy_images_and_labels(data)\n    image2, label2 = batch_to_numpy_images_and_labels(data_aug)\n    showimage(image1,image2)\n    if i == SHOW_LIMIT:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('All together')\ndataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=True)\ndataset_aug = dataset.map(data_augment, num_parallel_calls=AUTO)\nfor i,(data, data_aug) in enumerate(zip(dataset,dataset_aug)):\n    image1, label1 = batch_to_numpy_images_and_labels(data)\n    image2, label2 = batch_to_numpy_images_and_labels(data_aug)\n    showimage(image1,image2)\n    if i == SHOW_LIMIT:\n        break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## four_point_transform\n#### cutting out a Trapezoidal from image and then reshaping it to 512,512"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport cv2\ndef order_points(pts):\n    rect = np.zeros((4, 2), dtype = \"float32\")\n    s = pts.sum(axis = 1)\n    rect[0] = pts[np.argmin(s)]\n    rect[2] = pts[np.argmax(s)]\n    diff = np.diff(pts, axis = 1)\n    rect[1] = pts[np.argmin(diff)]\n    rect[3] = pts[np.argmax(diff)]\n    return rect","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def four_point_transform(image,label):\n    cords = \"[(0, 0), (0, 512), (512, 400), (512, 112)]\"\n    pts = np.array(eval(cords), dtype = \"float32\")\n    rect = order_points(pts)\n    (tl, tr, br, bl) = rect\n    widthA = np.sqrt(((br[0] - bl[0]) ** 2) + ((br[1] - bl[1]) ** 2))\n    widthB = np.sqrt(((tr[0] - tl[0]) ** 2) + ((tr[1] - tl[1]) ** 2))\n    maxWidth = max(int(widthA), int(widthB))\n    heightA = np.sqrt(((tr[0] - br[0]) ** 2) + ((tr[1] - br[1]) ** 2))\n    heightB = np.sqrt(((tl[0] - bl[0]) ** 2) + ((tl[1] - bl[1]) ** 2))\n    maxHeight = max(int(heightA), int(heightB))\n    dst = np.array([\n        [0, 0],\n        [maxWidth - 1, 0],\n        [maxWidth - 1, maxHeight - 1],\n        [0, maxHeight - 1]], dtype = \"float32\")\n    M = cv2.getPerspectiveTransform(rect, dst)\n    warped = cv2.warpPerspective(image, M, (maxWidth, maxHeight))\n    return warped","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=True)\nfor i,data in enumerate(dataset):\n    image1, label1 = batch_to_numpy_images_and_labels(data)\n    image2 = four_point_transform(image1,None)\n    showimage(image1,image2)\n    if i == SHOW_LIMIT:\n        break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# This will not work directly, openCV can't work with TPU map functionality <br>\nWill have to keep everything in tensor"},{"metadata":{"trusted":true},"cell_type":"code","source":"## this will not work directly will have to keep everthing in tf\n# print('four_point_transform')\n# dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=True)\n# dataset_aug = dataset.map(four_point_transform)\n# for i,(data, data_aug) in enumerate(zip(dataset,dataset_aug)):\n#     image1, label1 = batch_to_numpy_images_and_labels(data)\n#     image2, label2 = batch_to_numpy_images_and_labels(data_aug)\n#     showimage(image1,image2)\n#     if i == SHOW_LIMIT:\n#         break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}