{"cells":[{"metadata":{"id":"4k1813v_04w_"},"cell_type":"markdown","source":"# Bi-Tempered Logistic Loss"},{"metadata":{"id":"jbNBKyZ-04xH"},"cell_type":"markdown","source":"# Dependency\n1. [Add this Modified BiTempered Logstic loss](https://www.kaggle.com/durbin164/bitempered-logistic-loss-tensorflow-v2) as dataset in your kernel OR Copy pest [this modified code](https://github.com/durbin-164/BiTemperedLogisticLossTensorflow/blob/master/bi_tempered_loss.py) in you kernel. "},{"metadata":{"id":"NyZ83_AW04xI"},"cell_type":"markdown","source":"* ## Acknowledgement:\n\n1. [TPU Custom training loop ](https://www.kaggle.com/mgornergoogle/custom-training-loop-with-100-flowers-on-tpu)\n\n2. [Bi-Tempered Logistic loss](https://github.com/google/bi-tempered-loss)\n\n3. Special Thanks [@Alexey Pronin](https://www.kaggle.com/graf10a) for helping me to make custom loss function as a class.\n4. [This awesome kernel](https://www.kaggle.com/dimitreoliveira/cassava-leaf-disease-tpu-tensorflow-training) help a lot to start training in TPU."},{"metadata":{"id":"N8An5bf504xI"},"cell_type":"markdown","source":"# [Inference](https://www.kaggle.com/durbin164/tpu-visual-transformer-vit-keras-tf-inferance)\n"},{"metadata":{"id":"idL5E2n004xJ"},"cell_type":"markdown","source":"## Tensorflow Version Update\n**Update Tensorflow version both in Kaggle kernel and TPU cluster.**"},{"metadata":{"trusted":true,"id":"tczCaxFI04xJ","executionInfo":{"status":"ok","timestamp":1610985435355,"user_tz":-360,"elapsed":1975,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}}},"cell_type":"code","source":"!pip install -U tensorflow==2.3.2\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"LqEB3mcx04xL","executionInfo":{"status":"ok","timestamp":1610985436273,"user_tz":-360,"elapsed":2874,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}}},"cell_type":"code","source":"print(\"update TPU server tensorflow version...\")\n\n!pip install cloud-tpu-client\nimport tensorflow as tf \nfrom cloud_tpu_client import Client\nprint(tf.__version__)\nClient().configure_tpu_version(tf.__version__, restart_type='ifNeeded')","execution_count":null,"outputs":[]},{"metadata":{"id":"QOJxFsoN04xL"},"cell_type":"markdown","source":"### Install model"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"id":"7F1s_WOM04xM","executionInfo":{"status":"ok","timestamp":1610985456261,"user_tz":-360,"elapsed":22851,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}},"outputId":"faff9f2d-de71-44b6-b000-310d9b0f3fe7"},"cell_type":"code","source":"# !pip install efficientnet\n! pip install vit-keras\n! pip install tensorflow_addons --upgrade\n# import os\n\n# os.environ['TF2_BEHAVIOR'] = '1'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"id":"axpSQev204xN","executionInfo":{"status":"ok","timestamp":1610985460646,"user_tz":-360,"elapsed":27224,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}},"outputId":"2d93134d-8f8c-4542-d98e-4c85b2a11e8e"},"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\n# import seaborn as sns\nimport os\nimport cv2\nimport re\nimport math\nimport datetime\nimport time\nfrom collections import namedtuple\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import confusion_matrix, classification_report\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.losses import CategoricalCrossentropy\nimport tensorflow as tf\n\nfrom tensorflow.keras.layers import Dense, Dropout,\\\n        Flatten,GlobalAveragePooling2D,BatchNormalization, Activation\n\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras import layers\n\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\n\nfrom kaggle_datasets import KaggleDatasets\n\n# import efficientnet.keras as efn \nfrom vit_keras import vit, utils\n\nprint(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"id":"UAdpWlTQ04xO"},"cell_type":"markdown","source":"#### Initialize TPU cluster"},{"metadata":{"trusted":true,"id":"IPBn1H5d04xO","executionInfo":{"status":"ok","timestamp":1610985472646,"user_tz":-360,"elapsed":39211,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}},"outputId":"89144637-bdc1-4a04-e3a1-341688f694b9"},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='')\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"s-zKSbr204xQ","executionInfo":{"status":"ok","timestamp":1610985472647,"user_tz":-360,"elapsed":39203,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}}},"cell_type":"code","source":"\n#setting\nSEED = 42\nDEBUG = False\nWANDB = False\n# TARGET_SIZE = 512\nVALIDATION_SIZE = 0.2\nBATCH_SIZE = 7 *REPLICAS\nLEARNING_RATE = 3e-5 * REPLICAS\nEPOCHS=40\nMODEL_NAME = \"ViTB16\"\nN_FOLDS = 5\nAUG_BATCH = BATCH_SIZE\n\nPATIENCE = 15  #This is for early stopping.\n\n#For BiTempered loss function\nT_1 = 0.3\nT_2 = 1.2\nSMOOTH_FRACTION = 0.05\nN_ITER = 5\n\n\nHEIGHT = 512\nWIDTH = 512\nHEIGHT_RS = HEIGHT\nWIDTH_RS = WIDTH\nCHANNELS = 3\nN_CLASSES = 5\nCLASSES = 5\nIMAGE_SIZE = [HEIGHT, WIDTH]\n\nMODEL_SAVE_PATH = \"\"\n\nif DEBUG:\n    EPOCHS = 2\n    # LEARNING_RATE = 3e-5 * REPLICAS\n    # TARGET_SIZE = 512\n    # BATCH_SIZE = 8*REPLICAS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"8t6qeZrD04xQ","executionInfo":{"status":"ok","timestamp":1610985472647,"user_tz":-360,"elapsed":39194,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}},"outputId":"3de3840a-8a92-4002-9623-89d997bee452"},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r'-([0-9]*)\\.').search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\noriginal_data = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification')+'/train_tfrecords' # Original dataset\n# # original_data = 'gs://kds-3128bc3ed453f4b14d6378d8d9756faab6734c137e2c9c1d6e52bc6e/train_tfrecords'\ncustom_512_512 = KaggleDatasets().get_gcs_path('cassava-leaf-disease-tfrecords-center-512x512')\n# external_custom_512_512 = KaggleDatasets().get_gcs_path('cassava-leaf-disease-tfrecords-external-512x512')\n\n# original_data ='gs://kds-8a8a0e757020ef17f93b37a540d540ccfa0003dbd9620ed6ef47ea9b/train_tfrecords'\n# custom_512_512 = 'gs://kds-917a8706bba391808573b1b102c95667cf39b48f2078727efccdbd38'\n\nprint(original_data)\nprint(custom_512_512)\n\n# GCS_PATH = os.path.join(external_custom_512_512)\n# print(GCS_PATH)\n# DATASET_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/*.tfrec')\n\n# NUM_FILES = len(DATASET_FILENAMES)\n# print(\"Number of files: \",NUM_FILES)\n\n# display(DATASET_FILENAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"q9N66zmF04xR","executionInfo":{"status":"ok","timestamp":1610985472648,"user_tz":-360,"elapsed":39184,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}},"outputId":"9013dc62-0489-4700-96ec-b3855bdf66d5"},"cell_type":"code","source":"\n# original_files = tf.io.gfile.glob(original_data + '/*.tfrec')\ncustom_files = tf.io.gfile.glob(custom_512_512 + '/*.tfrec')\n# external_files = tf.io.gfile.glob(external_custom_512_512 + '/*.tfrec')\n\nDATASET_FILENAMES = custom_files\n\n\nNUM_FILES = len(DATASET_FILENAMES)\nprint(\"Number of files: \",NUM_FILES)\n\nnp.random.shuffle(DATASET_FILENAMES)\n\ndisplay(DATASET_FILENAMES)\n","execution_count":null,"outputs":[]},{"metadata":{"id":"eafurvly04xT"},"cell_type":"markdown","source":"## Data Augmentation"},{"metadata":{"trusted":true,"id":"LZqV2UYe04xT","executionInfo":{"status":"ok","timestamp":1610985473014,"user_tz":-360,"elapsed":39539,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}}},"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\n# CutOut\ndef data_augment_cutout(image, min_mask_size=(int(HEIGHT * .1), int(HEIGHT * .1)), \n                        max_mask_size=(int(HEIGHT * .125), int(HEIGHT * .125))):\n    p_cutout = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    if p_cutout > .85: # 10~15 cut outs\n        n_cutout = tf.random.uniform([], 10, 15, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    elif p_cutout > .6: # 5~10 cut outs\n        n_cutout = tf.random.uniform([], 5, 10, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    elif p_cutout > .25: # 2~5 cut outs\n        n_cutout = tf.random.uniform([], 2, 5, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    else: # 1 cut out\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=1)\n\n    return image\n\ndef random_cutout(image, height, width, channels=3, min_mask_size=(10, 10), max_mask_size=(80, 80), k=1):\n    assert height > min_mask_size[0]\n    assert width > min_mask_size[1]\n    assert height > max_mask_size[0]\n    assert width > max_mask_size[1]\n\n    for i in range(k):\n      mask_height = tf.random.uniform(shape=[], minval=min_mask_size[0], maxval=max_mask_size[0], dtype=tf.int32)\n      mask_width = tf.random.uniform(shape=[], minval=min_mask_size[1], maxval=max_mask_size[1], dtype=tf.int32)\n\n      pad_h = height - mask_height\n      pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)\n      pad_bottom = pad_h - pad_top\n\n      pad_w = width - mask_width\n      pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)\n      pad_right = pad_w - pad_left\n\n      cutout_area = tf.zeros(shape=[mask_height, mask_width, channels], dtype=tf.uint8)\n\n      cutout_mask = tf.pad([cutout_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)\n      cutout_mask = tf.squeeze(cutout_mask, axis=0)\n      image = tf.multiply(tf.cast(image, tf.float32), tf.cast(cutout_mask, tf.float32))\n\n    return image","execution_count":null,"outputs":[]},{"metadata":{"id":"VWcClNASnl_6","executionInfo":{"status":"ok","timestamp":1610985473016,"user_tz":-360,"elapsed":39534,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}},"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/cdeotte/cutmix-and-mixup-on-gpu-tpu/notebook\n\ndef cutmix(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = IMAGE_SIZE[0]\n    # CLASSES = 104\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2\n\n\ndef mixup(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = IMAGE_SIZE[0]\n    # CLASSES = 104\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        # MAKE CUTMIX LABEL\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2\n\n\ndef aug_transform(image,label):\n    # THIS FUNCTION APPLIES BOTH CUTMIX AND MIXUP\n    DIM = IMAGE_SIZE[0]\n    # CLASSES = 104\n    SWITCH = 0.5\n    CUTMIX_PROB = 0.5\n    MIXUP_PROB = 0.5\n    # FOR SWITCH PERCENT OF TIME WE DO CUTMIX AND (1-SWITCH) WE DO MIXUP\n    image2, label2 = cutmix(image, label, CUTMIX_PROB)\n    image3, label3 = mixup(image, label, MIXUP_PROB)\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        P = tf.cast( tf.random.uniform([],0,1)<=SWITCH, tf.float32)\n        imgs.append(P*image2[j,]+(1-P)*image3[j,])\n        labs.append(P*label2[j,]+(1-P)*label3[j,])\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image4,label4\n#     return image2, label2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"8p7xhW7t04xZ","executionInfo":{"status":"ok","timestamp":1610985473018,"user_tz":-360,"elapsed":39529,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}}},"cell_type":"code","source":"def data_augment(image, label):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_cutout = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n\n    left_flip = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    right_flip = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n#     # Shear\n#     if p_shear > .2:\n#         if p_shear > .6:\n#             image = transform_shear(image, HEIGHT, shear=20.)\n#         else:\n#             image = transform_shear(image, HEIGHT, shear=-20.)\n            \n    # Rotation\n    if p_rotation > .3:\n        if p_rotation > .6:\n            image = transform_rotation(image, HEIGHT, rotation=15.)\n        else:\n            image = transform_rotation(image, HEIGHT, rotation=-15.)\n            \n    # Flips\n    if left_flip >0.4:\n        image = tf.image.random_flip_left_right(image)\n    if right_flip > 0.4:\n        image = tf.image.random_flip_up_down(image)\n#     if p_spatial > .75:\n#         image = tf.image.transpose(image)\n        \n#     # Rotates\n#     if p_rotate > .75:\n#         image = tf.image.rot90(image, k=3) # rotate 270º\n#     elif p_rotate > .5:\n#         image = tf.image.rot90(image, k=2) # rotate 180º\n#     elif p_rotate > .25:\n#         image = tf.image.rot90(image, k=1) # rotate 90º\n        \n    # # Pixel-level transforms\n    # if p_pixel_1 >= .4:\n    #     image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n#     if p_pixel_2 >= .4:\n#         image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n#     if p_pixel_3 >= .4:\n#         image = tf.image.random_brightness(image, max_delta=.1)\n        \n#     # Crops\n#     if p_crop > .6:\n#         if p_crop > .9:\n#             image = tf.image.central_crop(image, central_fraction=.5)\n#         elif p_crop > .8:\n#             image = tf.image.central_crop(image, central_fraction=.6)\n#         elif p_crop > .7:\n#             image = tf.image.central_crop(image, central_fraction=.7)\n#         else:\n#             image = tf.image.central_crop(image, central_fraction=.8)\n#     elif p_crop > .3:\n#         crop_size = tf.random.uniform([], int(HEIGHT*.6), HEIGHT, dtype=tf.int32)\n#         image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n            \n    image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n\n#     if p_cutout > .4:\n#         image = data_augment_cutout(image)\n\n        \n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"id":"ynbyIG4G04xe"},"cell_type":"markdown","source":"## Dataset prepare"},{"metadata":{"trusted":true,"id":"VIZ-1DOC04xf","executionInfo":{"status":"ok","timestamp":1610985473431,"user_tz":-360,"elapsed":39936,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}}},"cell_type":"code","source":"def one_hot(image, label):\n    \n    label = tf.one_hot(label, N_CLASSES, dtype = tf.float32)\n    \n    \n    return image,label\n\ndef get_normalize(image, label):\n    mean = tf.convert_to_tensor([0.485, 0.456, 0.406])\n    std = tf.convert_to_tensor([0.229, 0.224, 0.225])\n\n    image = (image-mean)/std\n\n    return image, label\n    \n\n# Datasets utility functions\ndef decode_image(image_data):\n    \"\"\"\n        1. Decode a JPEG-encoded image to a uint8 tensor.\n        2. Cast tensor to float and normalizes (range between 0 and 1).\n        3. Resize and reshape images to the expected size.\n    \"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n                      \n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_tfrecord(example, labeled=True):\n    \"\"\"\n        1. Parse data based on the 'TFREC_FORMAT' map.\n        2. Decode image.\n        3. If 'labeled' returns (image, label) if not (image, name).\n    \"\"\"\n    if labeled:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'target': tf.io.FixedLenFeature([], tf.int64), \n        }\n    else:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'image_name': tf.io.FixedLenFeature([], tf.string), \n        }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    if labeled:\n#         label_or_name = tf.one_hot(tf.cast(example['target'], tf.int32), N_CLASSES, dtype=tf.float64)\n        label_or_name = tf.cast(example['target'], tf.int32)\n    else:\n        label_or_name =  example['image_name']\n    return image, label_or_name\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    \"\"\"\n        Create a Tensorflow dataset from TFRecords.\n    \"\"\"\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda x: read_tfrecord(x, labeled=labeled), num_parallel_calls=AUTO)\n    return dataset\n\ndef get_dataset(FILENAMES, labeled=True, ordered=False, repeated=False, augment=False,\n                drop_remainder=False, transform = False, normalize = False):\n    \"\"\"\n        Return a Tensorflow dataset ready for training or inference.\n    \"\"\"\n    dataset = load_dataset(FILENAMES, labeled=labeled, ordered=ordered)\n    dataset = dataset.map(one_hot, num_parallel_calls=AUTO)\n    if augment:\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    \n    if repeated:\n        dataset = dataset.repeat()\n    if transform:\n        dataset = dataset.batch(AUG_BATCH)\n        dataset = dataset.map(aug_transform, num_parallel_calls=AUTO) # note we put AFTER batching\n        dataset = dataset.unbatch()\n    \n    if normalize :\n        dataset = dataset.map(get_normalize, num_parallel_calls=AUTO)\n        \n    if not ordered:\n        dataset = dataset.shuffle(SEED)\n    dataset = dataset.batch(BATCH_SIZE, drop_remainder=drop_remainder)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"id":"c5Sal8t504xg"},"cell_type":"markdown","source":"## Bi-Tempered Logistic loss function"},{"metadata":{"id":"piNFihUe04xg"},"cell_type":"markdown","source":"### If you add [this Loss function](https://www.kaggle.com/durbin164/bitempered-logistic-loss-tensorflow-v2) as dataset keep next cell otherwise replace next cell with [this link code](https://github.com/durbin-164/BiTemperedLogisticLossTensorflow/blob/master/bi_tempered_loss.py).\n\n"},{"metadata":{"trusted":true,"id":"80n5FdM704xh","executionInfo":{"status":"ok","timestamp":1610985474684,"user_tz":-360,"elapsed":41181,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}},"outputId":"69bf27a5-7990-4804-b192-70a3681f7435"},"cell_type":"code","source":"!git clone https://github.com/durbin-164/BiTemperedLogisticLossTensorflow.git\nimport sys\nsys.path.append('./BiTemperedLogisticLossTensorflow')\n# import all our functions\nfrom bi_tempered_loss import bi_tempered_logistic_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"qzqJ9_KS04xh","executionInfo":{"status":"ok","timestamp":1610985474685,"user_tz":-360,"elapsed":41173,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}}},"cell_type":"code","source":"with strategy.scope():\n  class BiTemperedLogisticLoss(tf.keras.losses.Loss):\n    def __init__(self, t1, t2, lbl_smth, n_iter):\n      super(BiTemperedLogisticLoss, self).__init__()\n      self.t1 = t1\n      self.t2 = t2\n      self.lbl_smth = lbl_smth\n      self.n_iter = n_iter\n\n    def call(self, y_true, y_pred):\n      return bi_tempered_logistic_loss(y_pred, y_true, self.t1, self.t2, self.lbl_smth, self.n_iter)","execution_count":null,"outputs":[]},{"metadata":{"id":"f_na2nL004xh"},"cell_type":"markdown","source":"## Prepared Model"},{"metadata":{"trusted":true,"id":"vRBLZ6-Y04xi","executionInfo":{"status":"ok","timestamp":1610985474686,"user_tz":-360,"elapsed":41167,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}}},"cell_type":"code","source":"with strategy.scope():\n    class LRSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n        def __call__(self, step):\n            return lrfn(epoch=step//STEPS_PER_EPOCH)\n    \n    def get_model():\n\n        # backbon = efn.EfficientNetB4(weights='noisy-student',\n        #                              include_top=False,\n        #                              input_shape=(HEIGHT, WIDTH,3))\n        \n        model = vit.vit_l16(\n                      image_size=HEIGHT,\n                      activation='softmax',\n                      pretrained=True,\n                      include_top=True,\n                      pretrained_top=False,\n                      classes = 5,\n                      weights = 'imagenet21k'\n                  )\n\n        model.trainable = True # Full Training\n\n        # model = tf.keras.Sequential([\n        #     backbon,\n        #     # tf.keras.layers.GlobalAveragePooling2D(),\n        #     tf.keras.layers.Dense(5, activation='softmax', name = 'masud')\n            \n        # ], name = \"masud2\")\n\n        \n\n\n        optimizer = tf.keras.optimizers.Adam(learning_rate=3e-6)\n        # loss = BiTemperedLogisticLoss(t1=T_1, t2=T_2, lbl_smth=SMOOTH_FRACTION, n_iter=N_ITER),\n        # loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=0.2, reduction= tf.keras.losses.Reduction.AUTO),\n\n        \n        model.compile(optimizer=optimizer,\n                      loss=tf.keras.losses.CategoricalCrossentropy(from_logits=False, label_smoothing=0.05, reduction= tf.keras.losses.Reduction.AUTO),\n                      metrics=['categorical_accuracy'], )\n\n\n        return model","execution_count":null,"outputs":[]},{"metadata":{"id":"j9sPeavJioaW","executionInfo":{"status":"ok","timestamp":1610985489286,"user_tz":-360,"elapsed":55761,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}},"outputId":"58dfb327-aeb1-43a7-81b0-ad290ab15002","trusted":true},"cell_type":"code","source":"# model = get_model()\n# model.save(\"mm.h5\", overwrite=True, include_optimizer=False)","execution_count":null,"outputs":[]},{"metadata":{"id":"orNC3R5A04xi"},"cell_type":"markdown","source":"## Learning rate Scheduler"},{"metadata":{"trusted":true,"id":"i4t03WQ504xj","executionInfo":{"status":"ok","timestamp":1610985489288,"user_tz":-360,"elapsed":55754,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}},"outputId":"1f2014eb-b3cb-4ab6-c424-4ce26acd529b"},"cell_type":"code","source":"LR_START = 0.00001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = 0.0000001\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n        \n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"id":"o-UcQmTS04xj"},"cell_type":"markdown","source":"## Callback function"},{"metadata":{"trusted":true,"id":"zHQbmk_i04xk","executionInfo":{"status":"ok","timestamp":1610985489644,"user_tz":-360,"elapsed":56101,"user":{"displayName":"Md. Masud Rana","photoUrl":"","userId":"05518923985814358127"}}},"cell_type":"code","source":"\ndef get_checkpoint(model_save_path, is_save_best = True):\n    return ModelCheckpoint(model_save_path, \n                             monitor= 'val_loss', \n                             verbose=1, \n                             save_best_only=is_save_best, \n                             mode= 'min', \n                             save_weights_only = True)\n    \ndef get_early_stopping():\n    return EarlyStopping(monitor = 'val_loss', min_delta = 0.0001, \n                           patience = PATIENCE, mode = 'min', verbose = 1,\n                           restore_best_weights = True)\n    \n\ndef get_learning_rate_decay():\n  return ReduceLROnPlateau(monitor = 'val_loss', factor = 0.6, \n                              patience = 2, min_delta = 0.0001, \n                              mode = 'min', verbose = 1)\n\n\ndef get_model_callback( fold_num):\n    model_save_path_best = f'{MODEL_NAME}_best_fold_{fold_num}_.h5'\n    print(\"Best model save path: \", model_save_path_best)\n\n    model_save_path_last = f'{MODEL_NAME}_last_fold_{fold_num}_.h5'\n    print(\"Last model save path: \", model_save_path_last)\n\n    checkpoint_best = get_checkpoint(model_save_path_best, is_save_best = True)\n    # checkpoint_last = get_checkpoint(model_save_path_last, is_save_best = False)\n\n    early_stopping = get_early_stopping()\n    lr_callback = get_learning_rate_decay()\n    # lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n    \n#     if WANDB:\n#         wandb.run.name= f'{MODEL_NAME}_fold_{fold_num}'\n#         [WandbCallback(),checkpoint_best, checkpoint_last, early_stopping]\n\n    return [checkpoint_best, early_stopping, lr_callback]","execution_count":null,"outputs":[]},{"metadata":{"id":"3CR3lmH_04xk"},"cell_type":"markdown","source":"## K-flod and train model"},{"metadata":{"trusted":true,"id":"4EuN6euX04xl","outputId":"983166c2-95bf-40ac-c6cd-43989bf01714"},"cell_type":"code","source":"skf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\noof_pred = []; oof_labels = []; history_list = []\n\n\nfor fold , (X_train, X_valid) in enumerate(skf.split(np.arange(NUM_FILES))):\n\n    print(\"Start Fold: \",fold);\n    # print(\"X_valid: \", X_valid)\n    MODEL_SAVE_PATH = f\"{MODEL_NAME}_{fold}.h5\"\n\n    if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n\n#     TRAIN_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/ld_train%.2i*.tfrec' % x for x in X_train])\n#     VALID_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/ld_train%.2i*.tfrec' % x for x in X_valid])\n    \n    TRAIN_FILENAMES = [DATASET_FILENAMES[x] for x in X_train]\n    VALID_FILENAMES = [DATASET_FILENAMES[x] for x in X_valid]\n    \n    \n#     np.random.shuffle(DATASET_FILENAMES)\n    \n    ct_train = count_data_items(TRAIN_FILENAMES)\n    ct_valid = count_data_items(VALID_FILENAMES)\n\n    print(\"Train File: \",ct_train)\n    print(\"Valid File: \", ct_valid)\n\n    STEPS_PER_EPOCH =  ct_train// BATCH_SIZE\n    VALIDATION_STEPS = ct_valid // BATCH_SIZE\n    print(\"Train Step: \",STEPS_PER_EPOCH)\n    print(\"Valid Step: \",VALIDATION_STEPS)\n\n\n    callback_list = get_model_callback(fold)\n\n    tf.keras.backend.clear_session()\n    \n    with strategy.scope():\n        model = get_model()\n\n    history = model.fit( x=get_dataset(TRAIN_FILENAMES, labeled=True, ordered=False, repeated=True,\n                                       augment=True,drop_remainder=False, transform = True),\n                            steps_per_epoch = STEPS_PER_EPOCH,\n                            epochs = EPOCHS,\n                            validation_data = get_dataset(VALID_FILENAMES, labeled=True, ordered=True, \n                                                          repeated=False, augment=False, drop_remainder=False), \n                            validation_steps = VALIDATION_STEPS,\n                            callbacks = callback_list\n                          )\n\n    history_list.append(history)\n\n\n\n    ds_valid = get_dataset(VALID_FILENAMES, labeled=True, ordered=True, repeated=False, augment=False)\n    oof_labels.append([target.numpy() for img, target in iter(ds_valid.unbatch())])\n    x_oof = ds_valid.map(lambda image, image_name: image)\n    oof_pred.append(np.argmax(model.predict(x_oof), axis=-1))\n    \n    ## RESULTS\n    print(f\"#### FOLD {fold} OOF Accuracy = {np.max(history.history['val_categorical_accuracy']):.5f}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"VaVlCKh704xl"},"cell_type":"code","source":"\ndef plot_hist(hist):\n    X_epochs = len(hist.history[\"categorical_accuracy\"])\n    plt.figure(figsize=(15,5))\n    plt.plot(np.arange(X_epochs), hist.history[\"categorical_accuracy\"], '-o', label='Train Accuracy',color='#ff7f0e')\n    plt.plot(np.arange(X_epochs), hist.history[\"val_categorical_accuracy\"], '-o',label='Val Accuracy',color='#1f77b4')\n    plt.xlabel('Epoch',size=14)\n    plt.ylabel('Accuracy',size=14)\n    plt.legend(loc=2)\n    \n    plt2 = plt.gca().twinx()\n    plt2.plot(np.arange(X_epochs) ,hist.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n    plt2.plot(np.arange(X_epochs) ,hist.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n    plt.legend(loc=3)\n    plt.ylabel('Loss',size=14)\n    plt.title(\"Model Accuracy and loss\")\n    \n    #plt.legend([\"train\", \"validation\"], loc=\"upper left\")\n    \n    plt.savefig('loss.png')\n    plt.show()\n\nfor fold, history in enumerate(history_list):\n    print(f'\\nFOLD: {fold}')\n    plot_hist(history) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"kmfI7WTK04xm"},"cell_type":"code","source":"y_true = np.argmax(np.concatenate(oof_labels),-1)\ny_preds = np.concatenate(oof_pred)\n\nprint(classification_report(y_true, y_preds))\n\nprint(\"****************\")\nprint(confusion_matrix(y_true, y_preds ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"M-fxppj104xn"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"LmWt270o04xn"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}