{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Bi-Tempered Logistic Loss"},{"metadata":{},"cell_type":"markdown","source":"# Dependency\n1. [Add this Modified BiTempered Logstic loss](https://www.kaggle.com/durbin164/bitempered-logistic-loss-tensorflow-v2) as dataset in your kernel OR Copy pest [this modified code](https://github.com/durbin-164/BiTemperedLogisticLossTensorflow/blob/master/bi_tempered_loss.py) in you kernel. "},{"metadata":{},"cell_type":"markdown","source":"# [Inference](https://www.kaggle.com/durbin164/tpu-bitempered-logistic-loss-inferance)\n"},{"metadata":{},"cell_type":"markdown","source":"## Tensorflow Version Update\n**Update Tensorflow version both in Kaggle kernel and TPU cluster.**"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"#!pip install -U tensorflow==2.3.2\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"print(\"update TPU server tensorflow version...\")\n\n!pip install cloud-tpu-client\nimport tensorflow as tf \nfrom cloud_tpu_client import Client\nprint(tf.__version__)\nClient().configure_tpu_version(tf.__version__, restart_type='ifNeeded')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Install efficient net"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"#!pip install efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\n# import seaborn as sns\nimport os\nimport cv2\nimport re\nimport math\nimport datetime\nimport time\nfrom collections import namedtuple\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import confusion_matrix, classification_report\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.losses import CategoricalCrossentropy\nimport tensorflow as tf\n\nfrom tensorflow.keras.layers import Dense, Dropout,\\\n        Flatten,GlobalAveragePooling2D,BatchNormalization, Activation\n\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras import layers\n\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\n\nfrom kaggle_datasets import KaggleDatasets\n\nprint(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='')\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#setting\nSEED = 123\nDEBUG = False\nWANDB = False\n# TARGET_SIZE = 512\nVALIDATION_SIZE = 0.08\nBATCH_SIZE = 8 *REPLICAS\nLEARNING_RATE = 8e-4 * REPLICAS\nEPOCHS=16\nPATIENT = 10\nMODEL_NAME = \"NetB+\"\nN_FOLDS = 3\nPATIENCE = 25  #This is for early stopping.\n\n#For BiTempered loss function\nT_1 = 0.3\nT_2 = 1.2\nSMOOTH_FRACTION = 0.1\nN_ITER = 5\n\n\nHEIGHT = 512\nWIDTH = 512\nHEIGHT_RS = 512\nWIDTH_RS = 512\nCHANNELS = 3\nN_CLASSES = 5\nCLASSES = 5\n\nMODEL_SAVE_PATH = \"\"\n\nif DEBUG:\n    EPOCHS = 10\n    # LEARNING_RATE = 3e-5 * REPLICAS\n    # TARGET_SIZE = 512\n    # BATCH_SIZE = 8*REPLICAS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r'-([0-9]*)\\.').search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\noriginal_data = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification')+'/train_tfrecords' # Original dataset\n# original_data = 'gs://kds-3128bc3ed453f4b14d6378d8d9756faab6734c137e2c9c1d6e52bc6e/train_tfrecords'\ncustom_512_512 = KaggleDatasets().get_gcs_path('cassava-leaf-disease-tfrecords-center-512x512')\nexternal_custom_512_512 = KaggleDatasets().get_gcs_path('cassava-leaf-disease-tfrecords-external-512x512')\n\nprint(original_data)\nprint(custom_512_512)\n\n# GCS_PATH = os.path.join(external_custom_512_512)\n# print(GCS_PATH)\n# DATASET_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/*.tfrec')\n\n# NUM_FILES = len(DATASET_FILENAMES)\n# print(\"Number of files: \",NUM_FILES)\n\n# display(DATASET_FILENAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\noriginal_files = tf.io.gfile.glob(original_data + '/*.tfrec')\ncustom_files = tf.io.gfile.glob(custom_512_512 + '/*.tfrec')\nexternal_files = tf.io.gfile.glob(external_custom_512_512 + '/*.tfrec')\n\nDATASET_FILENAMES = custom_files\n\n\nNUM_FILES = len(DATASET_FILENAMES)\nprint(\"Number of files: \",NUM_FILES)\n\nnp.random.shuffle(DATASET_FILENAMES)\n\ndisplay(DATASET_FILENAMES)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\n# CutOut\ndef data_augment_cutout(image, min_mask_size=(int(HEIGHT * .1), int(HEIGHT * .1)), \n                        max_mask_size=(int(HEIGHT * .125), int(HEIGHT * .125))):\n    p_cutout = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    if p_cutout > .85: # 10~15 cut outs\n        n_cutout = tf.random.uniform([], 10, 15, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    elif p_cutout > .6: # 5~10 cut outs\n        n_cutout = tf.random.uniform([], 5, 10, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    elif p_cutout > .25: # 2~5 cut outs\n        n_cutout = tf.random.uniform([], 2, 5, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    else: # 1 cut out\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=1)\n\n    return image\n\ndef random_cutout(image, height, width, channels=3, min_mask_size=(10, 10), max_mask_size=(80, 80), k=1):\n    assert height > min_mask_size[0]\n    assert width > min_mask_size[1]\n    assert height > max_mask_size[0]\n    assert width > max_mask_size[1]\n\n    for i in range(k):\n      mask_height = tf.random.uniform(shape=[], minval=min_mask_size[0], maxval=max_mask_size[0], dtype=tf.int32)\n      mask_width = tf.random.uniform(shape=[], minval=min_mask_size[1], maxval=max_mask_size[1], dtype=tf.int32)\n\n      pad_h = height - mask_height\n      pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)\n      pad_bottom = pad_h - pad_top\n\n      pad_w = width - mask_width\n      pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)\n      pad_right = pad_w - pad_left\n\n      cutout_area = tf.zeros(shape=[mask_height, mask_width, channels], dtype=tf.uint8)\n\n      cutout_mask = tf.pad([cutout_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)\n      cutout_mask = tf.squeeze(cutout_mask, axis=0)\n      image = tf.multiply(tf.cast(image, tf.float32), tf.cast(cutout_mask, tf.float32))\n\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_cutout = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n#     # Shear\n#     if p_shear > .2:\n#         if p_shear > .6:\n#             image = transform_shear(image, HEIGHT, shear=20.)\n#         else:\n#             image = transform_shear(image, HEIGHT, shear=-20.)\n            \n    # Rotation\n    if p_rotation > .2:\n        if p_rotation > .6:\n            image = transform_rotation(image, HEIGHT, rotation=45.)\n        else:\n            image = transform_rotation(image, HEIGHT, rotation=-45.)\n            \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n#     if p_spatial > .75:\n#         image = tf.image.transpose(image)\n        \n#     # Rotates\n#     if p_rotate > .75:\n#         image = tf.image.rot90(image, k=3) # rotate 270º\n#     elif p_rotate > .5:\n#         image = tf.image.rot90(image, k=2) # rotate 180º\n#     elif p_rotate > .25:\n#         image = tf.image.rot90(image, k=1) # rotate 90º\n        \n    # Pixel-level transforms\n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n        \n#     # Crops\n#     if p_crop > .6:\n#         if p_crop > .9:\n#             image = tf.image.central_crop(image, central_fraction=.5)\n#         elif p_crop > .8:\n#             image = tf.image.central_crop(image, central_fraction=.6)\n#         elif p_crop > .7:\n#             image = tf.image.central_crop(image, central_fraction=.7)\n#         else:\n#             image = tf.image.central_crop(image, central_fraction=.8)\n#     elif p_crop > .3:\n#         crop_size = tf.random.uniform([], int(HEIGHT*.6), HEIGHT, dtype=tf.int32)\n#         image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n            \n    image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n\n#    if p_cutout > .5:\n#        image = data_augment_cutout(image)\n        \n    return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset prepare"},{"metadata":{"trusted":true},"cell_type":"code","source":"def one_hot(image, label):\n    \n    label = tf.one_hot(label, N_CLASSES, dtype = tf.float32)\n    \n    \n    return image,label\n    \n\n# Datasets utility functions\ndef decode_image(image_data):\n    \"\"\"\n        1. Decode a JPEG-encoded image to a uint8 tensor.\n        2. Cast tensor to float and normalizes (range between 0 and 1).\n        3. Resize and reshape images to the expected size.\n    \"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float64) / 255.0\n                      \n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_tfrecord(example, labeled=True):\n    \"\"\"\n        1. Parse data based on the 'TFREC_FORMAT' map.\n        2. Decode image.\n        3. If 'labeled' returns (image, label) if not (image, name).\n    \"\"\"\n    if labeled:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'target': tf.io.FixedLenFeature([], tf.int64), \n        }\n    else:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'image_name': tf.io.FixedLenFeature([], tf.string), \n        }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    if labeled:\n#         label_or_name = tf.one_hot(tf.cast(example['target'], tf.int32), N_CLASSES, dtype=tf.float64)\n        label_or_name = tf.cast(example['target'], tf.int32)\n    else:\n        label_or_name =  example['image_name']\n    return image, label_or_name\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    \"\"\"\n        Create a Tensorflow dataset from TFRecords.\n    \"\"\"\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda x: read_tfrecord(x, labeled=labeled), num_parallel_calls=AUTO)\n    return dataset\n\ndef get_dataset(FILENAMES, labeled=True, ordered=False, repeated=False, augment=False, drop_remainder=False):\n    \"\"\"\n        Return a Tensorflow dataset ready for training or inference.\n    \"\"\"\n    dataset = load_dataset(FILENAMES, labeled=labeled, ordered=ordered)\n    dataset = dataset.map(one_hot, num_parallel_calls=AUTO)\n    if augment:\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    if repeated:\n        dataset = dataset.repeat()\n    if not ordered:\n        dataset = dataset.shuffle(SEED)\n    dataset = dataset.batch(BATCH_SIZE, drop_remainder=drop_remainder)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Bi-Tempered Logistic loss function"},{"metadata":{},"cell_type":"markdown","source":"### If you add [this Loss function](https://www.kaggle.com/durbin164/bitempered-logistic-loss-tensorflow-v2) as dataset keep next cell otherwise replace next cell with [this link code](https://github.com/durbin-164/BiTemperedLogisticLossTensorflow/blob/master/bi_tempered_loss.py).\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import module we'll need to import our custom module\nfrom shutil import copyfile\n\n# copy our file into the working directory (make sure it has .py suffix)\ncopyfile(src = \"../input/bitempered-logistic-loss-tensorflow-v2/bi_tempered_loss.py\", dst = \"../working/loss.py\")\n\n# import all our functions\nfrom loss import bi_tempered_logistic_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n  class BiTemperedLogisticLoss(tf.keras.losses.Loss):\n    def __init__(self, t1, t2, lbl_smth, n_iter):\n      super(BiTemperedLogisticLoss, self).__init__()\n      self.t1 = t1\n      self.t2 = t2\n      self.lbl_smth = lbl_smth\n      self.n_iter = n_iter\n\n    def call(self, y_true, y_pred):\n      return bi_tempered_logistic_loss(y_pred, y_true, self.t1, self.t2, self.lbl_smth, self.n_iter)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend as K\n\nimport dill\n\n\ndef binary_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n\n        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\n    return binary_focal_loss_fixed\n\n\ndef categorical_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Softmax version of focal loss.\n           m\n      FL = ∑  -alpha * (1 - p_o,c)^gamma * y_o,c * log(p_o,c)\n          c=1\n      where m = number of classes, c = class and o = observation\n    Parameters:\n      alpha -- the same as weighing factor in balanced cross entropy\n      gamma -- focusing parameter for modulating factor (1-p)\n    Default value:\n      gamma -- 2.0 as mentioned in the paper\n      alpha -- 0.25 as mentioned in the paper\n    References:\n        Official paper: https://arxiv.org/pdf/1708.02002.pdf\n        https://www.tensorflow.org/api_docs/python/tf/keras/backend/categorical_crossentropy\n    Usage:\n     model.compile(loss=[categorical_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def categorical_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred: A tensor resulting from a softmax\n        :return: Output tensor.\n        \"\"\"\n\n        # Scale predictions so that the class probas of each sample sum to 1\n        y_pred /= K.sum(y_pred, axis=-1, keepdims=True)\n\n        # Clip the prediction value to prevent NaN's and Inf's\n        epsilon = K.epsilon()\n        y_pred = K.clip(y_pred, epsilon, 1. - epsilon)\n\n        # Calculate Cross Entropy\n        cross_entropy = -y_true * K.log(y_pred)\n\n        # Calculate Focal Loss\n        loss = alpha * K.pow(1 - y_pred, gamma) * cross_entropy\n\n        # Sum the losses in mini_batch\n        return K.sum(loss, axis=1)\n\n    return categorical_focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepared Model"},{"metadata":{},"cell_type":"markdown","source":"## Learning rate Scheduler"},{"metadata":{"trusted":true},"cell_type":"code","source":"LR_START = 0.00003\nLR_MAX = 0.0008 * strategy.num_replicas_in_sync\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n        \n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) \n    return lr\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Callback function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_checkpoint(model_save_path, is_save_best = True):\n    return ModelCheckpoint(model_save_path, \n                             monitor= 'val_loss', \n                             verbose=0, \n                             save_best_only=is_save_best, \n                             mode= 'min', \n                             save_weights_only = False)\n    \ndef get_early_stopping():\n    return EarlyStopping(monitor = 'val_loss', min_delta = 0.00001, \n                           patience = PATIENCE, mode = 'min', verbose = 0,\n                           restore_best_weights = True)\n    \n\ndef get_learning_rate_decay():\n    return ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, \n                              patience = 10, min_delta = 0.00001, \n                              mode = 'min', verbose = 0)\n\n\ndef get_model_callback( fold_num):\n\n\n    model_save_path = f'{MODEL_NAME}_last_fold_{fold_num}_.h5'\n    print(\"Last model save path: \", model_save_path)\n\n    checkpoint = get_checkpoint(model_save_path, is_save_best = True)\n\n    early_stopping = get_early_stopping()\n#     learning_rate_decay = get_learning_rate_decay()\n#    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    \n#     if WANDB:\n#         wandb.run.name= f'{MODEL_NAME}_fold_{fold_num}'\n#         [WandbCallback(),checkpoint_best, checkpoint_last, early_stopping]\n\n    return [checkpoint, early_stopping]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\nfrom keras.models import load_model\nIMAGE_SIZE = [512, 512]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_pred1 = []; oof_pred2 = []; oof_pred3 = []; oof_pred4 = []; oof_labels = []; history_list = []\n\n\nif tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n\n#     TRAIN_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/ld_train%.2i*.tfrec' % x for x in X_train])\n#     VALID_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/ld_train%.2i*.tfrec' % x for x in X_valid])\n\nTRAIN_FILENAMES = DATASET_FILENAMES\nVALID_FILENAMES = DATASET_FILENAMES[10:12]\n\n\n#     np.random.shuffle(DATASET_FILENAMES)\n\nct_train = count_data_items(TRAIN_FILENAMES)\nct_valid = count_data_items(VALID_FILENAMES)\n\nprint(\"Train File: \",ct_train)\nprint(\"Valid File: \", ct_valid)\n\nSTEPS_PER_EPOCH =  ct_train// BATCH_SIZE\nVALIDATION_STEPS = ct_valid // BATCH_SIZE\nprint(\"Train Step: \",STEPS_PER_EPOCH)\nprint(\"Valid Step: \",VALIDATION_STEPS)\n\n\ntf.keras.backend.clear_session()\n\nwith strategy.scope():\n    model1=keras.models.load_model('../input/cassava-w/d3x8_cassavaa.h5')\n\nprint('..........:::::::::::::::::::::::::::::::::LOADDED MODELS::::::::::::::::::::::::::::::::::::..............')\n\nds_valid = get_dataset(VALID_FILENAMES, labeled=True, ordered=True, repeated=False, augment=False)\noof_labels.append([target.numpy() for img, target in iter(ds_valid.unbatch())])\nx_oof = ds_valid.map(lambda image, image_name: image)\noof_pred1.append(np.argmax(model1.predict(x_oof), axis=-1))\nprint('pred-OK1')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_true = np.argmax(np.concatenate(oof_labels),-1)\ny_preds = np.concatenate(oof_pred1)\n\nprint(classification_report(y_true, y_preds))\n\nprint(\"****************cassavaNet\")\nprint(confusion_matrix(y_true, y_preds ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**all rights reserved**"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}