{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q --upgrade pip\n!pip install -q efficientnet","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_probability as tfp\nimport matplotlib.pyplot as plt\nimport efficientnet.tfkeras as efn\nimport seaborn as sns\n\nfrom kaggle_datasets import KaggleDatasets\nfrom keras.applications import ResNet50\n\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nfrom tqdm.notebook import tqdm\nfrom sklearn.metrics import classification_report, confusion_matrix\n\nimport sys\nimport glob\nimport math\nimport gc\nimport time\n\nprint(f'tensorflow version: {tf.__version__}')\nprint(f'tensorflow keras version: {tf.keras.__version__}')\nprint(f'python version: P{sys.version}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')\n\n# # set half precision policy\nmixed_precision.set_policy('mixed_bfloat16')\n\n# enable XLA optmizations\ntf.config.optimizer.set_jit(True)\n\nprint(f'Compute dtype: {mixed_precision.global_policy().compute_dtype}')\nprint(f'Variable dtype: {mixed_precision.global_policy().variable_dtype}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_HEIGHT = 600\nIMG_WIDTH = 800\n\nIMG_SIZE = 600\nIMG_TARGET_SIZE = 512\nN_CHANNELS = 3\n\nN_TRAIN_IMGS = 21642\nN_VAL_IMGS = 5410\nBATCH_SIZE_VAL = 128 * REPLICAS # 5410 / 8 / 4\n\nN_LABELS = 5\nN_FOLDS = 5\nEPOCHS = 30\n\nBATCH_SIZE_BASE = 16\nBATCH_SIZE = BATCH_SIZE_BASE * REPLICAS\n\nTARGET_DTYPE = tf.bfloat16\n\n# ImageNet mean and standard deviation\nIMAGENET_MEAN = tf.constant([0.485, 0.456, 0.406], dtype=tf.float32)\nIMAGENET_STD = tf.constant([0.229, 0.224, 0.225], dtype=tf.float32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path('cassava-leaf-disease-tfrecords-600x600')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_tfrecord_train(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'label': tf.io.FixedLenFeature([], tf.int64),\n        'height': tf.io.FixedLenFeature([], tf.int64),\n        'width': tf.io.FixedLenFeature([], tf.int64),\n    })\n    \n    height = features['height']\n    width = features['width']\n\n    image = tf.io.decode_jpeg(features['image'])\n    image = tf.reshape(image, [height, width, N_CHANNELS])\n    \n    # get random square\n    if height > width:\n        offset = tf.random.uniform(shape=(), minval=0, maxval=height-width, dtype=tf.int64)\n        image = tf.slice(image, [offset, 0, 0], [width, width, N_CHANNELS])\n    elif width > height:\n        offset = tf.random.uniform(shape=(), minval=0, maxval=width-height, dtype=tf.int64)\n        image = tf.slice(image, [0, offset, 0], [height, height, N_CHANNELS])\n    else:\n        image = tf.slice(image, [0, 0, 0], [height, width, N_CHANNELS])\n        \n    size = tf.cast(height if height < width else width, tf.float32)\n    \n    # cast label to int8\n    label = tf.cast(features['label'], tf.uint8)\n\n    return image, label, size","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# chance of x in y to return true, used for conditional data augmentation\ndef chance(x, y):\n    return tf.random.uniform(shape=[], minval=0, maxval=y, dtype=tf.int32) < x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augment_image(image, label, size):\n    # random flip image horizontally\n    image = tf.image.random_flip_left_right(image)\n    # random flip image vertically\n    image = tf.image.random_flip_up_down(image)\n    \n    # random transpose\n    if chance(1,2):\n        image = tf.image.transpose(image)\n    \n    # random crop between 75%-100%\n    crop_size = tf.random.uniform(shape=(), minval=size*0.75, maxval=size)\n    image = tf.image.random_crop(image, [crop_size, crop_size, N_CHANNELS])\n    \n    # cast to target dtype and resize\n    image = tf.image.resize(image, [IMG_TARGET_SIZE, IMG_TARGET_SIZE])\n    \n    # normalize according to imagenet mean and std\n    image /= 255.0\n    image = (image - IMAGENET_MEAN) / IMAGENET_STD\n    \n    # one hot encode label\n    label = tf.one_hot(label, N_LABELS, dtype=tf.float32)\n    \n    return image, label\n\ndef read_augment_image(record_bytes):\n    image, label, size = decode_tfrecord_train(record_bytes)\n    image, label = augment_image(image, label, size)\n    \n    return image, label\n\ndef get_mix_img_idx(labels_idxs, idx):\n    idx_candidates = tf.where(labels_idxs != idx)\n    r = tf.random.uniform(minval=0, maxval=len(idx_candidates), shape=[], dtype=tf.int32)\n    idx = tf.gather(idx_candidates, r)\n    idx = tf.cast(idx, tf.int32)\n    idx = tf.squeeze(idx)\n    \n    return idx","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mixup Implementation","metadata":{}},{"cell_type":"code","source":"def mixup(images, labels, alpha=0.40):\n    l = len(images)\n    # get image factors\n    a = tfp.distributions.Beta(alpha, alpha).sample(l)\n    a_label = tf.reshape(a, shape=(l,1))\n    a_label = tf.tile(a_label, [1, N_LABELS])\n    b_label = 1 - a_label\n    \n    a_image = tf.reshape(a, shape=(l,1,1,1))\n    a_image = tf.tile(a_image, [1, IMG_TARGET_SIZE, IMG_TARGET_SIZE ,N_CHANNELS])\n    a_image = tf.cast(a_image, tf.float32)\n    b_image = 1 - a_image\n    \n    # get mixup image indices\n    if l == 2:\n        idxs = tf.constant([1, 0])\n    else:\n        labels_idxs = tf.range(len(labels))\n        idxs = tf.map_fn(lambda idx: get_mix_img_idx(labels_idxs, idx), tf.range(len(labels)))\n    \n    images_mixup = tf.gather(images, idxs)\n    labels_mixup = tf.gather(labels, idxs)\n    \n    # mixup images and labels\n    images =  images * a_image + images_mixup * b_image\n    labels = labels * a_label + labels_mixup * b_label\n    \n    images = tf.cast(images, TARGET_DTYPE)\n    \n    return images, labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cutmix","metadata":{}},{"cell_type":"code","source":"def create_cutmix_mask(a):\n    # create random mask size and coordinates\n    r_w = tf.cast(IMG_TARGET_SIZE * tf.math.sqrt(1 - a), tf.int32)\n    r_h = tf.cast(IMG_TARGET_SIZE * tf.math.sqrt(1 - a), tf.int32)\n    \n    if r_w == IMG_TARGET_SIZE:\n        r_x = 0\n    else:\n        r_x = tf.random.uniform(minval=0, maxval=IMG_TARGET_SIZE - r_w, shape=[], dtype=tf.int32)\n        \n    if r_h == IMG_TARGET_SIZE:\n        r_y = 0\n    else:\n        r_y = tf.random.uniform(minval=0, maxval=IMG_TARGET_SIZE - r_w, shape=[], dtype=tf.int32)\n\n    # compute padding sizes\n    pad_left = r_x\n    pad_right = IMG_TARGET_SIZE - (r_x + r_w)\n    pad_top = r_y\n    pad_bottom = IMG_TARGET_SIZE - (r_y + r_h)\n    \n    # create mask_a and mask_b\n    mask_a = tf.ones(shape=[r_w, r_h], dtype=tf.float32)\n    mask_a = tf.pad(mask_a, [[pad_left, pad_right], [pad_top, pad_bottom]], mode='CONSTANT', constant_values=0)\n    mask_a = tf.expand_dims(mask_a, axis=2)\n    \n    return mask_a\n\ndef cutmix(images, labels):\n    l = len(images)\n    a_float32 = tfp.distributions.Beta(1.0, 1.0).sample([l])\n\n    mask_b = tf.map_fn(create_cutmix_mask, a_float32)\n    mask_a = tf.math.abs(mask_b - 1)\n    \n    # images_idxs\n    if l == 2:\n        idxs = tf.constant([1, 0])\n    else:\n        labels_idxs = tf.range(len(labels))\n        idxs = tf.map_fn(lambda idx: get_mix_img_idx(labels_idxs, idx), tf.range(len(labels)))\n    \n    images_cutmix = tf.gather(images, idxs)\n    labels_cutmix = tf.gather(labels, idxs)\n    \n    a_float32_labels = tf.expand_dims(a_float32, axis=1)\n    a_float32_labels = tf.repeat(a_float32_labels, N_LABELS, axis=1)\n    labels_factor = a_float32_labels\n    labels_cutmix_factor = 1 - a_float32_labels\n    \n    # cutmix images and labels\n    images = images * mask_a + images_cutmix * mask_b\n    labels = labels * labels_factor + labels_cutmix * labels_cutmix_factor\n    \n    images = tf.cast(images, TARGET_DTYPE)\n    \n    return images, labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Gridmask","metadata":{}},{"cell_type":"code","source":"def gridmask(images, labels):\n    l = len(images)\n    \n    d = tf.random.uniform(minval=int(IMG_TARGET_SIZE * (96/224)), maxval=IMG_TARGET_SIZE, shape=[], dtype=tf.int32)\n    grid = tf.constant([[[0], [1]],[[1], [0]]], dtype=tf.float32)\n    grid = tf.image.resize(grid, [d, d], method='nearest')\n    \n    # 50% chance to rotate mask\n    if chance(1, 2):\n        grid = tf.image.rot90(grid, 1)\n\n    repeats = IMG_TARGET_SIZE // d + 1\n    grid = tf.tile(grid, multiples=[repeats, repeats, 1])\n    grid = tf.image.random_crop(grid, [IMG_TARGET_SIZE, IMG_TARGET_SIZE, 1])\n    grid = tf.expand_dims(grid, axis=0)\n    grid = tf.tile(grid, multiples=[l, 1, 1, 1])\n\n    images = images * grid\n    images = tf.cast(images, TARGET_DTYPE)\n    \n    return images, labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augment_batch(images, labels, augmentations=None):\n    if augmentations is None:\n        r = tf.random.uniform(minval=0, maxval=4, shape=[], dtype=tf.int32)\n    else:\n        r = tf.random.uniform(minval=0, maxval=len(augmentations), shape=[], dtype=tf.int32)\n        r = tf.gather(augmentations, r)\n        \n    if r == 0:\n        images = tf.cast(images, TARGET_DTYPE)\n        return images, labels\n    elif r == 1:\n        return mixup(images, labels)\n    elif r == 2:\n        return cutmix(images, labels)\n    elif r == 3:\n        return gridmask(images, labels)\n    else:\n        images = tf.cast(images, TARGET_DTYPE)\n        return images, labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reshape_batch(images, labels):\n    images = tf.reshape(images, shape=[BATCH_SIZE, IMG_TARGET_SIZE, IMG_TARGET_SIZE, N_CHANNELS])\n    labels = tf.reshape(labels, shape=[BATCH_SIZE, N_LABELS])\n    \n    random_idxs = tf.random.shuffle(tf.range(BATCH_SIZE))\n    images = tf.gather(images, random_idxs)\n    labels = tf.gather(labels, random_idxs)\n    \n    return images, labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_dataset(bs=BATCH_SIZE, fold=0, augmentations=None):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False\n    \n    FNAMES_TRAIN_TFRECORDS = tf.io.gfile.glob(f'{GCS_DS_PATH}/fold_{fold}/train/*.tfrecords')\n    train_dataset = tf.data.TFRecordDataset(FNAMES_TRAIN_TFRECORDS, num_parallel_reads=AUTO)\n    train_dataset = train_dataset.with_options(ignore_order)\n    train_dataset = train_dataset.prefetch(AUTO)\n    train_dataset = train_dataset.repeat()\n    train_dataset = train_dataset.map(read_augment_image, num_parallel_calls=AUTO)\n\n    train_dataset = train_dataset.batch(BATCH_SIZE_BASE)\n    train_dataset = train_dataset.map(lambda images, labels: augment_batch(images, labels, augmentations=augmentations), num_parallel_calls=REPLICAS)\n    \n    train_dataset = train_dataset.batch(REPLICAS)\n    train_dataset = train_dataset.map(reshape_batch, num_parallel_calls=1)\n    \n    train_dataset = train_dataset.prefetch(1)\n    \n    return train_dataset\n\ntrain_dataset = get_train_dataset()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def benchmark(num_epochs=3, n_steps_per_epoch=10, augmentations=None, bs=BATCH_SIZE):\n    dataset = get_train_dataset(augmentations=augmentations)\n    start_time = time.perf_counter()\n    for epoch_num in range(num_epochs):\n        epoch_start = time.perf_counter()\n        for idx, (images, labels) in enumerate(dataset.take(n_steps_per_epoch)):\n            if idx is 1:\n                print(images.shape, labels.shape)\n            pass\n        print(f'epoch {epoch_num} took: {round(time.perf_counter() - epoch_start, 2)}')\n    print(\"Execution time:\", round(time.perf_counter() - start_time, 2))\n    \nbenchmark(num_epochs=3, augmentations=[2,3])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Validation dataset","metadata":{}},{"cell_type":"code","source":"def resize_image(image, label, size):\n    image = tf.image.resize(image, [IMG_TARGET_SIZE, IMG_TARGET_SIZE])\n    \n    return image, label, tf.cast(IMG_TARGET_SIZE, tf.float32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_tfrecord_val(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'label': tf.io.FixedLenFeature([], tf.int64),\n        'height': tf.io.FixedLenFeature([], tf.int64),\n        'width': tf.io.FixedLenFeature([], tf.int64),\n    })\n    \n    height = features['height']\n    width = features['width']\n\n    image = tf.io.decode_jpeg(features['image'])\n    image = tf.reshape(image, [height, width, N_CHANNELS])\n    \n    # get random square\n    if height > width:\n        offset = (height - width) // 2\n        image = tf.slice(image, [offset, 0, 0], [width, width, N_CHANNELS])\n    elif width > height:\n        offset = (width - height) // 2\n        image = tf.slice(image, [0, offset, 0], [height, height, N_CHANNELS])\n    else:\n        image = tf.slice(image, [0, 0, 0], [height, width, N_CHANNELS])\n    \n    # resize to target size\n    image = tf.image.resize(image, [IMG_TARGET_SIZE, IMG_TARGET_SIZE])\n    \n    # normalize according to imagenet mean and std\n    image /= 255.0\n    image = (image - IMAGENET_MEAN) / IMAGENET_STD\n    \n    # cast to TARGET_DTYPE\n    image = tf.cast(image, TARGET_DTYPE)\n    \n    label = tf.cast(features['label'], tf.int32)\n    \n    # one hot encode label\n    label = tf.one_hot(label, N_LABELS, dtype=tf.int32)\n    \n    return image, label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_val_dataset(bs=BATCH_SIZE, fold=0):\n    FNAMES_VAL_TFRECORDS = tf.io.gfile.glob(f'{GCS_DS_PATH}/fold_{fold}/val/*.tfrecords')\n    val_dataset = tf.data.TFRecordDataset(FNAMES_VAL_TFRECORDS, num_parallel_reads=AUTO)\n    val_dataset = val_dataset.prefetch(BATCH_SIZE_VAL)\n    val_dataset = val_dataset.repeat()\n    val_dataset = val_dataset.map(decode_tfrecord_val, num_parallel_calls=AUTO)\n    val_dataset = val_dataset.batch(bs, drop_remainder=True)\n    val_dataset = val_dataset.prefetch(1)\n    \n    return val_dataset\n\nval_dataset = get_val_dataset()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lr scheduler","metadata":{}},{"cell_type":"code","source":"def lrfn(epoch, bs=BATCH_SIZE, epochs=EPOCHS):\n    # Config\n    LR_START = 1e-6\n    LR_MAX = 2e-4\n    LR_FINAL = 1e-6\n    LR_RAMPUP_EPOCHS = 4\n    LR_SUSTAIN_EPOCHS = 0\n    DECAY_EPOCHS = epochs  - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\n    LR_EXP_DECAY = (LR_FINAL / LR_MAX) ** (1 / (EPOCHS - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1))\n\n    if epoch < LR_RAMPUP_EPOCHS: # exponential warmup\n        lr = LR_START + (LR_MAX + LR_START) * (epoch / LR_RAMPUP_EPOCHS) ** 2.5\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS: # sustain lr\n        lr = LR_MAX\n    else: # cosine decay\n        epoch_diff = epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n        decay_factor = (epoch_diff / DECAY_EPOCHS) * math.pi\n        decay_factor= (tf.math.cos(decay_factor).numpy() + 1) / 2        \n        lr = LR_FINAL + (LR_MAX - LR_FINAL) * decay_factor\n\n    return lr\n\ndef lrfn2(epoch):\n    \n    LR_START = 0.00001\n    LR_MAX = 0.00005 * strategy.num_replicas_in_sync\n    LR_MIN = 0.00001\n    LR_RAMPUP_EPOCHS = 4\n    LR_SUSTAIN_EPOCHS = 4\n    LR_EXP_DECAY = .8\n\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using Binary and Categorical focal loss","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import backend as K\nimport dill\n\n\ndef binary_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n\n        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\n    return binary_focal_loss_fixed\n\n\ndef categorical_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Softmax version of focal loss.\n           m\n      FL = ∑  -alpha * (1 - p_o,c)^gamma * y_o,c * log(p_o,c)\n          c=1\n      where m = number of classes, c = class and o = observation\n    Parameters:\n      alpha -- the same as weighing factor in balanced cross entropy\n      gamma -- focusing parameter for modulating factor (1-p)\n    Default value:\n      gamma -- 2.0 as mentioned in the paper\n      alpha -- 0.25 as mentioned in the paper\n    References:\n        Official paper: https://arxiv.org/pdf/1708.02002.pdf\n        https://www.tensorflow.org/api_docs/python/tf/keras/backend/categorical_crossentropy\n    Usage:\n     model.compile(loss=[categorical_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def categorical_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred: A tensor resulting from a softmax\n        :return: Output tensor.\n        \"\"\"\n\n        # Scale predictions so that the class probas of each sample sum to 1\n        y_pred /= K.sum(y_pred, axis=-1, keepdims=True)\n\n        # Clip the prediction value to prevent NaN's and Inf's\n        epsilon = K.epsilon()\n        y_pred = K.clip(y_pred, epsilon, 1. - epsilon)\n        \n        tf.cast(y_pred, tf.float32)\n        tf.cast(y_true, tf.float32)\n        # Calculate Cross Entropy\n        cross_entropy = -y_true * K.log(y_pred)\n        print(f'type of y_pred ---- {type(y_pred)}')\n        print(f'type of y_true ---- {type(y_true)}')\n\n        # Calculate Focal Loss\n        loss = alpha * K.pow(1 - y_pred, gamma) * cross_entropy\n\n        print(f'type of k_sum ---- {type(K.sum(loss, axis=1))}')\n        tf.cast(loss, tf.float32)\n        # Sum the losses in mini_batch\n        return_list = K.sum(loss, axis=1)\n        \n        tf.cast(return_list, tf.float32)\n#         return K.sum(loss, axis=1)\n        return return_list\n\n    return categorical_focal_loss_fixed","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def get_model(choice):\n    # reset to free memory and training variables\n    tf.keras.backend.clear_session()\n    \n    \n    net = net_choices.get(choice)\n    with strategy.scope():\n        \n        \n        if (choice==0):\n            net = efn.EfficientNetB4(\n                include_top=False,\n                weights='noisy-student',\n                input_shape=(IMG_TARGET_SIZE, IMG_TARGET_SIZE, N_CHANNELS),\n            )\n        elif (choice==1):\n            net = ResNet50(\n                weights='imagenet',\n                include_top=False,\n            )\n        elif (choice==2):\n            net=tf.keras.applications.DenseNet201(\n                weights='imagenet',\n                include_top=False\n\n            )\n        \n        for layer in reversed(net.layers):\n            if isinstance(layer, tf.keras.layers.BatchNormalization):\n                layer.trainable = False\n                \n            else:\n                layer.trainable = True\n        \n        model = tf.keras.Sequential([\n            net,\n            tf.keras.layers.Dropout(0.45),\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dropout(0.45),\n            tf.keras.layers.Dense(N_LABELS, activation='softmax', dtype=tf.float32),\n        ])\n\n        # add metrics\n        metrics = [\n            tf.keras.metrics.CategoricalAccuracy(name='accuracy'),\n            tf.keras.metrics.TopKCategoricalAccuracy(k=2, name='top_2_accuracy'),\n        ]\n\n        optimizer = tf.keras.optimizers.Adam()\n        loss = tf.keras.losses.CategoricalCrossentropy()\n        cat_loss = categorical_focal_loss(gamma=2., alpha=.25)\n        \n        model.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n#         model.summary()\n        return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation function","metadata":{}},{"cell_type":"code","source":"def show_validation_report_per_class(model, dataset, steps, name, bs):\n    print(f'--- {name} REPORT ---')\n    # classification report\n    y = np.ndarray(shape=steps * bs, dtype=np.uint16)\n    y_pred = np.ndarray(shape=steps * bs, dtype=np.uint16)\n    for idx, (images, labels) in tqdm(enumerate(dataset.take(steps)), total=steps):\n        with tf.device('cpu:0'):\n            y[idx*bs:(idx+1)*bs] = np.argmax(labels, axis=1)\n            y_pred[idx*bs:(idx+1)*bs] = np.argmax(model.predict(images).astype(np.float32), axis=1)\n            \n    print(classification_report(y, y_pred))\n    \n    # Confusion matrix\n    fig, ax = plt.subplots(1, 1, figsize=(20, 12))\n    cfn_matrix = confusion_matrix(y, y_pred, labels=range(N_LABELS))\n    cfn_matrix = (cfn_matrix.T / cfn_matrix.sum(axis=1)).T\n    df_cm = pd.DataFrame(cfn_matrix, index=np.arange(N_LABELS), columns=np.arange(N_LABELS))\n    ax = sns.heatmap(df_cm, cmap='Blues', annot=True, fmt='.3f', linewidths=.7, annot_kws={'size':14}).set_title(f'{name} CONFUSION MATRIX')\n    plt.xticks(fontsize=16)\n    plt.yticks(fontsize=16)\n    plt.xlabel('PREDICTED', fontsize=24, labelpad=10)\n    plt.ylabel('ACTUAL', fontsize=24, labelpad=10)\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting curves function","metadata":{}},{"cell_type":"code","source":"def plot_history_metric(history, metric):\n    TRAIN_EPOCHS = len(history.history['loss'])\n    x = np.arange(TRAIN_EPOCHS)\n    x_axis_labels = list(map(str, np.arange(1, TRAIN_EPOCHS+1)))\n    val = 'val' in ''.join(history.history.keys())\n    # summarize history for accuracy\n    plt.figure(figsize=(20, 10))\n    plt.plot(history.history[metric])\n    if val:\n        plt.plot(history.history[f'val_{metric}'])\n    \n    plt.title(f'Model {metric}', fontsize=30)\n    plt.ylabel(metric, fontsize=26)\n    plt.yticks(fontsize=20)\n    plt.xlabel('epoch', fontsize=26)\n    plt.xticks(x, x_axis_labels, fontsize=16) # set tick step to 1 and let x axis start at 1\n    plt.legend(['train'] + ['test'] if val else [], loc='upper left')\n    plt.grid()\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Running model","metadata":{}},{"cell_type":"code","source":"print(f'TRAINING FOR {EPOCHS} EPOCHS WITH BATCH SIZE {BATCH_SIZE}\\n')\nprint(f'TRAIN IMAGES: {N_TRAIN_IMGS}, VAL IMAGES: {N_VAL_IMGS}\\n')\n\naugmentations_dic = dict({\n    0: 'None',\n    1: 'MixUp',\n    2: 'CutMix',\n    3: 'GridMask',\n})\n\nnet_choices = dict({\n    0: \"Efficientnet\",\n    1: \"ResNet\"\n})\n    \n\n\naugmentations = [2, 3] # only CutMix and GridMask is used\nchoice = 1   # choice can be 0 or 1 according to the dictionary given above\n\n# MEAN_VAL_ACC = []\n# fold = 0\n# epochs = EPOCHS\n\nfor choice in [0, 1, 2]:\n    MEAN_VAL_ACC = []\n    fold = 0\n    epochs = EPOCHS\n    for idx, fold in enumerate(range(N_FOLDS)):\n        # callbacks\n        lr_callback_1 = tf.keras.callbacks.LearningRateScheduler(lambda epoch: lrfn(epoch, epochs=epochs), verbose=1)\n    #     lr_callback_2 = tf.keras.callbacks.LearningRateScheduler(lrfn2, verbose = True)\n    #     show_lr_schedule(epochs=epochs)\n\n        # get the model\n        model = get_model(choice)\n\n        if idx is 0:\n            # model summary\n            model.summary()\n            # compute and variable data types\n            print(f'Compute dtype: {mixed_precision.global_policy().compute_dtype}')\n            print(f'Variable dtype: {mixed_precision.global_policy().variable_dtype}')\n\n        print('\\n')\n        print('*'*25, f'augmentations {augmentations}', '*'*25, '\\n')\n        print(f'fold: {fold}, epochs: {epochs}')\n        print(' AND '.join([augmentations_dic.get(i) for i in augmentations]), '\\n')\n\n        train_dataset = get_train_dataset(bs=BATCH_SIZE, fold=fold, augmentations=augmentations)\n        val_dataset = get_val_dataset(bs=BATCH_SIZE_VAL, fold=fold)\n\n        text_file = f\"profiling_{choice}.txt\"\n        \n        %prun -T text_file history = model.fit(train_dataset,steps_per_epoch = N_TRAIN_IMGS // BATCH_SIZE,validation_data = val_dataset,validation_steps = N_VAL_IMGS // BATCH_SIZE_VAL,epochs = epochs,callbacks = [lr_callback_1],verbose=1)\n\n        # add val accuracy to list\n        MEAN_VAL_ACC.append(history.history['val_accuracy'][-1])\n\n        # plot training histories\n        plot_history_metric(history, 'loss')\n        plot_history_metric(history, 'accuracy')\n        plot_history_metric(history, 'top_2_accuracy')\n\n        # show train and validation report\n        show_validation_report_per_class(model, val_dataset, N_VAL_IMGS // BATCH_SIZE_VAL, 'VALIDATION', BATCH_SIZE_VAL)\n\n        # save the model\n        model.save_weights(f'model_fold_{fold}_weights.h5')\n        model.save(f'model_{net_choices.get(choice)}_fold_{fold}.h5')\n\n        del model, train_dataset, val_dataset\n        gc.collect()\n    \n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}