{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q --upgrade pip\n!pip install -q efficientnet","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2021-04-14T12:07:53.337571Z","iopub.status.busy":"2021-04-14T12:07:53.336299Z","iopub.status.idle":"2021-04-14T12:08:09.0122Z","shell.execute_reply":"2021-04-14T12:08:09.011341Z"},"papermill":{"duration":15.704613,"end_time":"2021-04-14T12:08:09.012389","exception":false,"start_time":"2021-04-14T12:07:53.307776","status":"completed"},"tags":[],"id":"amateur-efficiency","outputId":"57473785-8a9e-4131-8737-6086fd117228"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_probability as tfp\nimport matplotlib.pyplot as plt\nimport efficientnet.tfkeras as efn\nimport seaborn as sns\n\nfrom kaggle_datasets import KaggleDatasets\nfrom keras.applications import ResNet50\n\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nfrom tqdm.notebook import tqdm\nfrom sklearn.metrics import classification_report, confusion_matrix\n\nimport sys\nimport glob\nimport math\nimport gc\nimport time\n\nprint(f'tensorflow version: {tf.__version__}')\nprint(f'tensorflow keras version: {tf.keras.__version__}')\nprint(f'python version: P{sys.version}')","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:09.067464Z","iopub.status.busy":"2021-04-14T12:08:09.066752Z","iopub.status.idle":"2021-04-14T12:08:17.528162Z","shell.execute_reply":"2021-04-14T12:08:17.527525Z"},"papermill":{"duration":8.49209,"end_time":"2021-04-14T12:08:17.528312","exception":false,"start_time":"2021-04-14T12:08:09.036222","status":"completed"},"tags":[],"id":"selective-recording","outputId":"cc49cedd-4573-45c4-9b86-f4f0f1eb7d41"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')\n\n# # set half precision policy\nmixed_precision.set_policy('mixed_bfloat16')\n\n# enable XLA optmizations\ntf.config.optimizer.set_jit(True)\n\nprint(f'Compute dtype: {mixed_precision.global_policy().compute_dtype}')\nprint(f'Variable dtype: {mixed_precision.global_policy().variable_dtype}')","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:17.587088Z","iopub.status.busy":"2021-04-14T12:08:17.581025Z","iopub.status.idle":"2021-04-14T12:08:23.209153Z","shell.execute_reply":"2021-04-14T12:08:23.208241Z"},"papermill":{"duration":5.65709,"end_time":"2021-04-14T12:08:23.209324","exception":false,"start_time":"2021-04-14T12:08:17.552234","status":"completed"},"tags":[],"id":"intellectual-bracket"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_HEIGHT = 600\nIMG_WIDTH = 800\n\nIMG_SIZE = 600\nIMG_TARGET_SIZE = 512\nN_CHANNELS = 3\n\nN_TRAIN_IMGS = 21642\nN_VAL_IMGS = 5410\nBATCH_SIZE_VAL = 128 * REPLICAS # 5410 / 8 / 4\n\nN_LABELS = 5\nN_FOLDS = 1\nEPOCHS = 30\n\nBATCH_SIZE_BASE = 16\nBATCH_SIZE = BATCH_SIZE_BASE * REPLICAS\n\nTARGET_DTYPE = tf.bfloat16\n\n# ImageNet mean and standard deviation\nIMAGENET_MEAN = tf.constant([0.485, 0.456, 0.406], dtype=tf.float32)\nIMAGENET_STD = tf.constant([0.229, 0.224, 0.225], dtype=tf.float32)","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:23.269262Z","iopub.status.busy":"2021-04-14T12:08:23.26836Z","iopub.status.idle":"2021-04-14T12:08:23.271074Z","shell.execute_reply":"2021-04-14T12:08:23.270513Z"},"papermill":{"duration":0.035573,"end_time":"2021-04-14T12:08:23.271216","exception":false,"start_time":"2021-04-14T12:08:23.235643","status":"completed"},"tags":[],"id":"usual-logan"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path('cassava-leaf-disease-tfrecords-600x600')","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:23.336093Z","iopub.status.busy":"2021-04-14T12:08:23.335338Z","iopub.status.idle":"2021-04-14T12:08:23.649441Z","shell.execute_reply":"2021-04-14T12:08:23.648298Z"},"papermill":{"duration":0.353753,"end_time":"2021-04-14T12:08:23.649622","exception":false,"start_time":"2021-04-14T12:08:23.295869","status":"completed"},"tags":[],"id":"modified-device"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_tfrecord_train(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'label': tf.io.FixedLenFeature([], tf.int64),\n        'height': tf.io.FixedLenFeature([], tf.int64),\n        'width': tf.io.FixedLenFeature([], tf.int64),\n    })\n    \n    height = features['height']\n    width = features['width']\n\n    image = tf.io.decode_jpeg(features['image'])\n    image = tf.reshape(image, [height, width, N_CHANNELS])\n    \n    # get random square\n    if height > width:\n        offset = tf.random.uniform(shape=(), minval=0, maxval=height-width, dtype=tf.int64)\n        image = tf.slice(image, [offset, 0, 0], [width, width, N_CHANNELS])\n    elif width > height:\n        offset = tf.random.uniform(shape=(), minval=0, maxval=width-height, dtype=tf.int64)\n        image = tf.slice(image, [0, offset, 0], [height, height, N_CHANNELS])\n    else:\n        image = tf.slice(image, [0, 0, 0], [height, width, N_CHANNELS])\n        \n    size = tf.cast(height if height < width else width, tf.float32)\n    \n    # cast label to int8\n    label = tf.cast(features['label'], tf.uint8)\n\n    return image, label, size","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:23.713266Z","iopub.status.busy":"2021-04-14T12:08:23.712244Z","iopub.status.idle":"2021-04-14T12:08:23.715892Z","shell.execute_reply":"2021-04-14T12:08:23.715393Z"},"papermill":{"duration":0.040357,"end_time":"2021-04-14T12:08:23.716067","exception":false,"start_time":"2021-04-14T12:08:23.67571","status":"completed"},"tags":[],"id":"silver-hybrid"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# chance of x in y to return true, used for conditional data augmentation\ndef chance(x, y):\n    return tf.random.uniform(shape=[], minval=0, maxval=y, dtype=tf.int32) < x","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:23.771125Z","iopub.status.busy":"2021-04-14T12:08:23.770433Z","iopub.status.idle":"2021-04-14T12:08:23.773646Z","shell.execute_reply":"2021-04-14T12:08:23.773104Z"},"papermill":{"duration":0.032833,"end_time":"2021-04-14T12:08:23.773791","exception":false,"start_time":"2021-04-14T12:08:23.740958","status":"completed"},"tags":[],"id":"exotic-implementation"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augment_image(image, label, size):\n    # random flip image horizontally\n    image = tf.image.random_flip_left_right(image)\n    # random flip image vertically\n    image = tf.image.random_flip_up_down(image)\n    \n    # random transpose\n    if chance(1,2):\n        image = tf.image.transpose(image)\n    \n    # random crop between 75%-100%\n    crop_size = tf.random.uniform(shape=(), minval=size*0.75, maxval=size)\n    image = tf.image.random_crop(image, [crop_size, crop_size, N_CHANNELS])\n    \n    # cast to target dtype and resize\n    image = tf.image.resize(image, [IMG_TARGET_SIZE, IMG_TARGET_SIZE])\n    \n    # normalize according to imagenet mean and std\n    image /= 255.0\n    image = (image - IMAGENET_MEAN) / IMAGENET_STD\n    \n    # one hot encode label\n    label = tf.one_hot(label, N_LABELS, dtype=tf.float32)\n    \n    return image, label\n\ndef read_augment_image(record_bytes):\n    image, label, size = decode_tfrecord_train(record_bytes)\n    image, label = augment_image(image, label, size)\n    \n    return image, label\n\ndef get_mix_img_idx(labels_idxs, idx):\n    idx_candidates = tf.where(labels_idxs != idx)\n    r = tf.random.uniform(minval=0, maxval=len(idx_candidates), shape=[], dtype=tf.int32)\n    idx = tf.gather(idx_candidates, r)\n    idx = tf.cast(idx, tf.int32)\n    idx = tf.squeeze(idx)\n    \n    return idx","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:23.83503Z","iopub.status.busy":"2021-04-14T12:08:23.834262Z","iopub.status.idle":"2021-04-14T12:08:23.83657Z","shell.execute_reply":"2021-04-14T12:08:23.837056Z"},"papermill":{"duration":0.038747,"end_time":"2021-04-14T12:08:23.837237","exception":false,"start_time":"2021-04-14T12:08:23.79849","status":"completed"},"tags":[],"id":"casual-classification"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mixup Implementation","metadata":{"papermill":{"duration":0.024851,"end_time":"2021-04-14T12:08:23.887108","exception":false,"start_time":"2021-04-14T12:08:23.862257","status":"completed"},"tags":[],"id":"announced-saudi"}},{"cell_type":"code","source":"def mixup(images, labels, alpha=0.40):\n    l = len(images)\n    # get image factors\n    a = tfp.distributions.Beta(alpha, alpha).sample(l)\n    a_label = tf.reshape(a, shape=(l,1))\n    a_label = tf.tile(a_label, [1, N_LABELS])\n    b_label = 1 - a_label\n    \n    a_image = tf.reshape(a, shape=(l,1,1,1))\n    a_image = tf.tile(a_image, [1, IMG_TARGET_SIZE, IMG_TARGET_SIZE ,N_CHANNELS])\n    a_image = tf.cast(a_image, tf.float32)\n    b_image = 1 - a_image\n    \n    # get mixup image indices\n    if l == 2:\n        idxs = tf.constant([1, 0])\n    else:\n        labels_idxs = tf.range(len(labels))\n        idxs = tf.map_fn(lambda idx: get_mix_img_idx(labels_idxs, idx), tf.range(len(labels)))\n    \n    images_mixup = tf.gather(images, idxs)\n    labels_mixup = tf.gather(labels, idxs)\n    \n    # mixup images and labels\n    images =  images * a_image + images_mixup * b_image\n    labels = labels * a_label + labels_mixup * b_label\n    \n    images = tf.cast(images, TARGET_DTYPE)\n    \n    return images, labels","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:23.940826Z","iopub.status.busy":"2021-04-14T12:08:23.940204Z","iopub.status.idle":"2021-04-14T12:08:23.949849Z","shell.execute_reply":"2021-04-14T12:08:23.950454Z"},"papermill":{"duration":0.038697,"end_time":"2021-04-14T12:08:23.950639","exception":false,"start_time":"2021-04-14T12:08:23.911942","status":"completed"},"tags":[],"id":"declared-naples"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cutmix","metadata":{"papermill":{"duration":0.026388,"end_time":"2021-04-14T12:08:24.002143","exception":false,"start_time":"2021-04-14T12:08:23.975755","status":"completed"},"tags":[],"id":"upset-selling"}},{"cell_type":"code","source":"def create_cutmix_mask(a):\n    # create random mask size and coordinates\n    r_w = tf.cast(IMG_TARGET_SIZE * tf.math.sqrt(1 - a), tf.int32)\n    r_h = tf.cast(IMG_TARGET_SIZE * tf.math.sqrt(1 - a), tf.int32)\n    \n    if r_w == IMG_TARGET_SIZE:\n        r_x = 0\n    else:\n        r_x = tf.random.uniform(minval=0, maxval=IMG_TARGET_SIZE - r_w, shape=[], dtype=tf.int32)\n        \n    if r_h == IMG_TARGET_SIZE:\n        r_y = 0\n    else:\n        r_y = tf.random.uniform(minval=0, maxval=IMG_TARGET_SIZE - r_w, shape=[], dtype=tf.int32)\n\n    # compute padding sizes\n    pad_left = r_x\n    pad_right = IMG_TARGET_SIZE - (r_x + r_w)\n    pad_top = r_y\n    pad_bottom = IMG_TARGET_SIZE - (r_y + r_h)\n    \n    # create mask_a and mask_b\n    mask_a = tf.ones(shape=[r_w, r_h], dtype=tf.float32)\n    mask_a = tf.pad(mask_a, [[pad_left, pad_right], [pad_top, pad_bottom]], mode='CONSTANT', constant_values=0)\n    mask_a = tf.expand_dims(mask_a, axis=2)\n    \n    return mask_a\n\ndef cutmix(images, labels):\n    l = len(images)\n    a_float32 = tfp.distributions.Beta(1.0, 1.0).sample([l])\n\n    mask_b = tf.map_fn(create_cutmix_mask, a_float32)\n    mask_a = tf.math.abs(mask_b - 1)\n    \n    # images_idxs\n    if l == 2:\n        idxs = tf.constant([1, 0])\n    else:\n        labels_idxs = tf.range(len(labels))\n        idxs = tf.map_fn(lambda idx: get_mix_img_idx(labels_idxs, idx), tf.range(len(labels)))\n    \n    images_cutmix = tf.gather(images, idxs)\n    labels_cutmix = tf.gather(labels, idxs)\n    \n    a_float32_labels = tf.expand_dims(a_float32, axis=1)\n    a_float32_labels = tf.repeat(a_float32_labels, N_LABELS, axis=1)\n    labels_factor = a_float32_labels\n    labels_cutmix_factor = 1 - a_float32_labels\n    \n    # cutmix images and labels\n    images = images * mask_a + images_cutmix * mask_b\n    labels = labels * labels_factor + labels_cutmix * labels_cutmix_factor\n    \n    images = tf.cast(images, TARGET_DTYPE)\n    \n    return images, labels","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:24.05724Z","iopub.status.busy":"2021-04-14T12:08:24.056452Z","iopub.status.idle":"2021-04-14T12:08:24.070768Z","shell.execute_reply":"2021-04-14T12:08:24.071446Z"},"papermill":{"duration":0.04404,"end_time":"2021-04-14T12:08:24.071641","exception":false,"start_time":"2021-04-14T12:08:24.027601","status":"completed"},"tags":[],"id":"multiple-grounds"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Gridmask","metadata":{"papermill":{"duration":0.025397,"end_time":"2021-04-14T12:08:24.122477","exception":false,"start_time":"2021-04-14T12:08:24.09708","status":"completed"},"tags":[],"id":"extensive-palmer"}},{"cell_type":"code","source":"def gridmask(images, labels):\n    l = len(images)\n    \n    d = tf.random.uniform(minval=int(IMG_TARGET_SIZE * (96/224)), maxval=IMG_TARGET_SIZE, shape=[], dtype=tf.int32)\n    grid = tf.constant([[[0], [1]],[[1], [0]]], dtype=tf.float32)\n    grid = tf.image.resize(grid, [d, d], method='nearest')\n    \n    # 50% chance to rotate mask\n    if chance(1, 2):\n        grid = tf.image.rot90(grid, 1)\n\n    repeats = IMG_TARGET_SIZE // d + 1\n    grid = tf.tile(grid, multiples=[repeats, repeats, 1])\n    grid = tf.image.random_crop(grid, [IMG_TARGET_SIZE, IMG_TARGET_SIZE, 1])\n    grid = tf.expand_dims(grid, axis=0)\n    grid = tf.tile(grid, multiples=[l, 1, 1, 1])\n\n    images = images * grid\n    images = tf.cast(images, TARGET_DTYPE)\n    \n    return images, labels","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:24.17717Z","iopub.status.busy":"2021-04-14T12:08:24.176445Z","iopub.status.idle":"2021-04-14T12:08:24.185499Z","shell.execute_reply":"2021-04-14T12:08:24.186161Z"},"papermill":{"duration":0.037944,"end_time":"2021-04-14T12:08:24.18635","exception":false,"start_time":"2021-04-14T12:08:24.148406","status":"completed"},"tags":[],"id":"living-christmas"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augment_batch(images, labels, augmentations=None):\n    if augmentations is None:\n        r = tf.random.uniform(minval=0, maxval=4, shape=[], dtype=tf.int32)\n    else:\n        r = tf.random.uniform(minval=0, maxval=len(augmentations), shape=[], dtype=tf.int32)\n        r = tf.gather(augmentations, r)\n        \n    if r == 0:\n        images = tf.cast(images, TARGET_DTYPE)\n        return images, labels\n    elif r == 1:\n        return mixup(images, labels)\n    elif r == 2:\n        return cutmix(images, labels)\n    elif r == 3:\n        return gridmask(images, labels)\n    else:\n        images = tf.cast(images, TARGET_DTYPE)\n        return images, labels","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:24.240224Z","iopub.status.busy":"2021-04-14T12:08:24.239537Z","iopub.status.idle":"2021-04-14T12:08:24.247204Z","shell.execute_reply":"2021-04-14T12:08:24.247723Z"},"papermill":{"duration":0.036221,"end_time":"2021-04-14T12:08:24.247906","exception":false,"start_time":"2021-04-14T12:08:24.211685","status":"completed"},"tags":[],"id":"caring-barcelona"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reshape_batch(images, labels):\n    images = tf.reshape(images, shape=[BATCH_SIZE, IMG_TARGET_SIZE, IMG_TARGET_SIZE, N_CHANNELS])\n    labels = tf.reshape(labels, shape=[BATCH_SIZE, N_LABELS])\n    \n    random_idxs = tf.random.shuffle(tf.range(BATCH_SIZE))\n    images = tf.gather(images, random_idxs)\n    labels = tf.gather(labels, random_idxs)\n    \n    return images, labels","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:24.301812Z","iopub.status.busy":"2021-04-14T12:08:24.301182Z","iopub.status.idle":"2021-04-14T12:08:24.306634Z","shell.execute_reply":"2021-04-14T12:08:24.3072Z"},"papermill":{"duration":0.034234,"end_time":"2021-04-14T12:08:24.30737","exception":false,"start_time":"2021-04-14T12:08:24.273136","status":"completed"},"tags":[],"id":"musical-render"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_dataset(bs=BATCH_SIZE, fold=0, augmentations=None):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False\n    \n    FNAMES_TRAIN_TFRECORDS = tf.io.gfile.glob(f'{GCS_DS_PATH}/fold_{fold}/train/*.tfrecords')\n    train_dataset = tf.data.TFRecordDataset(FNAMES_TRAIN_TFRECORDS, num_parallel_reads=AUTO)\n    train_dataset = train_dataset.with_options(ignore_order)\n    train_dataset = train_dataset.prefetch(AUTO)\n    train_dataset = train_dataset.repeat()\n    train_dataset = train_dataset.map(read_augment_image, num_parallel_calls=AUTO)\n\n    train_dataset = train_dataset.batch(BATCH_SIZE_BASE)\n    train_dataset = train_dataset.map(lambda images, labels: augment_batch(images, labels, augmentations=augmentations), num_parallel_calls=REPLICAS)\n    \n    train_dataset = train_dataset.batch(REPLICAS)\n    train_dataset = train_dataset.map(reshape_batch, num_parallel_calls=1)\n    \n    train_dataset = train_dataset.prefetch(1)\n    \n    return train_dataset\n\ntrain_dataset = get_train_dataset()","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:24.362Z","iopub.status.busy":"2021-04-14T12:08:24.361318Z","iopub.status.idle":"2021-04-14T12:08:27.070715Z","shell.execute_reply":"2021-04-14T12:08:27.070109Z"},"papermill":{"duration":2.737994,"end_time":"2021-04-14T12:08:27.070857","exception":false,"start_time":"2021-04-14T12:08:24.332863","status":"completed"},"tags":[],"id":"interstate-inclusion"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def benchmark(num_epochs=3, n_steps_per_epoch=10, augmentations=None, bs=BATCH_SIZE):\n    dataset = get_train_dataset(augmentations=augmentations)\n    start_time = time.perf_counter()\n    for epoch_num in range(num_epochs):\n        epoch_start = time.perf_counter()\n        for idx, (images, labels) in enumerate(dataset.take(n_steps_per_epoch)):\n            if idx is 1:\n                print(images.shape, labels.shape)\n            pass\n        print(f'epoch {epoch_num} took: {round(time.perf_counter() - epoch_start, 2)}')\n    print(\"Execution time:\", round(time.perf_counter() - start_time, 2))\n    \nbenchmark(num_epochs=3, augmentations=[2,3])","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:27.127631Z","iopub.status.busy":"2021-04-14T12:08:27.126949Z","iopub.status.idle":"2021-04-14T12:08:39.461374Z","shell.execute_reply":"2021-04-14T12:08:39.460698Z"},"papermill":{"duration":12.364861,"end_time":"2021-04-14T12:08:39.461527","exception":false,"start_time":"2021-04-14T12:08:27.096666","status":"completed"},"tags":[],"id":"gross-serbia"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Validation dataset","metadata":{"papermill":{"duration":0.02722,"end_time":"2021-04-14T12:08:39.51734","exception":false,"start_time":"2021-04-14T12:08:39.49012","status":"completed"},"tags":[],"id":"union-technician"}},{"cell_type":"code","source":"def resize_image(image, label, size):\n    image = tf.image.resize(image, [IMG_TARGET_SIZE, IMG_TARGET_SIZE])\n    \n    return image, label, tf.cast(IMG_TARGET_SIZE, tf.float32)","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:39.5787Z","iopub.status.busy":"2021-04-14T12:08:39.577818Z","iopub.status.idle":"2021-04-14T12:08:39.580515Z","shell.execute_reply":"2021-04-14T12:08:39.579864Z"},"papermill":{"duration":0.035739,"end_time":"2021-04-14T12:08:39.58065","exception":false,"start_time":"2021-04-14T12:08:39.544911","status":"completed"},"tags":[],"id":"second-cannon"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_tfrecord_val(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'label': tf.io.FixedLenFeature([], tf.int64),\n        'height': tf.io.FixedLenFeature([], tf.int64),\n        'width': tf.io.FixedLenFeature([], tf.int64),\n    })\n    \n    height = features['height']\n    width = features['width']\n\n    image = tf.io.decode_jpeg(features['image'])\n    image = tf.reshape(image, [height, width, N_CHANNELS])\n    \n    # get random square\n    if height > width:\n        offset = (height - width) // 2\n        image = tf.slice(image, [offset, 0, 0], [width, width, N_CHANNELS])\n    elif width > height:\n        offset = (width - height) // 2\n        image = tf.slice(image, [0, offset, 0], [height, height, N_CHANNELS])\n    else:\n        image = tf.slice(image, [0, 0, 0], [height, width, N_CHANNELS])\n    \n    # resize to target size\n    image = tf.image.resize(image, [IMG_TARGET_SIZE, IMG_TARGET_SIZE])\n    \n    # normalize according to imagenet mean and std\n    image /= 255.0\n    image = (image - IMAGENET_MEAN) / IMAGENET_STD\n    \n    # cast to TARGET_DTYPE\n    image = tf.cast(image, TARGET_DTYPE)\n    \n    label = tf.cast(features['label'], tf.int32)\n    \n    # one hot encode label\n    label = tf.one_hot(label, N_LABELS, dtype=tf.int32)\n    \n    return image, label","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:39.650059Z","iopub.status.busy":"2021-04-14T12:08:39.649307Z","iopub.status.idle":"2021-04-14T12:08:39.653221Z","shell.execute_reply":"2021-04-14T12:08:39.65248Z"},"papermill":{"duration":0.044936,"end_time":"2021-04-14T12:08:39.653393","exception":false,"start_time":"2021-04-14T12:08:39.608457","status":"completed"},"tags":[],"id":"suspected-tourism"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_val_dataset(bs=BATCH_SIZE, fold=0):\n    FNAMES_VAL_TFRECORDS = tf.io.gfile.glob(f'{GCS_DS_PATH}/fold_{fold}/val/*.tfrecords')\n    val_dataset = tf.data.TFRecordDataset(FNAMES_VAL_TFRECORDS, num_parallel_reads=AUTO)\n    val_dataset = val_dataset.prefetch(BATCH_SIZE_VAL)\n    val_dataset = val_dataset.repeat()\n    val_dataset = val_dataset.map(decode_tfrecord_val, num_parallel_calls=AUTO)\n    val_dataset = val_dataset.batch(bs, drop_remainder=True)\n    val_dataset = val_dataset.prefetch(1)\n    \n    return val_dataset\n\nval_dataset = get_val_dataset()","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:39.718521Z","iopub.status.busy":"2021-04-14T12:08:39.717763Z","iopub.status.idle":"2021-04-14T12:08:40.109969Z","shell.execute_reply":"2021-04-14T12:08:40.110608Z"},"papermill":{"duration":0.427858,"end_time":"2021-04-14T12:08:40.110799","exception":false,"start_time":"2021-04-14T12:08:39.682941","status":"completed"},"tags":[],"id":"right-placement"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lr scheduler","metadata":{"papermill":{"duration":0.028096,"end_time":"2021-04-14T12:08:40.167711","exception":false,"start_time":"2021-04-14T12:08:40.139615","status":"completed"},"tags":[],"id":"egyptian-agent"}},{"cell_type":"code","source":"def lrfn(epoch, bs=BATCH_SIZE, epochs=EPOCHS):\n    # Config\n    LR_START = 1e-6\n    LR_MAX = 2e-4\n    LR_FINAL = 1e-6\n    LR_RAMPUP_EPOCHS = 4\n    LR_SUSTAIN_EPOCHS = 0\n    DECAY_EPOCHS = epochs  - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\n    LR_EXP_DECAY = (LR_FINAL / LR_MAX) ** (1 / (EPOCHS - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1))\n\n    if epoch < LR_RAMPUP_EPOCHS: # exponential warmup\n        lr = LR_START + (LR_MAX + LR_START) * (epoch / LR_RAMPUP_EPOCHS) ** 2.5\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS: # sustain lr\n        lr = LR_MAX\n    else: # cosine decay\n        epoch_diff = epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n        decay_factor = (epoch_diff / DECAY_EPOCHS) * math.pi\n        decay_factor= (tf.math.cos(decay_factor).numpy() + 1) / 2        \n        lr = LR_FINAL + (LR_MAX - LR_FINAL) * decay_factor\n\n    return lr\n\ndef lrfn2(epoch):\n    \n    LR_START = 0.00001\n    LR_MAX = 0.00005 * strategy.num_replicas_in_sync\n    LR_MIN = 0.00001\n    LR_RAMPUP_EPOCHS = 4\n    LR_SUSTAIN_EPOCHS = 4\n    LR_EXP_DECAY = .8\n\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \n","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:40.235829Z","iopub.status.busy":"2021-04-14T12:08:40.235117Z","iopub.status.idle":"2021-04-14T12:08:40.238196Z","shell.execute_reply":"2021-04-14T12:08:40.237656Z"},"papermill":{"duration":0.041759,"end_time":"2021-04-14T12:08:40.238347","exception":false,"start_time":"2021-04-14T12:08:40.196588","status":"completed"},"tags":[],"id":"organizational-hydrogen"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using Binary and Categorical focal loss","metadata":{"papermill":{"duration":0.028794,"end_time":"2021-04-14T12:08:40.295491","exception":false,"start_time":"2021-04-14T12:08:40.266697","status":"completed"},"tags":[],"id":"backed-johnston"}},{"cell_type":"code","source":"from tensorflow.keras import backend as K\nimport dill\n\n\ndef binary_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n\n        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\n    return binary_focal_loss_fixed\n\n\ndef categorical_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Softmax version of focal loss.\n           m\n      FL = ∑  -alpha * (1 - p_o,c)^gamma * y_o,c * log(p_o,c)\n          c=1\n      where m = number of classes, c = class and o = observation\n    Parameters:\n      alpha -- the same as weighing factor in balanced cross entropy\n      gamma -- focusing parameter for modulating factor (1-p)\n    Default value:\n      gamma -- 2.0 as mentioned in the paper\n      alpha -- 0.25 as mentioned in the paper\n    References:\n        Official paper: https://arxiv.org/pdf/1708.02002.pdf\n        https://www.tensorflow.org/api_docs/python/tf/keras/backend/categorical_crossentropy\n    Usage:\n     model.compile(loss=[categorical_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def categorical_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred: A tensor resulting from a softmax\n        :return: Output tensor.\n        \"\"\"\n\n        # Scale predictions so that the class probas of each sample sum to 1\n        y_pred /= K.sum(y_pred, axis=-1, keepdims=True)\n\n        # Clip the prediction value to prevent NaN's and Inf's\n        epsilon = K.epsilon()\n        y_pred = K.clip(y_pred, epsilon, 1. - epsilon)\n        \n        tf.cast(y_pred, tf.float32)\n        tf.cast(y_true, tf.float32)\n        # Calculate Cross Entropy\n        cross_entropy = -y_true * K.log(y_pred)\n        print(f'type of y_pred ---- {type(y_pred)}')\n        print(f'type of y_true ---- {type(y_true)}')\n\n        # Calculate Focal Loss\n        loss = alpha * K.pow(1 - y_pred, gamma) * cross_entropy\n\n        print(f'type of k_sum ---- {type(K.sum(loss, axis=1))}')\n        tf.cast(loss, tf.float32)\n        # Sum the losses in mini_batch\n        return_list = K.sum(loss, axis=1)\n        \n        tf.cast(return_list, tf.float32)\n#         return K.sum(loss, axis=1)\n        return return_list\n\n    return categorical_focal_loss_fixed","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:40.367938Z","iopub.status.busy":"2021-04-14T12:08:40.361519Z","iopub.status.idle":"2021-04-14T12:08:40.370428Z","shell.execute_reply":"2021-04-14T12:08:40.370997Z"},"papermill":{"duration":0.046993,"end_time":"2021-04-14T12:08:40.371175","exception":false,"start_time":"2021-04-14T12:08:40.324182","status":"completed"},"tags":[],"id":"demographic-teens"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.027933,"end_time":"2021-04-14T12:08:40.427575","exception":false,"start_time":"2021-04-14T12:08:40.399642","status":"completed"},"tags":[],"id":"limited-passport"}},{"cell_type":"code","source":"def get_model(choice):\n    # reset to free memory and training variables\n    tf.keras.backend.clear_session()\n    \n    \n    net = net_choices.get(choice)\n    with strategy.scope():\n        \n        \n        if (choice==0):\n            net = efn.EfficientNetB4(\n                include_top=False,\n                weights='noisy-student',\n                input_shape=(IMG_TARGET_SIZE, IMG_TARGET_SIZE, N_CHANNELS),\n            )\n        elif (choice==1):\n            net = ResNet50(\n                weights='imagenet',\n                include_top=False,\n            )\n        elif (choice==2):\n            net=tf.keras.applications.DenseNet201(\n                weights='imagenet',\n                include_top=False\n            )\n        \n        for layer in reversed(net.layers):\n            if isinstance(layer, tf.keras.layers.BatchNormalization):\n                layer.trainable = False\n                \n            else:\n                layer.trainable = True\n        \n        model = tf.keras.Sequential([\n            net,\n            tf.keras.layers.Dropout(0.45),\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dropout(0.45),\n            tf.keras.layers.Dense(N_LABELS, activation='softmax', dtype=tf.float32),\n        ])\n\n        # add metrics\n        metrics = [\n            tf.keras.metrics.CategoricalAccuracy(name='accuracy'),\n            tf.keras.metrics.TopKCategoricalAccuracy(k=2, name='top_2_accuracy'),\n        ]\n\n        optimizer = tf.keras.optimizers.Adam()\n        loss = tf.keras.losses.CategoricalCrossentropy()\n        cat_loss = categorical_focal_loss(gamma=2., alpha=.25)\n        \n        model.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n#         model.summary()\n        return model","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:40.487728Z","iopub.status.busy":"2021-04-14T12:08:40.487047Z","iopub.status.idle":"2021-04-14T12:08:40.497935Z","shell.execute_reply":"2021-04-14T12:08:40.49853Z"},"papermill":{"duration":0.042973,"end_time":"2021-04-14T12:08:40.498735","exception":false,"start_time":"2021-04-14T12:08:40.455762","status":"completed"},"tags":[],"id":"supreme-class"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation function","metadata":{"papermill":{"duration":0.02791,"end_time":"2021-04-14T12:08:40.554931","exception":false,"start_time":"2021-04-14T12:08:40.527021","status":"completed"},"tags":[],"id":"academic-cedar"}},{"cell_type":"code","source":"def show_validation_report_per_class(model, dataset, steps, name, bs):\n    print(f'--- {name} REPORT ---')\n    # classification report\n    y = np.ndarray(shape=steps * bs, dtype=np.uint16)\n    y_pred = np.ndarray(shape=steps * bs, dtype=np.uint16)\n    for idx, (images, labels) in tqdm(enumerate(dataset.take(steps)), total=steps):\n        with tf.device('cpu:0'):\n            y[idx*bs:(idx+1)*bs] = np.argmax(labels, axis=1)\n            y_pred[idx*bs:(idx+1)*bs] = np.argmax(model.predict(images).astype(np.float32), axis=1)\n            \n    print(classification_report(y, y_pred))\n    \n    # Confusion matrix\n    fig, ax = plt.subplots(1, 1, figsize=(20, 12))\n    cfn_matrix = confusion_matrix(y, y_pred, labels=range(N_LABELS))\n    cfn_matrix = (cfn_matrix.T / cfn_matrix.sum(axis=1)).T\n    df_cm = pd.DataFrame(cfn_matrix, index=np.arange(N_LABELS), columns=np.arange(N_LABELS))\n    ax = sns.heatmap(df_cm, cmap='Blues', annot=True, fmt='.3f', linewidths=.7, annot_kws={'size':14}).set_title(f'{name} CONFUSION MATRIX')\n    plt.xticks(fontsize=20)\n    plt.yticks(fontsize=20)\n    plt.xlabel('PREDICTED', fontsize=26, labelpad=10)\n    plt.ylabel('ACTUAL', fontsize=26, labelpad=10)\n    plt.show()","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:40.615151Z","iopub.status.busy":"2021-04-14T12:08:40.614482Z","iopub.status.idle":"2021-04-14T12:08:40.626538Z","shell.execute_reply":"2021-04-14T12:08:40.627072Z"},"papermill":{"duration":0.044176,"end_time":"2021-04-14T12:08:40.627265","exception":false,"start_time":"2021-04-14T12:08:40.583089","status":"completed"},"tags":[],"id":"addressed-cameroon"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting curves function","metadata":{"papermill":{"duration":0.02902,"end_time":"2021-04-14T12:08:40.684639","exception":false,"start_time":"2021-04-14T12:08:40.655619","status":"completed"},"tags":[],"id":"latin-insulation"}},{"cell_type":"code","source":"def plot_history_metric(history_array, metric, model_name):\n    TRAIN_EPOCHS = len(history_array[0].history['loss'])\n    x = np.arange(TRAIN_EPOCHS)\n    x_axis_labels = list(map(str, np.arange(1, TRAIN_EPOCHS+1)))\n    val = 'val' in ''.join(history_array[0].history.keys())\n    # summarize history for accuracy\n    plt.figure(figsize=(20, 10))\n    \n    for history in history_array:\n      plt.plot(history.history[metric])\n      if val:\n          plt.plot(history.history[f'val_{metric}'])\n    \n    plt.title(f'Model {metric}', fontsize=30)\n    plt.ylabel(metric, fontsize=26)\n    plt.yticks(fontsize=20)\n    plt.xlabel('epoch', fontsize=26)\n    plt.xticks(x, x_axis_labels, fontsize=20) # set tick step to 1 and let x axis start at 1\n    plt.legend(['train'] + ['test'] if val else [], loc='upper left')\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:40.745801Z","iopub.status.busy":"2021-04-14T12:08:40.745143Z","iopub.status.idle":"2021-04-14T12:08:40.75446Z","shell.execute_reply":"2021-04-14T12:08:40.753833Z"},"papermill":{"duration":0.041127,"end_time":"2021-04-14T12:08:40.754602","exception":false,"start_time":"2021-04-14T12:08:40.713475","status":"completed"},"tags":[],"id":"electrical-convertible"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Running model","metadata":{"papermill":{"duration":0.028934,"end_time":"2021-04-14T12:08:40.812287","exception":false,"start_time":"2021-04-14T12:08:40.783353","status":"completed"},"tags":[],"id":"harmful-trauma"}},{"cell_type":"code","source":"print(f'TRAINING FOR {EPOCHS} EPOCHS WITH BATCH SIZE {BATCH_SIZE}\\n')\nprint(f'TRAIN IMAGES: {N_TRAIN_IMGS}, VAL IMAGES: {N_VAL_IMGS}\\n')\n\naugmentations_dic = dict({\n    0: 'None',\n    1: 'MixUp',\n    2: 'CutMix',\n    3: 'GridMask',\n})\n\nnet_choices = dict({\n    0: \"Efficientnet\",\n    1: \"ResNet\",\n    2: \"DenseNet\"\n})\n    \n\n\naugmentations = [2, 3] # only CutMix and GridMask is used\nmodel_choices = [0, 1]   # choice can be 0 or 1 according to the dictionary given above\nhistory_array = []\n# MEAN_VAL_ACC = []\n# fold = 0\n# epochs = EPOCHS\n\nfor choice in model_choices:\n    MEAN_VAL_ACC = []\n    fold = 0\n    epochs = EPOCHS\n    for idx, fold in enumerate(range(N_FOLDS)):\n        # callbacks\n        lr_callback_1 = tf.keras.callbacks.LearningRateScheduler(lambda epoch: lrfn(epoch, epochs=epochs), verbose=1)\n    #     lr_callback_2 = tf.keras.callbacks.LearningRateScheduler(lrfn2, verbose = True)\n    #     show_lr_schedule(epochs=epochs)\n\n        # get the model\n        model = get_model(choice)\n\n        if idx is 0:\n            # model summary\n            model.summary()\n            # compute and variable data types\n            print(f'Compute dtype: {mixed_precision.global_policy().compute_dtype}')\n            print(f'Variable dtype: {mixed_precision.global_policy().variable_dtype}')\n\n        print('\\n')\n        print('*'*25, f'augmentations {augmentations}', '*'*25, '\\n')\n        print(f'fold: {fold}, epochs: {epochs}')\n        print(' AND '.join([augmentations_dic.get(i) for i in augmentations]), '\\n')\n\n        train_dataset = get_train_dataset(bs=BATCH_SIZE, fold=fold, augmentations=augmentations)\n        val_dataset = get_val_dataset(bs=BATCH_SIZE_VAL, fold=fold)\n\n        history = model.fit(\n            train_dataset,\n            steps_per_epoch = N_TRAIN_IMGS // BATCH_SIZE,\n\n            validation_data = val_dataset,\n            validation_steps = N_VAL_IMGS // BATCH_SIZE_VAL,\n\n            epochs = epochs,\n            callbacks = [lr_callback_1],\n            verbose=1\n        )\n\n        # add val accuracy to list\n        MEAN_VAL_ACC.append(history.history['val_accuracy'][-1])\n\n        # # plot training histories\n        # plot_history_metric(history, 'loss')\n        # plot_history_metric(history, 'accuracy')\n        # plot_history_metric(history, 'top_2_accuracy')\n\n        # # show train and validation report\n        # show_validation_report_per_class(model, val_dataset, N_VAL_IMGS // BATCH_SIZE_VAL, 'VALIDATION', BATCH_SIZE_VAL)\n\n        # save the model\n        model.save_weights(f'model_fold_{fold}_weights.h5')\n        model.save(f'model_{net_choices.get(choice)}_fold_{fold}.h5')\n        # show train and validation report\n        \n        show_validation_report_per_class(model, val_dataset, N_VAL_IMGS // BATCH_SIZE_VAL, 'VALIDATION', BATCH_SIZE_VAL)\n        \n        del model, train_dataset, val_dataset\n        gc.collect()\n    # plot training histories\n    history_array.append(history)\n    plot_history_metric(history_array, 'loss', net_choices.get(choice))\n    plot_history_metric(history_array, 'accuracy', net_choices.get(choice))\n    plot_history_metric(history_array, 'top_2_accuracy', net_choices.get(choice))\n\n    \n","metadata":{"execution":{"iopub.execute_input":"2021-04-14T12:08:40.873368Z","iopub.status.busy":"2021-04-14T12:08:40.8726Z","iopub.status.idle":"2021-04-14T15:25:14.186571Z","shell.execute_reply":"2021-04-14T15:25:14.187479Z"},"papermill":{"duration":11793.346937,"end_time":"2021-04-14T15:25:14.188005","exception":false,"start_time":"2021-04-14T12:08:40.841068","status":"completed"},"tags":[],"id":"specified-frame"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":16.047833,"end_time":"2021-04-14T15:25:45.955065","exception":false,"start_time":"2021-04-14T15:25:29.907232","status":"completed"},"tags":[],"id":"collaborative-roller"},"execution_count":null,"outputs":[]}]}