{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q --upgrade pip\n!pip install -q efficientnet","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","id":"amateur-efficiency","outputId":"57473785-8a9e-4131-8737-6086fd117228","papermill":{"duration":16.097954,"end_time":"2021-04-18T10:33:30.631445","exception":false,"start_time":"2021-04-18T10:33:14.533491","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_probability as tfp\nimport matplotlib.pyplot as plt\nimport efficientnet.tfkeras as efn\nimport seaborn as sns\n\nfrom kaggle_datasets import KaggleDatasets\nfrom keras.applications import ResNet50\n\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nfrom tqdm.notebook import tqdm\nfrom sklearn.metrics import classification_report, confusion_matrix\n\nimport sys\nimport glob\nimport math\nimport gc\nimport time\n\nprint(f'tensorflow version: {tf.__version__}')\nprint(f'tensorflow keras version: {tf.keras.__version__}')\nprint(f'python version: P{sys.version}')","metadata":{"_kg_hide-input":true,"id":"selective-recording","outputId":"cc49cedd-4573-45c4-9b86-f4f0f1eb7d41","papermill":{"duration":8.784784,"end_time":"2021-04-18T10:33:39.439466","exception":false,"start_time":"2021-04-18T10:33:30.654682","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')\n\n# # set half precision policy\n# mixed_precision.set_policy('mixed_bfloat16')\nmixed_precision.set_policy('mixed_float16')\n# mixed_precision.set_policy('float32')\n# enable XLA optmizations\ntf.config.optimizer.set_jit(True)\n\nprint(f'Compute dtype: {mixed_precision.global_policy().compute_dtype}')\nprint(f'Variable dtype: {mixed_precision.global_policy().variable_dtype}')","metadata":{"_kg_hide-input":true,"id":"intellectual-bracket","papermill":{"duration":5.765197,"end_time":"2021-04-18T10:33:45.228494","exception":false,"start_time":"2021-04-18T10:33:39.463297","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_HEIGHT = 600\nIMG_WIDTH = 800\n\nIMG_SIZE = 600\nIMG_TARGET_SIZE = 512\nN_CHANNELS = 3\n\nN_TRAIN_IMGS = 21642\nN_VAL_IMGS = 5410\nBATCH_SIZE_VAL = 128 * REPLICAS # 5410 / 8 / 4\n\nN_LABELS = 5\nN_FOLDS = 1\nEPOCHS = 30\n\nBATCH_SIZE_BASE = 16\nBATCH_SIZE = BATCH_SIZE_BASE * REPLICAS\n\nTARGET_DTYPE = tf.bfloat16\n\n# ImageNet mean and standard deviation\nIMAGENET_MEAN = tf.constant([0.485, 0.456, 0.406], dtype=tf.float32)\nIMAGENET_STD = tf.constant([0.229, 0.224, 0.225], dtype=tf.float32)","metadata":{"_kg_hide-input":true,"id":"usual-logan","papermill":{"duration":0.036449,"end_time":"2021-04-18T10:33:45.291578","exception":false,"start_time":"2021-04-18T10:33:45.255129","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path('cassava-leaf-disease-tfrecords-600x600')","metadata":{"id":"modified-device","papermill":{"duration":0.437999,"end_time":"2021-04-18T10:33:45.756417","exception":false,"start_time":"2021-04-18T10:33:45.318418","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_tfrecord_train(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'label': tf.io.FixedLenFeature([], tf.int64),\n        'height': tf.io.FixedLenFeature([], tf.int64),\n        'width': tf.io.FixedLenFeature([], tf.int64),\n    })\n    \n    height = features['height']\n    width = features['width']\n\n    image = tf.io.decode_jpeg(features['image'])\n    image = tf.reshape(image, [height, width, N_CHANNELS])\n    \n    # get random square\n    if height > width:\n        offset = tf.random.uniform(shape=(), minval=0, maxval=height-width, dtype=tf.int64)\n        image = tf.slice(image, [offset, 0, 0], [width, width, N_CHANNELS])\n    elif width > height:\n        offset = tf.random.uniform(shape=(), minval=0, maxval=width-height, dtype=tf.int64)\n        image = tf.slice(image, [0, offset, 0], [height, height, N_CHANNELS])\n    else:\n        image = tf.slice(image, [0, 0, 0], [height, width, N_CHANNELS])\n        \n    size = tf.cast(height if height < width else width, tf.float32)\n    \n    # cast label to int8\n    label = tf.cast(features['label'], tf.uint8)\n\n    return image, label, size","metadata":{"_kg_hide-input":true,"id":"silver-hybrid","papermill":{"duration":0.040989,"end_time":"2021-04-18T10:33:45.823208","exception":false,"start_time":"2021-04-18T10:33:45.782219","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# chance of x in y to return true, used for conditional data augmentation\ndef chance(x, y):\n    return tf.random.uniform(shape=[], minval=0, maxval=y, dtype=tf.int32) < x","metadata":{"_kg_hide-input":true,"id":"exotic-implementation","papermill":{"duration":0.033162,"end_time":"2021-04-18T10:33:45.882027","exception":false,"start_time":"2021-04-18T10:33:45.848865","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augment_image(image, label, size):\n    # random flip image horizontally\n    image = tf.image.random_flip_left_right(image)\n    # random flip image vertically\n    image = tf.image.random_flip_up_down(image)\n    \n    # random transpose\n    if chance(1,2):\n        image = tf.image.transpose(image)\n    \n    # random crop between 75%-100%\n    crop_size = tf.random.uniform(shape=(), minval=size*0.75, maxval=size)\n    image = tf.image.random_crop(image, [crop_size, crop_size, N_CHANNELS])\n    \n    # cast to target dtype and resize\n    image = tf.image.resize(image, [IMG_TARGET_SIZE, IMG_TARGET_SIZE])\n    \n    # normalize according to imagenet mean and std\n    image /= 255.0\n    image = (image - IMAGENET_MEAN) / IMAGENET_STD\n    \n    # one hot encode label\n    label = tf.one_hot(label, N_LABELS, dtype=tf.float32)\n    \n    return image, label\n\ndef read_augment_image(record_bytes):\n    image, label, size = decode_tfrecord_train(record_bytes)\n    image, label = augment_image(image, label, size)\n    \n    return image, label\n\ndef get_mix_img_idx(labels_idxs, idx):\n    idx_candidates = tf.where(labels_idxs != idx)\n    r = tf.random.uniform(minval=0, maxval=len(idx_candidates), shape=[], dtype=tf.int32)\n    idx = tf.gather(idx_candidates, r)\n    idx = tf.cast(idx, tf.int32)\n    idx = tf.squeeze(idx)\n    \n    return idx","metadata":{"_kg_hide-input":true,"id":"casual-classification","papermill":{"duration":0.038555,"end_time":"2021-04-18T10:33:45.945901","exception":false,"start_time":"2021-04-18T10:33:45.907346","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mixup Implementation","metadata":{"id":"announced-saudi","papermill":{"duration":0.024205,"end_time":"2021-04-18T10:33:45.994703","exception":false,"start_time":"2021-04-18T10:33:45.970498","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def mixup(images, labels, alpha=0.40):\n    l = len(images)\n    # get image factors\n    a = tfp.distributions.Beta(alpha, alpha).sample(l)\n    a_label = tf.reshape(a, shape=(l,1))\n    a_label = tf.tile(a_label, [1, N_LABELS])\n    b_label = 1 - a_label\n    \n    a_image = tf.reshape(a, shape=(l,1,1,1))\n    a_image = tf.tile(a_image, [1, IMG_TARGET_SIZE, IMG_TARGET_SIZE ,N_CHANNELS])\n    a_image = tf.cast(a_image, tf.float32)\n    b_image = 1 - a_image\n    \n    # get mixup image indices\n    if l == 2:\n        idxs = tf.constant([1, 0])\n    else:\n        labels_idxs = tf.range(len(labels))\n        idxs = tf.map_fn(lambda idx: get_mix_img_idx(labels_idxs, idx), tf.range(len(labels)))\n    \n    images_mixup = tf.gather(images, idxs)\n    labels_mixup = tf.gather(labels, idxs)\n    \n    # mixup images and labels\n    images =  images * a_image + images_mixup * b_image\n    labels = labels * a_label + labels_mixup * b_label\n    \n    images = tf.cast(images, TARGET_DTYPE)\n    \n    return images, labels","metadata":{"_kg_hide-input":true,"id":"declared-naples","papermill":{"duration":0.039173,"end_time":"2021-04-18T10:33:46.061036","exception":false,"start_time":"2021-04-18T10:33:46.021863","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cutmix","metadata":{"id":"upset-selling","papermill":{"duration":0.02619,"end_time":"2021-04-18T10:33:46.114534","exception":false,"start_time":"2021-04-18T10:33:46.088344","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def create_cutmix_mask(a):\n    # create random mask size and coordinates\n    r_w = tf.cast(IMG_TARGET_SIZE * tf.math.sqrt(1 - a), tf.int32)\n    r_h = tf.cast(IMG_TARGET_SIZE * tf.math.sqrt(1 - a), tf.int32)\n    \n    if r_w == IMG_TARGET_SIZE:\n        r_x = 0\n    else:\n        r_x = tf.random.uniform(minval=0, maxval=IMG_TARGET_SIZE - r_w, shape=[], dtype=tf.int32)\n        \n    if r_h == IMG_TARGET_SIZE:\n        r_y = 0\n    else:\n        r_y = tf.random.uniform(minval=0, maxval=IMG_TARGET_SIZE - r_w, shape=[], dtype=tf.int32)\n\n    # compute padding sizes\n    pad_left = r_x\n    pad_right = IMG_TARGET_SIZE - (r_x + r_w)\n    pad_top = r_y\n    pad_bottom = IMG_TARGET_SIZE - (r_y + r_h)\n    \n    # create mask_a and mask_b\n    mask_a = tf.ones(shape=[r_w, r_h], dtype=tf.float32)\n    mask_a = tf.pad(mask_a, [[pad_left, pad_right], [pad_top, pad_bottom]], mode='CONSTANT', constant_values=0)\n    mask_a = tf.expand_dims(mask_a, axis=2)\n    \n    return mask_a\n\ndef cutmix(images, labels):\n    l = len(images)\n    a_float32 = tfp.distributions.Beta(1.0, 1.0).sample([l])\n\n    mask_b = tf.map_fn(create_cutmix_mask, a_float32)\n    mask_a = tf.math.abs(mask_b - 1)\n    \n    # images_idxs\n    if l == 2:\n        idxs = tf.constant([1, 0])\n    else:\n        labels_idxs = tf.range(len(labels))\n        idxs = tf.map_fn(lambda idx: get_mix_img_idx(labels_idxs, idx), tf.range(len(labels)))\n    \n    images_cutmix = tf.gather(images, idxs)\n    labels_cutmix = tf.gather(labels, idxs)\n    \n    a_float32_labels = tf.expand_dims(a_float32, axis=1)\n    a_float32_labels = tf.repeat(a_float32_labels, N_LABELS, axis=1)\n    labels_factor = a_float32_labels\n    labels_cutmix_factor = 1 - a_float32_labels\n    \n    # cutmix images and labels\n    images = images * mask_a + images_cutmix * mask_b\n    labels = labels * labels_factor + labels_cutmix * labels_cutmix_factor\n    \n    images = tf.cast(images, TARGET_DTYPE)\n    \n    return images, labels","metadata":{"_kg_hide-input":true,"id":"multiple-grounds","papermill":{"duration":0.043693,"end_time":"2021-04-18T10:33:46.184677","exception":false,"start_time":"2021-04-18T10:33:46.140984","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Gridmask","metadata":{"id":"extensive-palmer","papermill":{"duration":0.024397,"end_time":"2021-04-18T10:33:46.233900","exception":false,"start_time":"2021-04-18T10:33:46.209503","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def gridmask(images, labels):\n    l = len(images)\n    \n    d = tf.random.uniform(minval=int(IMG_TARGET_SIZE * (96/224)), maxval=IMG_TARGET_SIZE, shape=[], dtype=tf.int32)\n    grid = tf.constant([[[0], [1]],[[1], [0]]], dtype=tf.float32)\n    grid = tf.image.resize(grid, [d, d], method='nearest')\n    \n    # 50% chance to rotate mask\n    if chance(1, 2):\n        grid = tf.image.rot90(grid, 1)\n\n    repeats = IMG_TARGET_SIZE // d + 1\n    grid = tf.tile(grid, multiples=[repeats, repeats, 1])\n    grid = tf.image.random_crop(grid, [IMG_TARGET_SIZE, IMG_TARGET_SIZE, 1])\n    grid = tf.expand_dims(grid, axis=0)\n    grid = tf.tile(grid, multiples=[l, 1, 1, 1])\n\n    images = images * grid\n    images = tf.cast(images, TARGET_DTYPE)\n    \n    return images, labels","metadata":{"_kg_hide-input":true,"id":"living-christmas","papermill":{"duration":0.039463,"end_time":"2021-04-18T10:33:46.299300","exception":false,"start_time":"2021-04-18T10:33:46.259837","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augment_batch(images, labels, augmentations=None):\n    if augmentations is None:\n        r = tf.random.uniform(minval=0, maxval=4, shape=[], dtype=tf.int32)\n    else:\n        r = tf.random.uniform(minval=0, maxval=len(augmentations), shape=[], dtype=tf.int32)\n        r = tf.gather(augmentations, r)\n        \n    if r == 0:\n        images = tf.cast(images, TARGET_DTYPE)\n        return images, labels\n    elif r == 1:\n        return mixup(images, labels)\n    elif r == 2:\n        return cutmix(images, labels)\n    elif r == 3:\n        return gridmask(images, labels)\n    else:\n        images = tf.cast(images, TARGET_DTYPE)\n        return images, labels","metadata":{"_kg_hide-input":true,"id":"caring-barcelona","papermill":{"duration":0.036784,"end_time":"2021-04-18T10:33:46.361170","exception":false,"start_time":"2021-04-18T10:33:46.324386","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reshape_batch(images, labels):\n    images = tf.reshape(images, shape=[BATCH_SIZE, IMG_TARGET_SIZE, IMG_TARGET_SIZE, N_CHANNELS])\n    labels = tf.reshape(labels, shape=[BATCH_SIZE, N_LABELS])\n    \n    random_idxs = tf.random.shuffle(tf.range(BATCH_SIZE))\n    images = tf.gather(images, random_idxs)\n    labels = tf.gather(labels, random_idxs)\n    \n    return images, labels","metadata":{"_kg_hide-input":true,"id":"musical-render","papermill":{"duration":0.037652,"end_time":"2021-04-18T10:33:46.425541","exception":false,"start_time":"2021-04-18T10:33:46.387889","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_dataset(bs=BATCH_SIZE, fold=0, augmentations=None):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False\n    \n    FNAMES_TRAIN_TFRECORDS = tf.io.gfile.glob(f'{GCS_DS_PATH}/fold_{fold}/train/*.tfrecords')\n    train_dataset = tf.data.TFRecordDataset(FNAMES_TRAIN_TFRECORDS, num_parallel_reads=AUTO)\n    train_dataset = train_dataset.with_options(ignore_order)\n    train_dataset = train_dataset.prefetch(AUTO)\n    train_dataset = train_dataset.repeat()\n    train_dataset = train_dataset.map(read_augment_image, num_parallel_calls=AUTO)\n\n    train_dataset = train_dataset.batch(BATCH_SIZE_BASE)\n    train_dataset = train_dataset.map(lambda images, labels: augment_batch(images, labels, augmentations=augmentations), num_parallel_calls=REPLICAS)\n    \n    train_dataset = train_dataset.batch(REPLICAS)\n    train_dataset = train_dataset.map(reshape_batch, num_parallel_calls=1)\n    \n    train_dataset = train_dataset.prefetch(1)\n    \n    return train_dataset\n\ntrain_dataset = get_train_dataset()","metadata":{"_kg_hide-input":true,"id":"interstate-inclusion","papermill":{"duration":2.814048,"end_time":"2021-04-18T10:33:49.266732","exception":false,"start_time":"2021-04-18T10:33:46.452684","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def benchmark(num_epochs=3, n_steps_per_epoch=10, augmentations=None, bs=BATCH_SIZE):\n    dataset = get_train_dataset(augmentations=augmentations)\n    start_time = time.perf_counter()\n    for epoch_num in range(num_epochs):\n        epoch_start = time.perf_counter()\n        for idx, (images, labels) in enumerate(dataset.take(n_steps_per_epoch)):\n            if idx is 1:\n                print(images.shape, labels.shape)\n            pass\n        print(f'epoch {epoch_num} took: {round(time.perf_counter() - epoch_start, 2)}')\n    print(\"Execution time:\", round(time.perf_counter() - start_time, 2))\n    \nbenchmark(num_epochs=3, augmentations=[2,3])","metadata":{"_kg_hide-input":true,"id":"gross-serbia","papermill":{"duration":12.969484,"end_time":"2021-04-18T10:34:02.262871","exception":false,"start_time":"2021-04-18T10:33:49.293387","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Validation dataset","metadata":{"id":"union-technician","papermill":{"duration":0.027122,"end_time":"2021-04-18T10:34:02.317806","exception":false,"start_time":"2021-04-18T10:34:02.290684","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def resize_image(image, label, size):\n    image = tf.image.resize(image, [IMG_TARGET_SIZE, IMG_TARGET_SIZE])\n    \n    return image, label, tf.cast(IMG_TARGET_SIZE, tf.float32)","metadata":{"_kg_hide-input":true,"id":"second-cannon","papermill":{"duration":0.036998,"end_time":"2021-04-18T10:34:02.383356","exception":false,"start_time":"2021-04-18T10:34:02.346358","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_tfrecord_val(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'label': tf.io.FixedLenFeature([], tf.int64),\n        'height': tf.io.FixedLenFeature([], tf.int64),\n        'width': tf.io.FixedLenFeature([], tf.int64),\n    })\n    \n    height = features['height']\n    width = features['width']\n\n    image = tf.io.decode_jpeg(features['image'])\n    image = tf.reshape(image, [height, width, N_CHANNELS])\n    \n    # get random square\n    if height > width:\n        offset = (height - width) // 2\n        image = tf.slice(image, [offset, 0, 0], [width, width, N_CHANNELS])\n    elif width > height:\n        offset = (width - height) // 2\n        image = tf.slice(image, [0, offset, 0], [height, height, N_CHANNELS])\n    else:\n        image = tf.slice(image, [0, 0, 0], [height, width, N_CHANNELS])\n    \n    # resize to target size\n    image = tf.image.resize(image, [IMG_TARGET_SIZE, IMG_TARGET_SIZE])\n    \n    # normalize according to imagenet mean and std\n    image /= 255.0\n    image = (image - IMAGENET_MEAN) / IMAGENET_STD\n    \n    # cast to TARGET_DTYPE\n    image = tf.cast(image, TARGET_DTYPE)\n    \n    label = tf.cast(features['label'], tf.int32)\n    \n    # one hot encode label\n    label = tf.one_hot(label, N_LABELS, dtype=tf.int32)\n    \n    return image, label","metadata":{"_kg_hide-input":true,"id":"suspected-tourism","papermill":{"duration":0.044869,"end_time":"2021-04-18T10:34:02.455297","exception":false,"start_time":"2021-04-18T10:34:02.410428","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_val_dataset(bs=BATCH_SIZE, fold=0):\n    FNAMES_VAL_TFRECORDS = tf.io.gfile.glob(f'{GCS_DS_PATH}/fold_{fold}/val/*.tfrecords')\n    val_dataset = tf.data.TFRecordDataset(FNAMES_VAL_TFRECORDS, num_parallel_reads=AUTO)\n    val_dataset = val_dataset.prefetch(BATCH_SIZE_VAL)\n    val_dataset = val_dataset.repeat()\n    val_dataset = val_dataset.map(decode_tfrecord_val, num_parallel_calls=AUTO)\n    val_dataset = val_dataset.batch(bs, drop_remainder=True)\n    val_dataset = val_dataset.prefetch(1)\n    \n    return val_dataset\n\nval_dataset = get_val_dataset()","metadata":{"_kg_hide-input":true,"id":"right-placement","papermill":{"duration":0.427252,"end_time":"2021-04-18T10:34:02.911966","exception":false,"start_time":"2021-04-18T10:34:02.484714","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lr scheduler","metadata":{"id":"egyptian-agent","papermill":{"duration":0.027846,"end_time":"2021-04-18T10:34:02.967829","exception":false,"start_time":"2021-04-18T10:34:02.939983","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def lrfn(epoch, bs=BATCH_SIZE, epochs=EPOCHS):\n    # Config\n    LR_START = 1e-6\n    LR_MAX = 2e-4\n    LR_FINAL = 1e-6\n    LR_RAMPUP_EPOCHS = 4\n    LR_SUSTAIN_EPOCHS = 0\n    DECAY_EPOCHS = epochs  - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\n    LR_EXP_DECAY = (LR_FINAL / LR_MAX) ** (1 / (EPOCHS - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1))\n\n    if epoch < LR_RAMPUP_EPOCHS: # exponential warmup\n        lr = LR_START + (LR_MAX + LR_START) * (epoch / LR_RAMPUP_EPOCHS) ** 2.5\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS: # sustain lr\n        lr = LR_MAX\n    else: # cosine decay\n        epoch_diff = epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n        decay_factor = (epoch_diff / DECAY_EPOCHS) * math.pi\n        decay_factor= (tf.math.cos(decay_factor).numpy() + 1) / 2        \n        lr = LR_FINAL + (LR_MAX - LR_FINAL) * decay_factor\n\n    return lr\n\ndef lrfn2(epoch):\n    \n    LR_START = 0.00001\n    LR_MAX = 0.00005 * strategy.num_replicas_in_sync\n    LR_MIN = 0.00001\n    LR_RAMPUP_EPOCHS = 4\n    LR_SUSTAIN_EPOCHS = 4\n    LR_EXP_DECAY = .8\n\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \n","metadata":{"_kg_hide-input":true,"id":"organizational-hydrogen","papermill":{"duration":0.042732,"end_time":"2021-04-18T10:34:03.038120","exception":false,"start_time":"2021-04-18T10:34:02.995388","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using Binary and Categorical focal loss","metadata":{"id":"backed-johnston","papermill":{"duration":0.028363,"end_time":"2021-04-18T10:34:03.094726","exception":false,"start_time":"2021-04-18T10:34:03.066363","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Model","metadata":{"id":"limited-passport","papermill":{"duration":0.027404,"end_time":"2021-04-18T10:34:03.225488","exception":false,"start_time":"2021-04-18T10:34:03.198084","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_model(choice):\n    # reset to free memory and training variables\n    tf.keras.backend.clear_session()\n    \n    \n    net = net_choices.get(choice)\n    \n            \n    with strategy.scope():\n        if (choice==0):\n            net = efn.EfficientNetB4(\n                include_top=False,\n                weights='noisy-student',\n                input_shape=(IMG_TARGET_SIZE, IMG_TARGET_SIZE, N_CHANNELS),\n            )\n        elif (choice==1):\n            net = ResNet50(\n                weights='imagenet',\n                include_top=False,\n            )\n        elif (choice==2):\n            net=tf.keras.applications.DenseNet201(\n                weights='imagenet',\n                include_top=False\n            )\n\n        \n        for layer in reversed(net.layers):\n            if isinstance(layer, tf.keras.layers.BatchNormalization):\n                layer.trainable = False\n\n            else:\n                layer.trainable = True\n\n        \n        model = tf.keras.Sequential([\n            net,\n            tf.keras.layers.Dropout(0.45),\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dropout(0.45),\n            tf.keras.layers.Dense(N_LABELS, activation='softmax', dtype=tf.float32),\n        ])\n\n        # add metrics\n        metrics = [\n            tf.keras.metrics.CategoricalAccuracy(name='accuracy'),\n            tf.keras.metrics.TopKCategoricalAccuracy(k=2, name='top_2_accuracy'),\n        ]\n\n        optimizer = tf.keras.optimizers.Adam()\n        loss = tf.keras.losses.CategoricalCrossentropy()\n#         cat_loss = categorical_focal_loss(gamma=2., alpha=.25)\n        \n        model.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n#         model.summary()\n        return model","metadata":{"_kg_hide-input":true,"id":"supreme-class","papermill":{"duration":0.045881,"end_time":"2021-04-18T10:34:03.299224","exception":false,"start_time":"2021-04-18T10:34:03.253343","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation function","metadata":{"id":"academic-cedar","papermill":{"duration":0.028861,"end_time":"2021-04-18T10:34:03.356053","exception":false,"start_time":"2021-04-18T10:34:03.327192","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def show_validation_report_per_class(model, dataset, steps, name, bs):\n    print(f'--- {name} REPORT ---')\n    # classification report\n    y = np.ndarray(shape=steps * bs, dtype=np.uint16)\n    y_pred = np.ndarray(shape=steps * bs, dtype=np.uint16)\n    for idx, (images, labels) in tqdm(enumerate(dataset.take(steps)), total=steps):\n        with tf.device('cpu:0'):\n            y[idx*bs:(idx+1)*bs] = np.argmax(labels, axis=1)\n            y_pred[idx*bs:(idx+1)*bs] = np.argmax(model.predict(images).astype(np.float32), axis=1)\n            \n    print(classification_report(y, y_pred))\n    \n    # Confusion matrix\n    fig, ax = plt.subplots(1, 1, figsize=(20, 12))\n    cfn_matrix = confusion_matrix(y, y_pred, labels=range(N_LABELS))\n    cfn_matrix = (cfn_matrix.T / cfn_matrix.sum(axis=1)).T\n    df_cm = pd.DataFrame(cfn_matrix, index=np.arange(N_LABELS), columns=np.arange(N_LABELS))\n    ax = sns.heatmap(df_cm, cmap='Blues', annot=True, fmt='.3f', linewidths=.7, annot_kws={'size':14}).set_title(f'{name} CONFUSION MATRIX')\n    plt.xticks(fontsize=20)\n    plt.yticks(fontsize=20)\n    plt.xlabel('PREDICTED', fontsize=26, labelpad=10)\n    plt.ylabel('ACTUAL', fontsize=26, labelpad=10)\n    plt.show()","metadata":{"_kg_hide-input":true,"id":"addressed-cameroon","papermill":{"duration":0.043531,"end_time":"2021-04-18T10:34:03.427526","exception":false,"start_time":"2021-04-18T10:34:03.383995","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting curves function","metadata":{"id":"latin-insulation","papermill":{"duration":0.027952,"end_time":"2021-04-18T10:34:03.483933","exception":false,"start_time":"2021-04-18T10:34:03.455981","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def plot_history_metric(history_array, metric):\n    TRAIN_EPOCHS = len(history_array[0].history['loss'])\n    x = np.arange(TRAIN_EPOCHS)\n    x_axis_labels = list(map(str, np.arange(1, TRAIN_EPOCHS+1)))\n    val = 'val' in ''.join(history_array[0].history.keys())\n    # summarize history for accuracy\n    plt.figure(figsize=(20, 10))\n    \n    \n    handles_array = []\n    val_handles_array = []\n    labels_array = []\n    \n    for count, history in enumerate(history_array):\n#         handle, = plt.plot(history.history[metric])\n#         handles_array.append(handle)\n#         label = f\"{net_choices.get(count)}-{metric}\"\n#         labels_array.append(label)\n\n        if val:\n            val_handle, = plt.plot(history.history[f'val_{metric}'])\n            val_handles_array.append(val_handle)\n            label = f\"{net_choices.get(count)}-{metric}\"\n            labels_array.append(label)\n            \n    \n    plt.title(f'Model {metric}', fontsize=30)\n    plt.ylabel(metric, fontsize=26)\n    plt.yticks(fontsize=20)\n    plt.xlabel('epoch', fontsize=26)\n    plt.xticks(x, x_axis_labels, fontsize=20) # set tick step to 1 and let x axis start at 1\n    plt.legend(handles=handles_array+val_handles_array, labels = labels_array, loc='upper left')\n    plt.grid()\n    plt.show()","metadata":{"id":"electrical-convertible","papermill":{"duration":0.041691,"end_time":"2021-04-18T10:34:03.553651","exception":false,"start_time":"2021-04-18T10:34:03.511960","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Running model","metadata":{"id":"harmful-trauma","papermill":{"duration":0.029109,"end_time":"2021-04-18T10:34:03.613971","exception":false,"start_time":"2021-04-18T10:34:03.584862","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(f'TRAINING FOR {EPOCHS} EPOCHS WITH BATCH SIZE {BATCH_SIZE}\\n')\nprint(f'TRAIN IMAGES: {N_TRAIN_IMGS}, VAL IMAGES: {N_VAL_IMGS}\\n')\n\naugmentations_dic = dict({\n    0: 'None',\n    1: 'MixUp',\n    2: 'CutMix',\n    3: 'GridMask',\n})\n\nnet_choices = dict({\n    0: \"Efficientnet\",\n    1: \"ResNet\",\n    2: \"DenseNet\"\n})\n    \n\n\naugmentations = [2, 3] # only CutMix and GridMask is used\nmodel_choices = [0]   # choice as given in above dictionary\nhistory_array = []\n# MEAN_VAL_ACC = []\n# fold = 0\n# epochs = EPOCHS\n\nfor choice in model_choices:\n    MEAN_VAL_ACC = []\n    fold = 0\n    epochs = EPOCHS\n    for idx, fold in enumerate(range(N_FOLDS)):\n        # callbacks\n        lr_callback_1 = tf.keras.callbacks.LearningRateScheduler(lambda epoch: lrfn(epoch, epochs=epochs), verbose=1)\n    #     lr_callback_2 = tf.keras.callbacks.LearningRateScheduler(lrfn2, verbose = True)\n    #     show_lr_schedule(epochs=epochs)\n\n        # get the model\n        model = get_model(choice)\n\n        if idx is 0:\n            # model summary\n            model.summary()\n            # compute and variable data types\n            print(f'Compute dtype: {mixed_precision.global_policy().compute_dtype}')\n            print(f'Variable dtype: {mixed_precision.global_policy().variable_dtype}')\n\n        print('\\n')\n        print('*'*25, f'augmentations {augmentations}', '*'*25, '\\n')\n        print(f'fold: {fold}, epochs: {epochs}')\n        print(' AND '.join([augmentations_dic.get(i) for i in augmentations]), '\\n')\n\n        train_dataset = get_train_dataset(bs=BATCH_SIZE, fold=fold, augmentations=augmentations)\n        val_dataset = get_val_dataset(bs=BATCH_SIZE_VAL, fold=fold)\n        \n        with strategy.scope():\n            history = model.fit(\n                train_dataset,\n                steps_per_epoch = N_TRAIN_IMGS // BATCH_SIZE,\n\n                validation_data = val_dataset,\n                validation_steps = N_VAL_IMGS // BATCH_SIZE_VAL,\n\n                epochs = epochs,\n                callbacks = [lr_callback_1],\n                verbose=1\n            )\n\n        # add val accuracy to list\n        MEAN_VAL_ACC.append(history.history['val_accuracy'][-1])\n\n        # # plot training histories\n        # plot_history_metric(history, 'loss')\n        # plot_history_metric(history, 'accuracy')\n        # plot_history_metric(history, 'top_2_accuracy')\n\n        # # show train and validation report\n        # show_validation_report_per_class(model, val_dataset, N_VAL_IMGS // BATCH_SIZE_VAL, 'VALIDATION', BATCH_SIZE_VAL)\n\n        # save the model\n        model.save_weights(f'model_fold_{fold}_weights.h5')\n        model.save(f'model_{net_choices.get(choice)}_fold_{fold}.h5')\n        # show train and validation report\n        \n        show_validation_report_per_class(model, val_dataset, N_VAL_IMGS // BATCH_SIZE_VAL, 'VALIDATION', BATCH_SIZE_VAL)\n        \n        del model, train_dataset, val_dataset\n        gc.collect()\n    # plot training histories\n    history_array.append(history)\n    \nplot_history_metric(history_array, 'loss')\nplot_history_metric(history_array, 'accuracy')\nplot_history_metric(history_array, 'top_2_accuracy')\n\n    \n","metadata":{"id":"specified-frame","papermill":{"duration":2171.807993,"end_time":"2021-04-18T11:10:15.457186","exception":false,"start_time":"2021-04-18T10:34:03.649193","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}