{"cells":[{"metadata":{},"cell_type":"markdown","source":"These materials below helped me a lot in making this notebook:\n- DimitreOliveira's [inference notebook](https://www.kaggle.com/dimitreoliveira/cassava-leaf-disease-tpu-v2-pods-inference).\n- Modified Bi-tempered loss [implementation](https://github.com/Diulhio/bitemperedloss-tf). (**Note:** if you use bi-tempered loss during training, remember to add ```tempered_softmax``` function after predictions are made, skip this step if you just use normal cross entropy loss and your models contain softmax layer on top)."},{"metadata":{},"cell_type":"markdown","source":"## Basic setup"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --quiet /kaggle/input/kerasapplications\n!pip install --quiet /kaggle/input/efficientnet-git","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nimport cv2\nimport math\nimport functools\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"ROOT_DIR = '/kaggle/input/'\nFILE_PATH = '../input/cassava-leaf-disease-classification/test_images/'\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nHEIGHT, WIDTH = 512, 512\nBATCH_SIZE = 16\nTTA_STEPS = 5\nBT_LOSS = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# def data_augment(image, label):\n#     p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n#     p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n#     p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n#     p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n#     p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n            \n#     # Flips\n#     image = tf.image.random_flip_left_right(image)\n#     image = tf.image.random_flip_up_down(image)\n#     if p_spatial > .75:\n#         image = tf.image.transpose(image)\n        \n#     # Rotates\n#     if p_rotate > .75:\n#         image = tf.image.rot90(image, k=3) # rotate 270º\n#     elif p_rotate > .5:\n#         image = tf.image.rot90(image, k=2) # rotate 180º\n#     elif p_rotate > .25:\n#         image = tf.image.rot90(image, k=1) # rotate 90º\n        \n#     # Pixel-level transforms\n#     if p_pixel_1 >= .4:\n#         image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n#     if p_pixel_2 >= .4:\n#         image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n#     if p_pixel_3 >= .4:\n#         image = tf.image.random_brightness(image, max_delta=.1)\n            \n\n#     return image, label\n\ndef data_augment(image, label):\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n            \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > .75:\n        image = tf.image.transpose(image)\n        \n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n        \n    # Pixel-level transforms\n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n        \n    #Crops\n    if p_crop > .7:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.8)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.9)\n    elif p_crop > .4:\n        crop_size = tf.random.uniform([], int(HEIGHT*.8), HEIGHT, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, 3])\n    \n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n#     # Crops\n#     if p_crop > .6:\n#         if p_crop > .9:\n#             image = tf.image.central_crop(image, central_fraction=.5)\n#         elif p_crop > .8:\n#             image = tf.image.central_crop(image, central_fraction=.6)\n#         elif p_crop > .7:\n#             image = tf.image.central_crop(image, central_fraction=.7)\n#         else:\n#             image = tf.image.central_crop(image, central_fraction=.8)\n#     elif p_crop > .3:\n#         crop_size = tf.random.uniform([], int(HEIGHT*.6), HEIGHT, dtype=tf.int32)\n#         image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n            \n\n    return image, label\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(img):\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.cast(img, tf.float32) / 255.0\n    img = tf.image.resize(img, (HEIGHT, WIDTH))\n    return img\n\ndef process_path(file_path):\n    parts = tf.strings.split(file_path, os.path.sep)\n    filename = parts[-1]\n    img = tf.io.read_file(file_path)\n    img = decode_image(img)\n    return img, filename\n\ndef get_dataset(file_path, batch_size=32, shuffle=False, tta=False):\n    dataset = tf.data.Dataset.list_files(file_path + '*.jpg', shuffle=shuffle)\n    dataset = dataset.map(process_path, num_parallel_calls=AUTOTUNE)\n    if tta:\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.batch(batch_size)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _parse_function(example, feature_description):\n    parsed_example = tf.io.parse_single_example(example, feature_description)\n    image = tf.io.decode_jpeg(parsed_example['image'], channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, (HEIGHT, WIDTH))\n    if 'target' in feature_description:\n        target = parsed_example['target']\n        return image, target\n    return image, parsed_example['image_name']\n\n\ndef load_data(path, batch_size=32, train=True, tta=False):\n    filenames = tf.io.gfile.glob(path)\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=tf.data.experimental.AUTOTUNE)\n    if train:\n        feature_description = {\n            'image': tf.io.FixedLenFeature([], tf.string),\n            'image_name': tf.io.FixedLenFeature([], tf.string),\n            'target': tf.io.FixedLenFeature([], tf.int64)\n        }\n    else:\n        feature_description = {\n            'image': tf.io.FixedLenFeature([], tf.string),\n            'image_name': tf.io.FixedLenFeature([], tf.string)\n        }\n    parsed_dataset = dataset.map(lambda x: _parse_function(x, feature_description))\n    if tta:\n        parsed_dataset = parsed_dataset.map(data_augment, num_parallel_calls=AUTOTUNE)\n    return parsed_dataset.batch(batch_size).prefetch(AUTOTUNE)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_set = load_data(ROOT_DIR + 'cassava-leaf-disease-classification/test_tfrecords/ld_test*.tfrec', batch_size=BATCH_SIZE, train=False, tta=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from sklearn.model_selection import train_test_split\n# train_filenames = tf.io.gfile.glob(ROOT_DIR + 'cassava-leaf-disease-classification/train_tfrecords/' + 'ld_train*.tfrec')\n# train_filenames, val_filenames = train_test_split(train_filenames, test_size=0.3, random_state=37)\n# val_set = load_data([ROOT_DIR+'cassava-leaf-disease-classification/train_tfrecords/ld_train00-1338.tfrec'], batch_size=BATCH_SIZE, train=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Softmax function for Bi-tempered loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"def for_loop(num_iters, body, initial_args):\n    \"\"\"Runs a simple for-loop with given body and initial_args.\n    Args:\n      num_iters: Maximum number of iterations.\n      body: Body of the for-loop.\n      initial_args: Args to the body for the first iteration.\n    Returns:\n      Output of the final iteration.\n    \"\"\"\n    for i in range(num_iters):\n        if i == 0:\n            outputs = body(*initial_args)\n        else:\n            outputs = body(*outputs)\n    return outputs\n\n\ndef log_t(u, t):\n    \"\"\"Compute log_t for `u`.\"\"\"\n\n    def _internal_log_t(u, t):\n        return (u ** (1.0 - t) - 1.0) / (1.0 - t)\n\n    return tf.cond(\n        tf.math.equal(t, 1.0), lambda: tf.math.log(u),\n        functools.partial(_internal_log_t, u, t))\n\n\ndef exp_t(u, t):\n    \"\"\"Compute exp_t for `u`.\"\"\"\n\n    def _internal_exp_t(u, t):\n        return tf.nn.relu(1.0 + (1.0 - t) * u) ** (1.0 / (1.0 - t))\n\n    return tf.cond(\n        tf.math.equal(t, 1.0), lambda: tf.math.exp(u),\n        functools.partial(_internal_exp_t, u, t))\n\n\ndef compute_normalization_fixed_point(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example (t > 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n\n    mu = tf.math.reduce_max(activations, -1, keepdims=True)\n    normalized_activations_step_0 = activations - mu\n    shape_normalized_activations = tf.shape(normalized_activations_step_0)\n\n    def iter_body(i, normalized_activations):\n        logt_partition = tf.math.reduce_sum(\n            exp_t(normalized_activations, t), -1, keepdims=True)\n        normalized_activations_t = tf.reshape(\n            normalized_activations_step_0 * tf.math.pow(logt_partition, 1.0 - t),\n            shape_normalized_activations)\n        return [i + 1, normalized_activations_t]\n\n    _, normalized_activations_t = for_loop(num_iters, iter_body,\n                                           [0, normalized_activations_step_0])\n\n    logt_partition = tf.math.reduce_sum(\n        exp_t(normalized_activations_t, t), -1, keepdims=True)\n    return -log_t(1.0 / logt_partition, t) + mu\n\n\ndef compute_normalization_binary_search(activations, t, num_iters=10):\n    \"\"\"Returns the normalization value for each example (t < 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (< 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    mu = tf.math.reduce_max(activations, -1, keepdims=True)\n    normalized_activations = activations - mu\n    shape_activations = tf.shape(activations)\n    effective_dim = tf.cast(\n        tf.math.reduce_sum(\n            tf.cast(\n                tf.greater(normalized_activations, -1.0 / (1.0 - t)), tf.int32),\n            -1,\n            keepdims=True), tf.float32)\n    shape_partition = tf.concat([shape_activations[:-1], [1]], 0)\n    lower = tf.zeros(shape_partition)\n    upper = -log_t(1.0 / effective_dim, t) * tf.ones(shape_partition)\n\n    def iter_body(i, lower, upper):\n        logt_partition = (upper + lower) / 2.0\n        sum_probs = tf.math.reduce_sum(exp_t(\n            normalized_activations - logt_partition, t), -1, keepdims=True)\n        update = tf.cast(tf.less(sum_probs, 1.0), tf.float32)\n        lower = tf.reshape(lower * update + (1.0 - update) * logt_partition,\n                           shape_partition)\n        upper = tf.reshape(upper * (1.0 - update) + update * logt_partition,\n                           shape_partition)\n        return [i + 1, lower, upper]\n\n    _, lower, upper = for_loop(num_iters, iter_body, [0, lower, upper])\n    logt_partition = (upper + lower) / 2.0\n    return logt_partition + mu\n\n\ndef compute_normalization(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (< 1.0 for finite support, > 1.0 for tail heaviness).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    return tf.cond(\n        tf.less(t, 1.0),\n        functools.partial(compute_normalization_binary_search, activations, t,\n                          num_iters),\n        functools.partial(compute_normalization_fixed_point, activations, t,\n                          num_iters))\n\n\ndef tempered_softmax(activations, t, num_iters=5):\n    \"\"\"Tempered softmax function.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature tensor > 0.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    t = tf.convert_to_tensor(t)\n    normalization_constants = tf.cond(\n        tf.math.equal(t, 1.0),\n        lambda: tf.math.log(tf.math.reduce_sum(tf.exp(activations), -1, keepdims=True)),\n        functools.partial(compute_normalization, activations, t, num_iters))\n    return exp_t(activations - normalization_constants, t)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"files_path = ROOT_DIR + '/cassava-leaf-disease-classification/test_images/'\ntest_size = len(os.listdir(files_path))\ntest_preds = np.zeros((test_size, 5))\nfolds_to_choose = [1, 2, 3, 4, 5]\nN_FOLDS = len(folds_to_choose)\ntempered_params = {\n    'EfnB3': [0.7, 1.5],\n    'EfnB4': [0.7, 1.5],\n    'EfnB4_bigT': [0.5, 2.0],\n    'EfnB3_bigT': [0.5, 2.0]\n}\n# model_paths = {'EfnB3': 'cassava-training-efn-b3', 'EfnB4': 'cassava-training', 'EfnB4_bigT': 'cassava-training-big-tempered'}\n# model_weights = {'EfnB3': 6, 'EfnB4': 4}\n# model_paths = {'EfnB4': 'cassava-training'}\n# model_weights = {'EfnB4': 1}\n# model_paths = {'EfnB3': 'cassava-training-efn-b3'}\n# model_weights = {'EfnB3': 1}\nmodel_paths = {'EfnB3_bigT': 'cassava-training-big-tempered-efnb3', 'EfnB4_bigT': 'cassava-training-big-tempered'}\nmodel_weights = {'EfnB3_bigT': 1, 'EfnB4_bigT': 1}\n\nfor fold in folds_to_choose:\n    print('FOLD {}'.format(fold))\n    for model_name in model_paths:\n        print('Model {} is making predictions...'.format(model_name))\n        tf.keras.backend.clear_session()\n        model = tf.keras.models.load_model(ROOT_DIR + model_paths[model_name] + '/model_fold_{}.h5'.format(fold))\n        if TTA_STEPS:\n            test_set = load_data(ROOT_DIR + 'cassava-leaf-disease-classification/test_tfrecords/ld_test*.tfrec', batch_size=BATCH_SIZE, train=False, tta=True).repeat()\n#             test_set = get_dataset(ROOT_DIR + 'cassava-leaf-disease-classification/test_images/', batch_size=BATCH_SIZE, shuffle=False, tta=True).repeat()\n            ct_steps = TTA_STEPS * math.ceil(test_size / BATCH_SIZE)\n            preds = model.predict(test_set, steps=ct_steps, verbose=1)[:(test_size * TTA_STEPS)]\n            if BT_LOSS:\n                preds = tempered_softmax(preds, tempered_params[model_name][1]).numpy()\n            preds = np.mean(preds.reshape(test_size, TTA_STEPS, 5, order='F'), axis=1)\n        else:\n            test_set = load_data(ROOT_DIR + 'cassava-leaf-disease-classification/test_tfrecords/ld_test*.tfrec', batch_size=BATCH_SIZE, train=False, tta=False)\n    #         test_set = get_dataset(ROOT_DIR + 'cassava-leaf-disease-classification/test_images/', batch_size=BATCH_SIZE, shuffle=False, tta=False)\n            preds = model.predict(test_set)\n            if BT_LOSS:\n                preds = tempered_softmax(preds, tempered_params[model_name][1]).numpy()\n        test_preds += model_weights[model_name] * preds\n    print()\n    \n\ntest_preds /= (N_FOLDS * sum(model_weights.values())) # normalize s.t. sum of every prediction = 1\ntest_preds = np.argmax(test_preds, axis=-1)\ntest_ds = load_data(ROOT_DIR + 'cassava-leaf-disease-classification/test_tfrecords/ld_test*.tfrec', batch_size=BATCH_SIZE, train=False, tta=False)\n# test_ds = get_dataset(ROOT_DIR + 'cassava-leaf-disease-classification/test_images/', batch_size=BATCH_SIZE, shuffle=False, tta=False)\nimage_names = [img_name.numpy().decode('utf-8') for img, img_name in iter(test_ds.unbatch())]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.DataFrame({'image_id': image_names, 'label': test_preds})\nsubmission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}