{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction\n\nI am beginner in kaggle and this is my first notebook so I would appreciate it very much if you could point out something wrong and rude if exists.\n\n* In this notebook I share source code of bi-tempered logistic loss and taylor cross entropy loss for keras/TF\n* I also demonstrate how these loss functions act\n* I modified previous source code of bi-tempered loss function; I separate it into tempered softmax activation and tempered loss function\n\n# Reference & Acknowledgements\n\n* Notebook\n *  https://www.kaggle.com/capiru/cassavanet-starter-easy-gpu-tpu-cv-0-9\n* Discussion\n * https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/209773\n * https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/209782\n* Paper\n * https://www.ijcai.org/Proceedings/2020/0305.pdf\n * https://arxiv.org/pdf/1906.03361.pdf\n* Github\n * https://github.com/google/bi-tempered-loss\n * https://github.com/Diulhio/bitemperedloss-tf"},{"metadata":{},"cell_type":"markdown","source":"# Source code"},{"metadata":{},"cell_type":"markdown","source":"## Import libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras.activations import softmax\nfrom tensorflow.keras.losses import CategoricalCrossentropy","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tempered Softmax Activation\n\n* Bi-tempered logistic loss is one of solution to manage noisy labels.\n\n* Previous code are integrated with tempered softmax function and tempered loss function but I want separate into two function.\n\n* So I separate Bi-tempered logistic loss function into two classes; TemperedSoftmax and BiTemperedLogisticLoss"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Tempered Softmax Activation\n\ndef log_t(u, t):\n    epsilon = 1e-7\n    \"\"\"Compute log_t for `u`.\"\"\"\n    if t == 1.0:\n        return tf.math.log(u + epsilon)\n    else:\n        return (u**(1.0 - t) - 1.0) / (1.0 - t)\n\ndef exp_t(u, t):\n    \"\"\"Compute exp_t for `u`.\"\"\"\n    if t == 1.0:\n        return tf.math.exp(u)\n    else:\n        return tf.math.maximum(0.0, 1.0 + (1.0 - t) * u) ** (1.0 / (1.0 - t))\n\ndef compute_normalization_fixed_point(y_pred, t2, num_iters=5):\n    \"\"\"Returns the normalization value for each example (t > 1.0).\n    Args:\n    y_pred: A multi-dimensional tensor with last dimension `num_classes`.\n    t2: A temperature 2 (> 1.0 for tail heaviness).\n    num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as y_pred with the last dimension being 1.\n    \"\"\"\n    mu = tf.math.reduce_max(y_pred, -1, keepdims=True)\n    normalized_y_pred_step_0 = y_pred - mu\n    normalized_y_pred = normalized_y_pred_step_0\n    i = 0\n    while i < num_iters:\n        i += 1\n        logt_partition = tf.math.reduce_sum(exp_t(normalized_y_pred, t2),-1, keepdims=True)\n        normalized_y_pred = normalized_y_pred_step_0 * (logt_partition ** (1.0 - t2))\n  \n    logt_partition = tf.math.reduce_sum(exp_t(normalized_y_pred, t2), -1, keepdims=True)\n    return -log_t(1.0 / logt_partition, t2) + mu\n\ndef compute_normalization(y_pred, t2, num_iters=5):\n    \"\"\"Returns the normalization value for each example.\n    Args:\n    y_pred: A multi-dimensional tensor with last dimension `num_classes`.\n    t2: A temperature 2 (< 1.0 for finite support, > 1.0 for tail heaviness).\n    num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    if t2 < 1.0:\n        return None # not implemented as these values do not occur in the authors experiments...\n    else:\n        return compute_normalization_fixed_point(y_pred, t2, num_iters)\n\ndef tempered_softmax_activation(x, t2=1., num_iters=5):\n    \"\"\"Tempered softmax function.\n    Args:\n    x: A multi-dimensional tensor with last dimension `num_classes`.\n    t2: A temperature tensor > 0.0.\n    num_iters: Number of iterations to run the method.\n    Returns:\n    A probabilities tensor.\n    \"\"\"\n    if t2 == 1.0:\n        normalization_constants = tf.math.log(tf.math.reduce_sum(tf.math.exp(x), -1, keepdims=True))\n    else:\n        normalization_constants = compute_normalization(x, t2, num_iters)\n\n    return exp_t(x - normalization_constants, t2)\n\nclass TemperedSoftmax(tf.keras.layers.Layer):\n    def __init__(self, t2=1.0, num_iters=5, **kwargs):\n        super(TemperedSoftmax, self).__init__(**kwargs)\n        self.t2 = t2\n        self.num_iters = num_iters\n\n    def call(self, inputs):\n        return tempered_softmax_activation(inputs, t2=self.t2, num_iters=self.num_iters)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tempered Logistic Loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Bi Tempered Logistic Loss\n\ndef bi_tempered_logistic_loss(y_pred, y_true, t1, label_smoothing=0.0):\n    \"\"\"Bi-Tempered Logistic Loss with custom gradient.\n    Args:\n    y_pred: A multi-dimensional probability tensor with last dimension `num_classes`.\n    y_true: A tensor with shape and dtype as y_pred.\n    t1: Temperature 1 (< 1.0 for boundedness).\n    label_smoothing: A float in [0, 1] for label smoothing.\n    Returns:\n    A loss tensor.\n    \"\"\"\n    y_pred = tf.cast(y_pred, tf.float32)\n    y_true = tf.cast(y_true, tf.float32)\n\n    if label_smoothing > 0.0:\n        num_classes = tf.cast(tf.shape(y_true)[-1], tf.float32)\n        y_true = (1 - num_classes /(num_classes - 1) * label_smoothing) * y_true + label_smoothing / (num_classes - 1)\n\n    temp1 = (log_t(y_true + 1e-7, t1) - log_t(y_pred, t1)) * y_true\n    temp2 = (1 / (2 - t1)) * (tf.math.pow(y_true, 2 - t1) - tf.math.pow(y_pred, 2 - t1))\n    loss_values = temp1 - temp2\n\n    return tf.math.reduce_sum(loss_values, -1)\n\nclass BiTemperedLogisticLoss(tf.keras.losses.Loss):\n    def __init__(self, t1, label_smoothing=0.0):\n        super(BiTemperedLogisticLoss, self).__init__()\n        self.t1 = t1\n        self.label_smoothing = label_smoothing\n\n    def call(self, y_true, y_pred):\n        return bi_tempered_logistic_loss(y_pred, y_true, self.t1, self.label_smoothing)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Taylor Cross Entropy Loss\n\n* Taylor cross entropy loss is also a candidate for managing noisy labels\n* But no library is available for keras/TF\n* So I develop a taylor corss entropy loss code greatly referring to the original paper"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Taylor cross entropy loss\ndef taylor_cross_entropy_loss(y_pred, y_true, n=3, label_smoothing=0.0):\n    \"\"\"Taylor Cross Entropy Loss.\n    Args:\n    y_pred: A multi-dimensional probability tensor with last dimension `num_classes`.\n    y_true: A tensor with shape and dtype as y_pred.\n    n: An order of taylor expansion.\n    label_smoothing: A float in [0, 1] for label smoothing.\n    Returns:\n    A loss tensor.\n    \"\"\"\n    y_pred = tf.cast(y_pred, tf.float32)\n    y_true = tf.cast(y_true, tf.float32)\n\n    if label_smoothing > 0.0:\n        num_classes = tf.cast(tf.shape(y_true)[-1], tf.float32)\n        y_true = (1 - num_classes /(num_classes - 1) * label_smoothing) * y_true + label_smoothing / (num_classes - 1)\n    \n    y_pred_n_order = tf.math.maximum(tf.stack([1 - y_pred] * n), 1e-7) # avoide being too small value\n    numerator = tf.math.maximum(tf.math.cumprod(y_pred_n_order, axis=0), 1e-7) # avoide being too small value\n    denominator = tf.expand_dims(tf.expand_dims(tf.range(1, n+1, dtype=\"float32\"), axis=1), axis=1)\n    y_pred_taylor = tf.math.maximum(tf.math.reduce_sum(tf.math.divide(numerator, denominator), axis=0), 1e-7) # avoide being too small value\n    loss_values = tf.math.reduce_sum(y_true * y_pred_taylor, axis=1, keepdims=True)\n    return tf.math.reduce_sum(loss_values, -1)\n\nclass TaylorCrossEntropyLoss(tf.keras.losses.Loss):\n    def __init__(self, n=3, label_smoothing=0.0):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        self.n = n\n        self.label_smoothing = label_smoothing\n    \n    def call(self, y_true, y_pred):\n        return taylor_cross_entropy_loss(y_pred, y_true, n=self.n, label_smoothing=self.label_smoothing)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Experiment\n\n### Bi-tempered logistic loss\n\n* Bi-tempered logistic loss has a parameter t1\n* The closer t1 is to 1, the closer outputs is to that of categorical cross entropy\n\n### Taylor cross entropy\n\n* Taylor corss entropy has a parameter n which is an order of taylor expantion\n* The larger n is (it means developing high-order), the closer outputs are to that of categorical cross entropy"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = [[0.00001, 0.00002, 0.00003, 0.99994],\n          [0.99999999, 0.000000003, 0.000000001, 0.000000001],\n          [0.999950, 0.00003, 0.00001, 0.00001],\n          [0.999950, 0.00003, 0., 0.00002],\n          [8.67612648e-09, 3.44215927e-08, 5.46933476e-09, 1],\n          [0.25, 0.25, 0.25, 0.25]]\n\ny_true = [[0., 0., 0., 1.],\n          [1., 0., 0., 0.],\n          [0., 1., 0., 0.],\n          [1., 0., 0., 0.],\n          [0., 0., 0., 1.],\n          [0., 1., 0., 0.]]\n\nccel = CategoricalCrossentropy()\nbtll_02 = BiTemperedLogisticLoss(t1=0.2)\nbtll_08 = BiTemperedLogisticLoss(t1=0.8)\nbtll_0999 = BiTemperedLogisticLoss(t1=0.999)\nbtll_1 = BiTemperedLogisticLoss(t1=1)\ntcel_3 = TaylorCrossEntropyLoss(n=3)\ntcel_30 = TaylorCrossEntropyLoss(n=30)\ntcel_30000 = TaylorCrossEntropyLoss(n=30000)\n\n\nprint(\"Categorical cross entropy: %s\" % ccel(y_true, y_pred).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.2): %s\" % btll_02(y_true, y_pred).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.8): %s\" % btll_08(y_true, y_pred).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.999): %s\" % btll_0999(y_true, y_pred).numpy())\nprint(\"Bi-tempered logistic loss (t1=1.0): %s\" % btll_1(y_true, y_pred).numpy())\nprint(\"Taylor cross entropy loss (n=3): %s\" % tcel_3(y_true, y_pred).numpy())\nprint(\"Taylor cross entropy loss (n=30): %s\" % tcel_30(y_true, y_pred).numpy())\nprint(\"Taylor cross entropy loss (n=30000): %s\" % tcel_30000(y_true, y_pred).numpy())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### With label smoothing\n\n* I also investigate the effect of label smoothing"},{"metadata":{"trusted":true},"cell_type":"code","source":"label_smoothing = 0.1\nsmoothed_ccel = CategoricalCrossentropy(label_smoothing=label_smoothing)\nsmoothed_btll_02 = BiTemperedLogisticLoss(t1=0.2, label_smoothing=label_smoothing)\nsmoothed_btll_08 = BiTemperedLogisticLoss(t1=0.8, label_smoothing=label_smoothing)\nsmoothed_btll_0999 = BiTemperedLogisticLoss(t1=0.999, label_smoothing=label_smoothing)\nsmoothed_btll_1 = BiTemperedLogisticLoss(t1=1, label_smoothing=label_smoothing)\nsmoothed_tcel_3 = TaylorCrossEntropyLoss(n=3, label_smoothing=label_smoothing)\nsmoothed_tcel_30 = TaylorCrossEntropyLoss(n=30, label_smoothing=label_smoothing)\nsmoothed_tcel_30000 = TaylorCrossEntropyLoss(n=30000, label_smoothing=label_smoothing)\n\n\nprint(\"Smoothed categorical cross entropy: %s\" % smoothed_ccel(y_true, y_pred).numpy())\nprint(\"Smoothed bi-tempered logistic loss (t1=0.2): %s\" % smoothed_btll_02(y_true, y_pred).numpy())\nprint(\"Smoothed bi-tempered logistic loss (t1=0.8): %s\" % smoothed_btll_08(y_true, y_pred).numpy())\nprint(\"Smoothed bi-tempered logistic loss (t1=0.999): %s\" % smoothed_btll_0999(y_true, y_pred).numpy())\nprint(\"Smoothed bi-tempered logistic loss (t1=1.0): %s\" % smoothed_btll_1(y_true, y_pred).numpy())\nprint(\"Smoothed taylor cross entropy loss (n=3): %s\" % smoothed_tcel_3(y_true, y_pred).numpy())\nprint(\"Smoothed taylor cross entropy loss (n=30): %s\" % smoothed_tcel_30(y_true, y_pred).numpy())\nprint(\"Smoothed taylor cross entropy loss (n=30000): %s\" % smoothed_tcel_30000(y_true, y_pred).numpy())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Tempered Softmax Activation\n\n* Tempered softmax activation has a parameter t2\n* The closer t2 is to 1, the closer outputs are to that of simple softmax activation"},{"metadata":{"trusted":true},"cell_type":"code","source":"activation = [[0.00, 0.003, 0.002, 5.],\n              [8.1, 0.003, 0.01, 0.0003],\n              [2.0, 0.005, 0.006, 0.0001],\n              [6.0, 0.01, 0.001, 0.001],\n              [10., 0.0002, 0.002, 0.3],\n              [5.3, 0.001, 0.4, 0.3]]\nactivation_tf = tf.cast(activation, tf.float32)\n\ntempered_softmax_2 = TemperedSoftmax(t2=2)\ntempered_softmax_4 = TemperedSoftmax(t2=4)\ntempered_softmax_1001 = TemperedSoftmax(t2=1.001)\n\ny_pred_softmax = softmax(activation_tf)\ny_pred_tempered_softmax_2 = tempered_softmax_2(activation_tf)\ny_pred_tempered_softmax_4 = tempered_softmax_4(activation_tf)\ny_pred_tempered_softmax_1001 = tempered_softmax_1001(activation_tf)\nprint(\"The softmax reult\")\nprint(y_pred_softmax)\nprint(\"\")\nprint(\"The tempered softmax reult (t2=2)\")\nprint(y_pred_tempered_softmax_2)\nprint(\"\")\nprint(\"The tempered softmax reult (t2=4)\")\nprint(y_pred_tempered_softmax_4)\nprint(\"\")\nprint(\"The tempered softmax reult (t2=1.001)\")\nprint(y_pred_tempered_softmax_1001)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Bi-tempered logistic loss\n\n* I integrate tempered softmax function and tempered logistic loss\n* The closer t1 and t2 are to 1, the closer outputs are to that of categorical corss entropy with softmax"},{"metadata":{"trusted":true},"cell_type":"code","source":"activation = [[0.001, 0.003, 0.002, 5.],\n              [8.1, 0.003, 0.01, 0.0003],\n              [2.0, 0.005, 0.006, 0.0001],\n              [6.0, 0.01, 0.001, 0.001],\n              [10., 0.0002, 0.002, 0.3],\n              [5.3, 0.001, 0.4, 0.3]]\nactivation_tf = tf.cast(activation, tf.float32)\ny_true = [[0., 0., 0., 1.],\n          [1., 0., 0., 0.],\n          [1., 0., 0., 0.],\n          [1., 0., 0., 0.],\n          [0., 1., 0., 0.],\n          [0., 1., 0., 0.]]\n\nprint(\"Categorical cross entropy with softmax: %s\" % ccel(y_true, softmax(activation_tf)).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.2, t2=1.): %s\" % btll_02(y_true, softmax(activation_tf)).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.2, t2=1.001): %s\" % btll_02(y_true, tempered_softmax_1001(activation_tf)).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.2, t2=2.): %s\" % btll_02(y_true, tempered_softmax_2(activation_tf)).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.2, t2=4.): %s\" % btll_02(y_true, tempered_softmax_4(activation_tf)).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.8, t2=1.): %s\" % btll_08(y_true, softmax(activation_tf)).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.2, t2=1.001): %s\" % btll_08(y_true, tempered_softmax_1001(activation_tf)).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.8, t2=2.): %s\" % btll_08(y_true, tempered_softmax_2(activation_tf)).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.8, t2=4.): %s\" % btll_08(y_true, tempered_softmax_4(activation_tf)).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.999, t2=1.): %s\" % btll_0999(y_true, softmax(activation_tf)).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.999, t2=1.001): %s\" % btll_0999(y_true, tempered_softmax_1001(activation_tf)).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.999, t2=2.): %s\" % btll_0999(y_true, tempered_softmax_2(activation_tf)).numpy())\nprint(\"Bi-tempered logistic loss (t1=0.999, t2=4.): %s\" % btll_0999(y_true, tempered_softmax_4(activation_tf)).numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}