{"cells":[{"metadata":{},"cell_type":"markdown","source":"## An implementation of the competition metric for TPU usage\n\nThis kernel uses https://www.kaggle.com/anokas/weighted-auc-metric-updated to validate the TPU ready metric. \n\nThe TPU metric modifies the class tf.keras.metrics.AUC. Do not modify the class more than once in a single session. \n\nThis technique of modifying the class is not recommended, but there were issues when attempting to subclass tf.keras.metric.AUC that I didn't want to deal with. \n\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# good code to test my bad code \n\nfrom sklearn import metrics\nimport numpy as np\n\ndef alaska_weighted_auc(y_true, y_valid):\n    tpr_thresholds = [0.0, 0.4, 1.0]\n    weights =        [       2,   1]\n    \n    fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)\n    \n    # size of subsets\n    areas = np.array(tpr_thresholds[1:]) - np.array(tpr_thresholds[:-1])\n    \n    # The total area is normalized by the sum of weights such that the final weighted AUC is between 0 and 1.\n    normalization = np.dot(areas, weights)\n\n    competition_metric = 0\n    \n    for idx, weight in enumerate(weights):\n        y_min = tpr_thresholds[idx]\n        y_max = tpr_thresholds[idx + 1]\n        mask = (y_min < tpr) & (tpr < y_max)\n\n        x_padding = np.linspace(fpr[mask][-1], 1, 100)\n\n        x = np.concatenate([fpr[mask], x_padding])\n        y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])\n        y = y - y_min # normalize such that curve starts at y=0\n        score = metrics.auc(x, y)\n        submetric = score * weight\n        best_subscore = (y_max - y_min) * weight\n        competition_metric += submetric\n        \n    return competition_metric / normalization","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# bad code, but it works? \n\nimport tensorflow as tf \nfrom tensorflow.python.ops import math_ops\nimport types\n    \ndef fix_auc(binary = False):\n    # only tested with default parameters for the AUC class\n    # do not run this again in the same session \n    \n    def update_state(self, y_true, y_pred, sample_weight=None):\n        # for sparse categorical accuracy \n        \n        y_true = tf.cast(y_true != 0, tf.int64)\n        y_pred = 1 - y_pred[:, 0]\n        return self._update_state(y_true, y_pred, sample_weight)\n\n    def result(self):\n\n        normalization = 1.4\n\n        recall = math_ops.div_no_nan(self.true_positives,\n                                     self.true_positives + self.false_negatives)\n\n        fp_rate = math_ops.div_no_nan(self.false_positives,\n                                    self.false_positives + self.true_negatives)\n        x = fp_rate\n        y = recall\n\n        heights = (y[:self.num_thresholds - 1] + y[1:]) / 2.\n\n        regular_auc = math_ops.reduce_sum(math_ops.multiply(x[:self.num_thresholds - 1] - x[1:], heights), name=self.name)\n\n        under40_auc = math_ops.reduce_sum(math_ops.multiply(x[:self.num_thresholds - 1] - x[1:], tf.clip_by_value (heights, 0, 0.4)), name=self.name)\n\n        return (regular_auc + under40_auc) / 1.4\n    \n    if not binary: \n        if not hasattr(tf.keras.metrics.AUC, '_update_state'):\n            tf.keras.metrics.AUC._update_state = tf.keras.metrics.AUC.update_state\n        tf.keras.metrics.AUC.update_state = update_state\n    \n    tf.keras.metrics.AUC.result = result\n    \n    return tf.keras.metrics.AUC\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check if it actually works\n# differences may be due to thresholds or estimation method\n\n\ntf.keras.metrics.AUC = fix_auc(binary = True)\n\nfor i in range(10):\n    signal = np.random.random(1000)\n    labels = (signal > 0.5).astype(int)\n    preds = np.random.random(1000) + (signal - 0.5) * i * 0.1\n    preds = np.clip(preds, 0, 1)\n    \n    \n    auc = metrics.roc_auc_score(labels, preds)\n    weighted_auc = alaska_weighted_auc(labels, preds)\n    accuracy = (labels == (preds > 0.5)).mean()\n    \n    print(auc, weighted_auc, tf.keras.metrics.AUC()(labels, preds))\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# usage. remember to initialize the metric class instance in the scope \n\ntf.keras.metrics.AUC = fix_auc(binary = False)\n\n'''\nwith strategy.scope():\n    model = your_model\n    \n    metrics = [tf.keras.metrics.AUC()]\n    \n    model.compile(\n        optimizer='adam',\n        loss = losses,\n        metrics= metrics        \n    )\n\n'''","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}