{"cells":[{"metadata":{},"cell_type":"markdown","source":"> ## About this kernel\n\nThis kernel builds on top of the great work of\n https://www.kaggle.com/xhlulu/alaska2-efficientnet-on-tpus\n \nMetric has been updated (see https://www.kaggle.com/c/alaska2-image-steganalysis/discussion/147182)\n \nWe use the trained model and calculate the competition metric on the predictions of the validation set.\nThe predictions have been calcualted in version 1 of this notebook."},{"metadata":{},"cell_type":"markdown","source":"## Evaluation"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics import auc\nfrom sklearn import metrics\nimport matplotlib.pyplot as plt\nimport pickle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(\"../input/alaska2-efficientnet-on-tpus-valid-preds/valid.pkl\", \"rb\") as f:\n    [y_valid, y_true] = pickle.load(f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_metric(y_true, y_pred):\n    tpr_thresholds = [0.0, 0.4, 1.0]\n    weights =        [       2, 1]\n    \n    areas = np.array(tpr_thresholds[1:]) - np.array(tpr_thresholds[:-1])\n    normalization = np.dot(areas, weights)\n\n    fpr, tpr, thresholds = metrics.roc_curve(y_true, y_pred)\n    competition_metric = 0\n\n    for idx, weight in enumerate(weights):\n        y_min = tpr_thresholds[idx]\n        y_max = tpr_thresholds[idx + 1]\n        mask = (y_min < tpr) & (tpr < y_max)\n\n        x_padding = np.linspace(fpr[mask][-1], 1, 100)\n\n        x = np.concatenate([fpr[mask], x_padding])\n        y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])\n        y = y - y_min # normalize such that curve starts at y=0\n        score = auc(x, y)\n        submetric = score * weight\n        best_subscore = (y_max - y_min) * weight\n        competition_metric += submetric\n        \n    competition_metric = competition_metric / normalization\n    return competition_metric","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_auc_plots(y_true, y_pred):\n    fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid)\n    \n    plt.scatter(fpr, tpr)\n    plt.title(\"ROC Curve\")\n    plt.xlabel(\"False Positive Rate\")\n    plt.ylabel(\"True Positive Rate\")\n    plt.plot(np.linspace(0, 1), [0.4] *50, color=\"green\")\n    plt.show()\n    \n    tpr_thresholds = [0.0, 0.4, 1.0]\n    \n    for idx in [1, 0]:\n        \n        y_min = tpr_thresholds[idx]\n        y_max = tpr_thresholds[idx + 1]\n        mask = (y_min < tpr) & (tpr < y_max)\n\n        x_padding = np.linspace(fpr[mask][-1], 1, 100)\n\n        x = np.concatenate([fpr[mask], x_padding])\n        y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])\n        y = y - y_min # normalize such that curve starts at y=0\n        score = auc(x, y)\n\n        fig, ax = plt.subplots(1, 1)\n        ax.fill_between(x, 0, y)\n        ax.scatter(x, y)\n        plt.xlim(0, 1)\n        plt.ylim(0, y_max - y_min)\n        plt.xlabel(\"False Positive Rate\")\n        plt.ylabel(\"True Positive Rate\")\n        plt.title(f\"AUC curve between {y_min} and {y_max}. AUC: {score:.3f}, Best AUC: {y_max - y_min}\")\n        plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"calculate_metric(y_true, y_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_auc_plots(y_true, y_valid)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":4}