{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Model Error analysis\nIn this kernel I did some basic error analysis of one of my models predictions.  \nThis was a densenet model with lb near 0.96 but the types of error will be same  \nfor most of the models.\n\nI hope someone benifits from this."},{"metadata":{},"cell_type":"markdown","source":"imports and Loading validation set targets and predictions"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport random\nimport sklearn\n\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom PIL import Image\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import StratifiedKFold\n\nplt.style.use('bmh')\n\ntargets0 = np.load(\"../input/train-single-best-model-error-analysis/targets0.npy\")\ntargets1 = np.load(\"../input/train-single-best-model-error-analysis/targets1.npy\")\ntargets2 = np.load(\"../input/train-single-best-model-error-analysis/targets2.npy\")\npreds0 = np.load(\"../input/train-single-best-model-error-analysis/preds0.npy\")\npreds1 = np.load(\"../input/train-single-best-model-error-analysis/preds1.npy\")\npreds2 = np.load(\"../input/train-single-best-model-error-analysis/preds2.npy\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"helper function"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def print_confusion_matrix(confusion_matrix, class_names, figsize = (10,7), fontsize=14):\n    \"\"\"Prints a confusion matrix, as returned by sklearn.metrics.confusion_matrix, as a heatmap.\n    \n    Arguments\n    ---------\n    confusion_matrix: numpy.ndarray\n        The numpy.ndarray object returned from a call to sklearn.metrics.confusion_matrix. \n        Similarly constructed ndarrays can also be used.\n    class_names: list\n        An ordered list of class names, in the order they index the given confusion matrix.\n    figsize: tuple\n        A 2-long tuple, the first value determining the horizontal size of the ouputted figure,\n        the second determining the vertical size. Defaults to (10,7).\n    fontsize: int\n        Font size for axes labels. Defaults to 14.\n        \n    Returns\n    -------\n    matplotlib.figure.Figure\n        The resulting confusion matrix figure\n    \"\"\"\n    df_cm = pd.DataFrame(\n        confusion_matrix, index=class_names, columns=class_names, \n    )\n    fig = plt.figure(figsize=figsize)\n    sns.set(font_scale=1.5)\n    try:\n        heatmap = sns.heatmap(df_cm, annot=True, fmt=\"d\")\n    except ValueError:\n        raise ValueError(\"Confusion matrix values must be integers.\")\n    heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=fontsize)\n    heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=45, ha='right', fontsize=fontsize)\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    return fig","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1) vowel_diacritic"},{"metadata":{},"cell_type":"markdown","source":"## Confusion matix"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"confusion_matrix = sklearn.metrics.confusion_matrix(np.array(targets1), np.array(preds1))\nprint_confusion_matrix(confusion_matrix, [i for i in range(confusion_matrix.shape[0])], figsize = (20,15), fontsize=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Error rate per class"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from sklearn.metrics import classification_report\nprint(classification_report(targets1, preds1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"score_dict = classification_report(targets1, preds1, output_dict=True)\nsupport, recalls, precision, f1 , cls = [], [], [], [], []\nfor i in range(11):\n    cls.append(i)\n    support.append(score_dict[str(i)]['support'])\n    recalls.append(score_dict[str(i)]['recall'])\n    precision.append(score_dict[str(i)]['precision'])\n    f1.append(score_dict[str(i)]['f1-score'])\n\nf, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(20, 12))\nax1.bar(cls, recalls, width=0.8, bottom=None, align='center', color='#F65058')\nax1.set_title(\"recall\",fontsize= 25)\nax2.bar(cls, precision, width=0.8, bottom=None, align='center', color='#FBDE44')\nax2.set_title(\"precision\",fontsize= 25)\nax3.bar(cls, support, width=0.8, bottom=None, align='center', color=\"#28334A\")\nax3.set_title(\"support\",fontsize= 25)\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2) consonant_diacritic"},{"metadata":{},"cell_type":"markdown","source":"## Confusion matrix"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"confusion_matrix = sklearn.metrics.confusion_matrix(np.array(targets2), np.array(preds2))\nprint_confusion_matrix(confusion_matrix, [i for i in range(confusion_matrix.shape[0])], figsize = (20,15), fontsize=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Error rate per class"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from sklearn.metrics import classification_report\nprint(classification_report(targets2, preds2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"score_dict = classification_report(targets2, preds2, output_dict=True)\nsupport, recalls, precision, f1 , cls = [], [], [], [], []\nfor i in range(7):\n    cls.append(i)\n    support.append(score_dict[str(i)]['support'])\n    recalls.append(score_dict[str(i)]['recall'])\n    precision.append(score_dict[str(i)]['precision'])\n    f1.append(score_dict[str(i)]['f1-score'])\n\nf, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(20, 12))\nax1.bar(cls, recalls, width=0.8, bottom=None, align='center', color='#F65058')\nax1.set_title(\"recall\",fontsize= 25)\nax2.bar(cls, precision, width=0.8, bottom=None, align='center', color='#FBDE44')\nax2.set_title(\"precision\",fontsize= 25)\nax3.bar(cls, support, width=0.8, bottom=None, align='center', color=\"#28334A\")\nax3.set_title(\"support\",fontsize= 25)\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3) grapheme_root"},{"metadata":{},"cell_type":"markdown","source":"## Error rate per class"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from sklearn.metrics import classification_report\nprint(classification_report(targets0, preds0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"score_dict = classification_report(targets0, preds0, output_dict=True)\nsupport, recalls, precision, f1 , cls = [], [], [], [], []\nfor i in range(168):\n    cls.append(i)\n    support.append(score_dict[str(i)]['support'])\n    recalls.append(score_dict[str(i)]['recall'])\n    precision.append(score_dict[str(i)]['precision'])\n    f1.append(score_dict[str(i)]['f1-score'])\n\nf, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(20, 12))\nax1.bar(cls, recalls, width=0.8, bottom=None, align='center', color='#F65058')\nax1.set_title(\"recall\",fontsize= 25)\nax2.bar(cls, precision, width=0.8, bottom=None, align='center', color='#FBDE44')\nax2.set_title(\"precision\",fontsize= 25)\nax3.bar(cls, support, width=0.8, bottom=None, align='center', color=\"#28334A\")\nax3.set_title(\"support\",fontsize= 25)\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"sup = [i/(1 * max(support)) for i in support]\nf, ax = plt.subplots(figsize=(20, 5))\nax.bar(cls, recalls, width=0.7, bottom=None, align='center', color='#F65058')\nax.bar(cls, sup, width=0.7, bottom=None, align='center', color=\"#28334A\")\nax.set_title(\"Recall (red), with support (black)\",fontsize= 25)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### We can see that the classes with least recalls have less support"},{"metadata":{},"cell_type":"markdown","source":"## confusion matrix"},{"metadata":{},"cell_type":"markdown","source":"### Open image in new tab"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"confusion_matrix = sklearn.metrics.confusion_matrix(np.array(targets0), np.array(preds0))\nprint_confusion_matrix(confusion_matrix, [i for i in range(confusion_matrix.shape[0])], figsize = (50,40), fontsize=6)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"confusion_matrix = sklearn.metrics.confusion_matrix(np.array(targets0), np.array(preds0))\nerrors = []\nfor i in range(confusion_matrix.shape[0]):\n    for j in range(confusion_matrix.shape[0]):\n        if confusion_matrix[i][j] != 0 and i != j:\n            errors.append(confusion_matrix[i][j])\nfig, ax = plt.subplots(figsize=(20,5))\nsns.countplot(ax=ax, x=errors)\nax.set_title(\"Count of Non diagonal non zero entries\",fontsize= 25)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Non diagonal entries which are greater than 8 "},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"confusion_matrix = sklearn.metrics.confusion_matrix(np.array(targets0), np.array(preds0))\nprint(\"True\\tPred\\tcount/support\")\nfor i in range(confusion_matrix.shape[0]):\n    for j in range(confusion_matrix.shape[0]):\n        if confusion_matrix[i][j] != 0 and i != j:\n            if confusion_matrix[i][j] > 8:\n                print(f\"{i}\\t{j}\\t{confusion_matrix[i][j]}/{support[i]}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Please Upvote if you found this useful"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}