{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Analysis of Model flowerclass-efficientnetv2-2\n","metadata":{}},{"cell_type":"code","source":"import math, re, os\nimport numpy as np\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nprint(tf.__version__)\nprint(tfa.__version__)\n\nfrom flowerclass_read_tf_ds import get_datasets\nimport tensorflow_hub as hub\nimport pandas as pd\nimport math\nimport plotly_express as px\nfrom tqdm import tqdm\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport itertools","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:02:47.282552Z","iopub.execute_input":"2022-03-21T13:02:47.282808Z","iopub.status.idle":"2022-03-21T13:02:56.521551Z","shell.execute_reply.started":"2022-03-21T13:02:47.282735Z","shell.execute_reply":"2022-03-21T13:02:56.520801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.test.gpu_device_name()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:02:56.524292Z","iopub.execute_input":"2022-03-21T13:02:56.524756Z","iopub.status.idle":"2022-03-21T13:02:58.575093Z","shell.execute_reply.started":"2022-03-21T13:02:56.524716Z","shell.execute_reply":"2022-03-21T13:02:58.569949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# I. Data Loading","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"image_size = 224\nbatch_size = 64","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:02:58.576608Z","iopub.execute_input":"2022-03-21T13:02:58.577157Z","iopub.status.idle":"2022-03-21T13:02:58.591601Z","shell.execute_reply.started":"2022-03-21T13:02:58.577116Z","shell.execute_reply":"2022-03-21T13:02:58.590678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%debug (50, 480)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:02:58.594318Z","iopub.execute_input":"2022-03-21T13:02:58.594787Z","iopub.status.idle":"2022-03-21T13:02:58.599418Z","shell.execute_reply.started":"2022-03-21T13:02:58.594745Z","shell.execute_reply":"2022-03-21T13:02:58.598691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_names = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102\nlen(class_names)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:02:58.601977Z","iopub.execute_input":"2022-03-21T13:02:58.602499Z","iopub.status.idle":"2022-03-21T13:02:58.615428Z","shell.execute_reply.started":"2022-03-21T13:02:58.602459Z","shell.execute_reply":"2022-03-21T13:02:58.614493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# II. Model Loading: EfficientNetV2","metadata":{}},{"cell_type":"code","source":"effnet2_base = \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_s/feature_vector/2\"","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:02:58.616820Z","iopub.execute_input":"2022-03-21T13:02:58.618327Z","iopub.status.idle":"2022-03-21T13:02:58.626374Z","shell.execute_reply.started":"2022-03-21T13:02:58.618290Z","shell.execute_reply":"2022-03-21T13:02:58.625625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    effnet2_tfhub = tf.keras.Sequential([\n    # Explicitly define the input shape so the model can be properly\n    # loaded by the TFLiteConverter\n    tf.keras.layers.InputLayer(input_shape=(image_size, image_size,3)),\n    hub.KerasLayer(effnet2_base, trainable=False),\n    tf.keras.layers.Dropout(rate=0.2),\n    tf.keras.layers.Dense(104, activation='softmax')\n])\neffnet2_tfhub.build((None, image_size, image_size,3,)) #This is to be used for subclassed models, which do not know at instantiation time what their inputs look like.\n\n\neffnet2_tfhub.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:02:58.627763Z","iopub.execute_input":"2022-03-21T13:02:58.628241Z","iopub.status.idle":"2022-03-21T13:03:11.792752Z","shell.execute_reply.started":"2022-03-21T13:02:58.628205Z","shell.execute_reply":"2022-03-21T13:03:11.792035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_phase = 12\neffnet2_tfhub.load_weights(\"../input/flowerclass-efficientnetv2-2/training/\"+\"cp-\"+f\"{best_phase}\".rjust(4, '0')+\".ckpt\")","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:11.793879Z","iopub.execute_input":"2022-03-21T13:03:11.794350Z","iopub.status.idle":"2022-03-21T13:03:14.507458Z","shell.execute_reply.started":"2022-03-21T13:03:11.794310Z","shell.execute_reply":"2022-03-21T13:03:14.506763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# III. Model Analysis","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:14.508833Z","iopub.execute_input":"2022-03-21T13:03:14.509099Z","iopub.status.idle":"2022-03-21T13:03:14.512561Z","shell.execute_reply.started":"2022-03-21T13:03:14.509064Z","shell.execute_reply":"2022-03-21T13:03:14.511921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ensure that validation data loader returns fixed order of elements.","metadata":{}},{"cell_type":"code","source":"ds_train, ds_valid, ds_test = get_datasets(BATCH_SIZE=batch_size, IMAGE_SIZE=(image_size, image_size), \n                                           RESIZE=None, tpu=False)\n\nimg_preds = []\nimg_labels = []\nfor imgs, label in tqdm(ds_valid):\n    img_preds.append(effnet2_tfhub.predict(imgs, batch_size=batch_size))\n    img_labels.append(label.numpy())\n    \nimg_preds = np.concatenate([img_pred.argmax(1) for img_pred in img_preds])\nimg_labels = np.concatenate([img_label.argmax(1) for img_label in img_labels])\n","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:14.515815Z","iopub.execute_input":"2022-03-21T13:03:14.516265Z","iopub.status.idle":"2022-03-21T13:03:55.853786Z","shell.execute_reply.started":"2022-03-21T13:03:14.516230Z","shell.execute_reply":"2022-03-21T13:03:55.852897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_results = pd.DataFrame({'pred': img_preds, \"label\":img_labels})","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:55.855028Z","iopub.execute_input":"2022-03-21T13:03:55.855478Z","iopub.status.idle":"2022-03-21T13:03:55.863665Z","shell.execute_reply.started":"2022-03-21T13:03:55.855437Z","shell.execute_reply":"2022-03-21T13:03:55.862452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_results.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:55.864872Z","iopub.execute_input":"2022-03-21T13:03:55.865673Z","iopub.status.idle":"2022-03-21T13:03:55.892470Z","shell.execute_reply.started":"2022-03-21T13:03:55.865646Z","shell.execute_reply":"2022-03-21T13:03:55.891799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# IIIa) Overall Evaluation","metadata":{}},{"cell_type":"code","source":"confusion_matrix(val_results['label'], val_results['pred'])","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:55.893650Z","iopub.execute_input":"2022-03-21T13:03:55.893872Z","iopub.status.idle":"2022-03-21T13:03:55.910519Z","shell.execute_reply.started":"2022-03-21T13:03:55.893846Z","shell.execute_reply":"2022-03-21T13:03:55.909418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(val_results['label'], val_results['pred'], target_names=class_names))","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:55.911547Z","iopub.execute_input":"2022-03-21T13:03:55.911841Z","iopub.status.idle":"2022-03-21T13:03:55.934679Z","shell.execute_reply.started":"2022-03-21T13:03:55.911807Z","shell.execute_reply":"2022-03-21T13:03:55.934024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_report = pd.DataFrame.from_dict(classification_report(val_results['label'], val_results['pred'], target_names=class_names, output_dict=True)).T\n\nclass_report['class'] = class_report.index\nclass_report= class_report.reset_index(drop=True)\n\nclass_report.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:55.935844Z","iopub.execute_input":"2022-03-21T13:03:55.936248Z","iopub.status.idle":"2022-03-21T13:03:55.973320Z","shell.execute_reply.started":"2022-03-21T13:03:55.936215Z","shell.execute_reply":"2022-03-21T13:03:55.972652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"MOst problematic classes with f1 below 90:\n\n\n> How would improving these classes raise the macro f1 score?","metadata":{}},{"cell_type":"code","source":"class_report = class_report.loc[:103] # remove the summary statistics, e.g. accuracy","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:55.974564Z","iopub.execute_input":"2022-03-21T13:03:55.974807Z","iopub.status.idle":"2022-03-21T13:03:55.979312Z","shell.execute_reply.started":"2022-03-21T13:03:55.974773Z","shell.execute_reply":"2022-03-21T13:03:55.978257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_report = class_report.sort_values(\"f1-score\").reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:55.980530Z","iopub.execute_input":"2022-03-21T13:03:55.980872Z","iopub.status.idle":"2022-03-21T13:03:55.991560Z","shell.execute_reply.started":"2022-03-21T13:03:55.980831Z","shell.execute_reply":"2022-03-21T13:03:55.990857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_report.head(9)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:55.993129Z","iopub.execute_input":"2022-03-21T13:03:55.993395Z","iopub.status.idle":"2022-03-21T13:03:56.011609Z","shell.execute_reply.started":"2022-03-21T13:03:55.993362Z","shell.execute_reply":"2022-03-21T13:03:56.010857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> What is wrong with the rose class? bad performance despite many images","metadata":{}},{"cell_type":"markdown","source":"> * If we would improve all 8 worst-performing classes to f1 score of 1, it would still only raise performance by 1%! See below.\n> * \n","metadata":{}},{"cell_type":"code","source":"class_report_test = class_report.copy()\nclass_report_test.loc[:7, 'f1-score'] = 1\nclass_report_test['f1-score'].mean()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.013378Z","iopub.execute_input":"2022-03-21T13:03:56.013580Z","iopub.status.idle":"2022-03-21T13:03:56.023857Z","shell.execute_reply.started":"2022-03-21T13:03:56.013556Z","shell.execute_reply":"2022-03-21T13:03:56.022944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_report_test.loc[:20, 'f1-score'] = 1\nclass_report_test['f1-score'].mean()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.025452Z","iopub.execute_input":"2022-03-21T13:03:56.025824Z","iopub.status.idle":"2022-03-21T13:03:56.034789Z","shell.execute_reply.started":"2022-03-21T13:03:56.025785Z","shell.execute_reply":"2022-03-21T13:03:56.033826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> * Improve first 20 classes would raise by another 1%.\n> * It might be better to improve the overall performance of the model then trying to improve individual classes\n\n> * Nevertheless continue with error analysis","metadata":{}},{"cell_type":"code","source":"class_report.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.036273Z","iopub.execute_input":"2022-03-21T13:03:56.036575Z","iopub.status.idle":"2022-03-21T13:03:56.048114Z","shell.execute_reply.started":"2022-03-21T13:03:56.036523Z","shell.execute_reply":"2022-03-21T13:03:56.047217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(class_report['f1-score'], kde=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.049414Z","iopub.execute_input":"2022-03-21T13:03:56.050395Z","iopub.status.idle":"2022-03-21T13:03:56.351473Z","shell.execute_reply.started":"2022-03-21T13:03:56.050356Z","shell.execute_reply":"2022-03-21T13:03:56.350823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_report['f1-score'].describe().to_frame().T","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.352526Z","iopub.execute_input":"2022-03-21T13:03:56.353263Z","iopub.status.idle":"2022-03-21T13:03:56.369686Z","shell.execute_reply.started":"2022-03-21T13:03:56.353225Z","shell.execute_reply":"2022-03-21T13:03:56.368950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Group classes into a easy category (good performance) and bad performance.","metadata":{}},{"cell_type":"code","source":"class_report['difficulty'] = 'hard'\nclass_report.loc[8:, 'difficulty'] = class_report.loc[8:, 'f1-score'].apply(lambda x: 'easy' if x>0.969 else 'medium')","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.371064Z","iopub.execute_input":"2022-03-21T13:03:56.371472Z","iopub.status.idle":"2022-03-21T13:03:56.377629Z","shell.execute_reply.started":"2022-03-21T13:03:56.371436Z","shell.execute_reply":"2022-03-21T13:03:56.376947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_report.groupby(\"difficulty\").agg(['mean', 'median'])","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.378859Z","iopub.execute_input":"2022-03-21T13:03:56.379304Z","iopub.status.idle":"2022-03-21T13:03:56.409611Z","shell.execute_reply.started":"2022-03-21T13:03:56.379264Z","shell.execute_reply":"2022-03-21T13:03:56.408944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hypothesis test with nonparametric Mann-Whitney U test to compare the samples with label easy and hard above:","metadata":{}},{"cell_type":"code","source":"import scipy\nscipy.stats.mannwhitneyu(class_report.loc[class_report['difficulty'] == 'easy', 'support'],  \n                        class_report.loc[class_report['difficulty'] == 'hard', 'support'])\n","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.410900Z","iopub.execute_input":"2022-03-21T13:03:56.411309Z","iopub.status.idle":"2022-03-21T13:03:56.420522Z","shell.execute_reply.started":"2022-03-21T13:03:56.411273Z","shell.execute_reply":"2022-03-21T13:03:56.419720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> We cannot reject the null hypothesis that both samples, easy and hard, come from the same distribution. This means there is no evidence to reject the null hypothesis at the 5% level that the number of data points are a reason for the difference between the easy and and hard classes. ","metadata":{}},{"cell_type":"markdown","source":"### Common Errors","metadata":{}},{"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.422802Z","iopub.execute_input":"2022-03-21T13:03:56.423550Z","iopub.status.idle":"2022-03-21T13:03:56.434793Z","shell.execute_reply.started":"2022-03-21T13:03:56.423504Z","shell.execute_reply":"2022-03-21T13:03:56.433932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conf_matrix = confusion_matrix(val_results['label'], val_results['pred'])","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.440806Z","iopub.execute_input":"2022-03-21T13:03:56.441423Z","iopub.status.idle":"2022-03-21T13:03:56.452624Z","shell.execute_reply.started":"2022-03-21T13:03:56.441391Z","shell.execute_reply":"2022-03-21T13:03:56.451960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot_confusion_matrix(confusion_matrix(val_results['label'], val_results['pred']), class_names)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.455353Z","iopub.execute_input":"2022-03-21T13:03:56.455556Z","iopub.status.idle":"2022-03-21T13:03:56.460131Z","shell.execute_reply.started":"2022-03-21T13:03:56.455531Z","shell.execute_reply":"2022-03-21T13:03:56.459051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conf_matrix.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.461995Z","iopub.execute_input":"2022-03-21T13:03:56.462970Z","iopub.status.idle":"2022-03-21T13:03:56.469400Z","shell.execute_reply.started":"2022-03-21T13:03:56.462931Z","shell.execute_reply":"2022-03-21T13:03:56.468595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Confusion (matrix) of top 7 worst performing classes","metadata":{}},{"cell_type":"code","source":"class_names[:3]","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.470760Z","iopub.execute_input":"2022-03-21T13:03:56.471347Z","iopub.status.idle":"2022-03-21T13:03:56.480395Z","shell.execute_reply.started":"2022-03-21T13:03:56.471309Z","shell.execute_reply":"2022-03-21T13:03:56.479650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conf_matrix","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.481835Z","iopub.execute_input":"2022-03-21T13:03:56.482388Z","iopub.status.idle":"2022-03-21T13:03:56.489645Z","shell.execute_reply.started":"2022-03-21T13:03:56.482351Z","shell.execute_reply":"2022-03-21T13:03:56.488737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_names_mapping = {value:key for key, value in  enumerate(class_names)}","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.499109Z","iopub.execute_input":"2022-03-21T13:03:56.499976Z","iopub.status.idle":"2022-03-21T13:03:56.505834Z","shell.execute_reply.started":"2022-03-21T13:03:56.499937Z","shell.execute_reply":"2022-03-21T13:03:56.504947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_names_mapping\nclass_report['idx'] = class_report['class'].map(class_names_mapping)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.507525Z","iopub.execute_input":"2022-03-21T13:03:56.508043Z","iopub.status.idle":"2022-03-21T13:03:56.516624Z","shell.execute_reply.started":"2022-03-21T13:03:56.508007Z","shell.execute_reply":"2022-03-21T13:03:56.515876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_report.head(7)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:03:56.517835Z","iopub.execute_input":"2022-03-21T13:03:56.518560Z","iopub.status.idle":"2022-03-21T13:03:56.534552Z","shell.execute_reply.started":"2022-03-21T13:03:56.518510Z","shell.execute_reply":"2022-03-21T13:03:56.533790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"worst_classes_FN = conf_matrix[class_report.loc[:7, \"idx\"]]\nworst_classes_FN.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:38:00.504330Z","iopub.execute_input":"2022-03-21T13:38:00.504863Z","iopub.status.idle":"2022-03-21T13:38:00.510421Z","shell.execute_reply.started":"2022-03-21T13:38:00.504824Z","shell.execute_reply":"2022-03-21T13:38:00.509739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"worst_classes_FN_sub = worst_classes_FN[:, worst_classes_FN.sum(0) > 0]\nworst_classes_FN_sub.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:38:06.422303Z","iopub.execute_input":"2022-03-21T13:38:06.422555Z","iopub.status.idle":"2022-03-21T13:38:06.430353Z","shell.execute_reply.started":"2022-03-21T13:38:06.422527Z","shell.execute_reply":"2022-03-21T13:38:06.429594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from mpl_toolkits.axes_grid1 import make_axes_locatable\n\ndef plot_confusion_matrix(cm, xclasses, yclasses, title_prefix, figsize=(16,8)):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    fig = plt.figure(figsize=figsize)\n    plt.title(title_prefix+f\" top {len(yclasses)} classes by f1 score\")\n    ax = plt.gca()\n    cmap=plt.cm.Blues\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    #plt.colorbar(fraction=0.046, pad=0.04)\n    \n    tick_marks_y = np.arange(len(yclasses))\n    tick_marks_x = np.arange(len(xclasses))\n    plt.xticks(tick_marks_x, xclasses, rotation=45)\n    plt.yticks(tick_marks_y, yclasses)\n\n    for (j,i),label in np.ndenumerate(cm):\n        ax.text(i,j,label,ha='center',va='center')\n    \n    plt.tight_layout()\n    if not title_prefix=='FP':\n        plt.ylabel('True label')\n        plt.xlabel('Predicted label')\n    else:\n        plt.xlabel('True label')\n        plt.ylabel('Predicted label')","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:44:41.204071Z","iopub.execute_input":"2022-03-21T13:44:41.204430Z","iopub.status.idle":"2022-03-21T13:44:41.214686Z","shell.execute_reply.started":"2022-03-21T13:44:41.204397Z","shell.execute_reply":"2022-03-21T13:44:41.213826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_confusion_matrix(worst_classes_FN_sub, xclasses= np.array(class_names)[worst_classes_FN.sum(0) > 0], \n                     yclasses=class_report.loc[:7, \"class\"], title_prefix='FN' )","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:44:42.787382Z","iopub.execute_input":"2022-03-21T13:44:42.787994Z","iopub.status.idle":"2022-03-21T13:44:43.894422Z","shell.execute_reply.started":"2022-03-21T13:44:42.787951Z","shell.execute_reply":"2022-03-21T13:44:43.893738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### FN Results:\n* globe-flower (true): only 1 confused with buttercup\n* clematis: 2 confused with windflower and columbine\n* canterbury bells: no FN\n* mexican petunia: 1 confused with petunia, maybe label error?\n* black-eyed susan: 5 confused with sunflower.\n*  peruvian lily: 1 with lenten rose, one with rose. both are of type rose, by chance?\n* rose: 2 with sunflower, 3 with commun tulip, 1 confused with baberton daisy, daisy, 2 sunflower, 1 lotus: mix ups spread among classes\n* gazania: one tiger lily, 1 baberton daisy, 1 rose, 1 blanket flower.","metadata":{}},{"cell_type":"code","source":"worst_classes_FP = conf_matrix[:, class_report.loc[:7, \"idx\"]]\nprint(worst_classes_FP.shape)\n\nworst_classes_conf_FP = worst_classes_FP[worst_classes_FP.sum(1) > 0, :]\nworst_classes_conf_FP.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:44:46.571154Z","iopub.execute_input":"2022-03-21T13:44:46.571759Z","iopub.status.idle":"2022-03-21T13:44:46.580646Z","shell.execute_reply.started":"2022-03-21T13:44:46.571718Z","shell.execute_reply":"2022-03-21T13:44:46.579802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_confusion_matrix(worst_classes_conf_FP.T, xclasses= np.array(class_names)[worst_classes_FP.sum(1) > 0], \n                     yclasses=class_report.loc[:7, \"class\"], title_prefix='FP' )","metadata":{"execution":{"iopub.status.busy":"2022-03-21T13:44:46.757857Z","iopub.execute_input":"2022-03-21T13:44:46.758328Z","iopub.status.idle":"2022-03-21T13:44:47.884479Z","shell.execute_reply.started":"2022-03-21T13:44:46.758298Z","shell.execute_reply":"2022-03-21T13:44:47.883801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### FP Results\n* globe-flower: 1 confused with (true) lotus\n* dematis: no wrong detection in other classes\n* caterbury bells: confused with true balloon flower\n* mexican petunia: confusesd iwth 1 true petunia and 1 true desert rose\n* black-eyed susan: confused with 1 true daisy\n* peruvian lily: confused with 1 true tiger lily,\n* rose: confused with 1 true snapdragon, 1 true peruvian lily, 2  camation and other classes. algo thinks everything is a rose which could be due to the relatively larger amount of images for this class. \n* gazania: confused with 2 true marigold, \n\n> the differences between FN and FP confused classes for the top 8 indicates that the type of confusion of the algorithm might be of different nature.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}