{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Analysis of Model flowerclass-efficientnetv2-2 2: with Image Visualizations\n\n### Goals\n\n* Analysis of the top 8 worst performing classes\n* Leverage simple image visualizations to gain insight into algorithm\n","metadata":{}},{"cell_type":"code","source":"import math, re, os\nimport numpy as np\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nprint(tf.__version__)\nprint(tfa.__version__)\n\nfrom flowerclass_read_tf_ds import get_datasets, display_batch_by_class, display_batch_of_images #, load_dataset, display_batch_of_images, batch_to_numpy_images_and_labels, display_one_flower\nimport tensorflow_hub as hub\nimport pandas as pd\nimport math\nimport plotly_express as px\nfrom tqdm import tqdm\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport itertools","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:06.090119Z","iopub.execute_input":"2022-03-21T20:54:06.090357Z","iopub.status.idle":"2022-03-21T20:54:06.098634Z","shell.execute_reply.started":"2022-03-21T20:54:06.090326Z","shell.execute_reply":"2022-03-21T20:54:06.097943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.test.gpu_device_name()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:06.100231Z","iopub.execute_input":"2022-03-21T20:54:06.100543Z","iopub.status.idle":"2022-03-21T20:54:06.110687Z","shell.execute_reply.started":"2022-03-21T20:54:06.100446Z","shell.execute_reply":"2022-03-21T20:54:06.109924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# I. Data Loading","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"image_size = 224\nbatch_size = 64","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:06.112042Z","iopub.execute_input":"2022-03-21T20:54:06.11251Z","iopub.status.idle":"2022-03-21T20:54:06.120751Z","shell.execute_reply.started":"2022-03-21T20:54:06.112474Z","shell.execute_reply":"2022-03-21T20:54:06.12Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_names = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102\nlen(class_names)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:06.122276Z","iopub.execute_input":"2022-03-21T20:54:06.122947Z","iopub.status.idle":"2022-03-21T20:54:06.13615Z","shell.execute_reply.started":"2022-03-21T20:54:06.12291Z","shell.execute_reply":"2022-03-21T20:54:06.135343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# II. Model Loading and Predictions: EfficientNetV2","metadata":{}},{"cell_type":"code","source":"effnet2_base = \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_s/feature_vector/2\"","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:06.137629Z","iopub.execute_input":"2022-03-21T20:54:06.138684Z","iopub.status.idle":"2022-03-21T20:54:06.145555Z","shell.execute_reply.started":"2022-03-21T20:54:06.138645Z","shell.execute_reply":"2022-03-21T20:54:06.144772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    effnet2_tfhub = tf.keras.Sequential([\n    # Explicitly define the input shape so the model can be properly\n    # loaded by the TFLiteConverter\n    tf.keras.layers.InputLayer(input_shape=(image_size, image_size,3)),\n    hub.KerasLayer(effnet2_base, trainable=False),\n    tf.keras.layers.Dropout(rate=0.2),\n    tf.keras.layers.Dense(104, activation='softmax')\n])\neffnet2_tfhub.build((None, image_size, image_size,3,)) #This is to be used for subclassed models, which do not know at instantiation time what their inputs look like.\n\n\neffnet2_tfhub.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:06.146799Z","iopub.execute_input":"2022-03-21T20:54:06.147212Z","iopub.status.idle":"2022-03-21T20:54:16.261447Z","shell.execute_reply.started":"2022-03-21T20:54:06.147175Z","shell.execute_reply":"2022-03-21T20:54:16.260751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_phase = 12\neffnet2_tfhub.load_weights(\"../input/flowerclass-efficientnetv2-2/training/\"+\"cp-\"+f\"{best_phase}\".rjust(4, '0')+\".ckpt\")","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:16.263648Z","iopub.execute_input":"2022-03-21T20:54:16.264275Z","iopub.status.idle":"2022-03-21T20:54:17.283378Z","shell.execute_reply.started":"2022-03-21T20:54:16.264237Z","shell.execute_reply":"2022-03-21T20:54:17.282695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:17.284531Z","iopub.execute_input":"2022-03-21T20:54:17.284805Z","iopub.status.idle":"2022-03-21T20:54:17.289449Z","shell.execute_reply.started":"2022-03-21T20:54:17.284767Z","shell.execute_reply":"2022-03-21T20:54:17.288397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ensure that validation data loader returns fixed order of elements.","metadata":{}},{"cell_type":"code","source":"ds_train, ds_valid, ds_test = get_datasets(BATCH_SIZE=batch_size, IMAGE_SIZE=(image_size, image_size), \n                                           RESIZE=None, tpu=False, with_id=True)\n\nimg_preds = []\nimg_labels = []\nimg_ids = []\nfor imgs, label, imgs_id in tqdm(ds_valid):\n    img_preds.append(effnet2_tfhub.predict(imgs, batch_size=batch_size))\n    img_labels.append(label.numpy())\n    img_ids.append(imgs_id.numpy())\n    \nimg_preds = np.concatenate([img_pred.argmax(1) for img_pred in img_preds])\nimg_labels = np.concatenate([img_label.argmax(1) for img_label in img_labels])\nimg_ids = np.concatenate([img_id for img_id in img_ids])\n","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:17.294467Z","iopub.execute_input":"2022-03-21T20:54:17.294676Z","iopub.status.idle":"2022-03-21T20:54:37.881655Z","shell.execute_reply.started":"2022-03-21T20:54:17.294633Z","shell.execute_reply":"2022-03-21T20:54:37.880945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_results = pd.DataFrame({'pred': img_preds, \"label\":img_labels, \"id\": img_ids})\nval_results['id'] = val_results['id'].apply(lambda txt: txt.decode())","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:37.88286Z","iopub.execute_input":"2022-03-21T20:54:37.883453Z","iopub.status.idle":"2022-03-21T20:54:37.891625Z","shell.execute_reply.started":"2022-03-21T20:54:37.883409Z","shell.execute_reply":"2022-03-21T20:54:37.890952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_results.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:37.892848Z","iopub.execute_input":"2022-03-21T20:54:37.893372Z","iopub.status.idle":"2022-03-21T20:54:37.906173Z","shell.execute_reply.started":"2022-03-21T20:54:37.893332Z","shell.execute_reply":"2022-03-21T20:54:37.905274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# III. Analysis of low-performant classes","metadata":{}},{"cell_type":"code","source":"worst_classes = pd.DataFrame({'class':['globe-flower', 'clematis', 'canterbury bells', 'mexican petunia',\n                'black-eyed susan', 'peruvian lily']})","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:37.907784Z","iopub.execute_input":"2022-03-21T20:54:37.908118Z","iopub.status.idle":"2022-03-21T20:54:37.914548Z","shell.execute_reply.started":"2022-03-21T20:54:37.908081Z","shell.execute_reply":"2022-03-21T20:54:37.913885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_names_mapping = {value:key for key, value in  enumerate(class_names)}","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:37.916502Z","iopub.execute_input":"2022-03-21T20:54:37.917703Z","iopub.status.idle":"2022-03-21T20:54:37.922021Z","shell.execute_reply.started":"2022-03-21T20:54:37.917635Z","shell.execute_reply":"2022-03-21T20:54:37.921257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"worst_classes['idx'] = worst_classes['class'].map(class_names_mapping)\nworst_classes","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:37.92335Z","iopub.execute_input":"2022-03-21T20:54:37.923662Z","iopub.status.idle":"2022-03-21T20:54:37.937326Z","shell.execute_reply.started":"2022-03-21T20:54:37.923628Z","shell.execute_reply":"2022-03-21T20:54:37.936575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conf_matrix = confusion_matrix(val_results['label'], val_results['pred'])","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:37.938593Z","iopub.execute_input":"2022-03-21T20:54:37.939037Z","iopub.status.idle":"2022-03-21T20:54:37.95092Z","shell.execute_reply.started":"2022-03-21T20:54:37.939003Z","shell.execute_reply":"2022-03-21T20:54:37.950297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_results_classes = val_results[(val_results['pred'].isin(worst_classes['idx'])) | (val_results['label'].isin(worst_classes['idx']))]\nval_results_classes.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:37.95209Z","iopub.execute_input":"2022-03-21T20:54:37.952353Z","iopub.status.idle":"2022-03-21T20:54:37.963276Z","shell.execute_reply.started":"2022-03-21T20:54:37.952319Z","shell.execute_reply":"2022-03-21T20:54:37.962329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_results_classes.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:37.96452Z","iopub.execute_input":"2022-03-21T20:54:37.96497Z","iopub.status.idle":"2022-03-21T20:54:37.973537Z","shell.execute_reply.started":"2022-03-21T20:54:37.964935Z","shell.execute_reply":"2022-03-21T20:54:37.972882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# IIIa). globe-flower","metadata":{}},{"cell_type":"code","source":"class_name = 'globe-flower'","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:37.974644Z","iopub.execute_input":"2022-03-21T20:54:37.975232Z","iopub.status.idle":"2022-03-21T20:54:37.981964Z","shell.execute_reply.started":"2022-03-21T20:54:37.975196Z","shell.execute_reply":"2022-03-21T20:54:37.981206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_results_class = val_results[(val_results['pred'] == class_names_mapping[class_name]) | (val_results['label'] == class_names_mapping[class_name])].copy()\n\nclass_names_mapping_inv = {class_names_mapping[name]:name for name in class_names_mapping}\nfor el in ['pred', 'label']:\n    val_results_class.loc[:, f\"{el}_class\"] = val_results_class[el].map(class_names_mapping_inv)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:37.984522Z","iopub.execute_input":"2022-03-21T20:54:37.984703Z","iopub.status.idle":"2022-03-21T20:54:37.996188Z","shell.execute_reply.started":"2022-03-21T20:54:37.98468Z","shell.execute_reply":"2022-03-21T20:54:37.995404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_results_class","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:37.997434Z","iopub.execute_input":"2022-03-21T20:54:37.998229Z","iopub.status.idle":"2022-03-21T20:54:38.011335Z","shell.execute_reply.started":"2022-03-21T20:54:37.998193Z","shell.execute_reply":"2022-03-21T20:54:38.010629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_root = \"../input/tpu-getting-started\"\n\ndata_path = data_root + '/tfrecords-jpeg-224x224'\nval_224 = tf.io.gfile.glob(data_path + '/val/*.tfrec')\ntrain_224 = tf.io.gfile.glob(data_path + '/train/*.tfrec')\n\ndisplay_batch_by_class(val_224, name = class_name, top_n= 10)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:38.012673Z","iopub.execute_input":"2022-03-21T20:54:38.013184Z","iopub.status.idle":"2022-03-21T20:54:44.759066Z","shell.execute_reply.started":"2022-03-21T20:54:38.013148Z","shell.execute_reply":"2022-03-21T20:54:44.756176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vis_imgs = val_results_class.loc[val_results_class.id.isin(['ed3a59a35', '4a6f8b3ad'])]\nvis_imgs","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:44.760084Z","iopub.execute_input":"2022-03-21T20:54:44.76029Z","iopub.status.idle":"2022-03-21T20:54:44.775929Z","shell.execute_reply.started":"2022-03-21T20:54:44.760262Z","shell.execute_reply":"2022-03-21T20:54:44.775347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_images_by_ids(image_ids_search):\n    ds_train, ds_valid, ds_test = get_datasets(BATCH_SIZE=batch_size, IMAGE_SIZE=(image_size, image_size), \n                                               RESIZE=None, tpu=False, with_id=True)\n    \n    imgs_found = []\n    imgage_ids_found = []\n    labels_found = []\n    for imgs, labels, imgs_id in tqdm(ds_valid):\n        for img, img_id, label in zip(imgs, imgs_id, labels) :\n            if img_id in image_ids_search:\n                imgage_ids_found.append(img_id)\n                imgs_found.append(img)\n                labels_found.append(tf.argmax(label))\n                \n    return (tf.stack(imgs_found, 0), tf.cast(tf.concat(labels_found, 0), tf.int64)), imgage_ids_found","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:44.777149Z","iopub.execute_input":"2022-03-21T20:54:44.777618Z","iopub.status.idle":"2022-03-21T20:54:44.785714Z","shell.execute_reply.started":"2022-03-21T20:54:44.777581Z","shell.execute_reply":"2022-03-21T20:54:44.784962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_found,  imgage_ids_found= get_images_by_ids(vis_imgs['id'].values)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:44.787569Z","iopub.execute_input":"2022-03-21T20:54:44.788262Z","iopub.status.idle":"2022-03-21T20:54:52.123212Z","shell.execute_reply.started":"2022-03-21T20:54:44.788167Z","shell.execute_reply":"2022-03-21T20:54:52.122545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_batch_of_images(batch_found, predictions=vis_imgs['pred'].values, FIGSIZE=16, image_ids= vis_imgs['id'].values)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:52.148456Z","iopub.execute_input":"2022-03-21T20:54:52.148886Z","iopub.status.idle":"2022-03-21T20:54:52.907443Z","shell.execute_reply.started":"2022-03-21T20:54:52.148821Z","shell.execute_reply":"2022-03-21T20:54:52.906464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> * ed3a59a35 image: Flower shot from the side, and flower seem not to have opened yet. No such type of image exists in the val set. But the training set?\n> * 4a6f8b3ad image: the flower seems close to the other globe-flower flowers, in terms of flower and stem leaves. Is buttercup very similar?","metadata":{}},{"cell_type":"code","source":"display_batch_by_class(train_224, name = class_name, top_n= 10)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:54:52.908784Z","iopub.execute_input":"2022-03-21T20:54:52.909065Z","iopub.status.idle":"2022-03-21T20:55:19.525094Z","shell.execute_reply.started":"2022-03-21T20:54:52.90903Z","shell.execute_reply":"2022-03-21T20:55:19.52442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> ed3a59a35 image: Training set does not include does not include such an image. On what is the network focusing on?","metadata":{}},{"cell_type":"code","source":"display_batch_by_class(train_224, name = \"lotus\", top_n= 25)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T20:58:40.86732Z","iopub.execute_input":"2022-03-21T20:58:40.867629Z","iopub.status.idle":"2022-03-21T20:59:03.280463Z","shell.execute_reply.started":"2022-03-21T20:58:40.867596Z","shell.execute_reply":"2022-03-21T20:59:03.279881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Given the form of hte flower in ed3a59a35 image with some of the lotus flowers, it is reasonable to assume it belongs to the class ","metadata":{}},{"cell_type":"code","source":"display_batch_by_class(train_224, name = \"buttercup\", top_n= 25)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T21:02:25.662063Z","iopub.execute_input":"2022-03-21T21:02:25.662598Z","iopub.status.idle":"2022-03-21T21:02:48.94415Z","shell.execute_reply.started":"2022-03-21T21:02:25.662562Z","shell.execute_reply":"2022-03-21T21:02:48.943372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> In its closed flower-closed form, buttercup flowers resemble the flower in image 4a6f8b3ad.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}