{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Analysis of Model flowerclass-efficientnetv2-2 2: with XAI SHAP method\n\n### Goals\n\n* Apply SHAP to explain decisions leading to model errors in `flowerclass-efficientnetv2-2-analysis2-imgvis` notebook\n* Focus on local explanation of image, hence use  Kernel SHAP as we do not need to calculate explanations for many instances for a global interpretations\n\nNote:\n\nAdapted from https://shap-lrjball.readthedocs.io/en/latest/example_notebooks/kernel_explainer/ImageNet%20VGG16%20Model%20with%20Keras.html","metadata":{}},{"cell_type":"code","source":"import math, re, os\nimport numpy as np\nimport tensorflow as tf\n\ngpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n  try:\n    # Currently, memory growth needs to be the same across GPUs\n    for gpu in gpus:\n      tf.config.experimental.set_memory_growth(gpu, True)\n    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n  except RuntimeError as e:\n    # Memory growth must be set before GPUs have been initialized\n    print(e)\n\n\nimport tensorflow_hub as hub\n\nfrom flowerclass_read_tf_ds import get_validation_dataset\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-03-29T12:26:47.946342Z","iopub.execute_input":"2022-03-29T12:26:47.947288Z","iopub.status.idle":"2022-03-29T12:26:55.755031Z","shell.execute_reply.started":"2022-03-29T12:26:47.947188Z","shell.execute_reply":"2022-03-29T12:26:55.753608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.test.gpu_device_name()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T12:26:55.756577Z","iopub.execute_input":"2022-03-29T12:26:55.756928Z","iopub.status.idle":"2022-03-29T12:26:55.775200Z","shell.execute_reply.started":"2022-03-29T12:26:55.756891Z","shell.execute_reply":"2022-03-29T12:26:55.774523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# I. Data Loading","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"image_size = 224\nbatch_size = 64","metadata":{"execution":{"iopub.status.busy":"2022-03-29T12:26:59.543270Z","iopub.execute_input":"2022-03-29T12:26:59.543540Z","iopub.status.idle":"2022-03-29T12:26:59.547170Z","shell.execute_reply.started":"2022-03-29T12:26:59.543510Z","shell.execute_reply":"2022-03-29T12:26:59.546475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_names = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102\nlen(class_names)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T13:43:16.428896Z","iopub.execute_input":"2022-03-29T13:43:16.429167Z","iopub.status.idle":"2022-03-29T13:43:16.443073Z","shell.execute_reply.started":"2022-03-29T13:43:16.429138Z","shell.execute_reply":"2022-03-29T13:43:16.442373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_names_mapping = {k:classname for k, classname in enumerate(class_names)}","metadata":{"execution":{"iopub.status.busy":"2022-03-29T13:45:34.641253Z","iopub.execute_input":"2022-03-29T13:45:34.641522Z","iopub.status.idle":"2022-03-29T13:45:34.645406Z","shell.execute_reply.started":"2022-03-29T13:45:34.641493Z","shell.execute_reply":"2022-03-29T13:45:34.644727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# II. Model Loading and Predictions: EfficientNetV2","metadata":{}},{"cell_type":"code","source":"effnet2_base = \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_s/feature_vector/2\"","metadata":{"execution":{"iopub.status.busy":"2022-03-29T12:27:00.210599Z","iopub.execute_input":"2022-03-29T12:27:00.211244Z","iopub.status.idle":"2022-03-29T12:27:00.215067Z","shell.execute_reply.started":"2022-03-29T12:27:00.211206Z","shell.execute_reply":"2022-03-29T12:27:00.214191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    effnet2_tfhub = tf.keras.Sequential([\n    # Explicitly define the input shape so the model can be properly\n    # loaded by the TFLiteConverter\n    tf.keras.layers.InputLayer(input_shape=(image_size, image_size,3)),\n    hub.KerasLayer(effnet2_base, trainable=False),\n    tf.keras.layers.Dropout(rate=0.2),\n    tf.keras.layers.Dense(104, activation='softmax')\n])\neffnet2_tfhub.build((None, image_size, image_size,3,)) #This is to be used for subclassed models, which do not know at instantiation time what their inputs look like.\n\n\neffnet2_tfhub.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T12:27:01.679277Z","iopub.execute_input":"2022-03-29T12:27:01.682220Z","iopub.status.idle":"2022-03-29T12:27:14.365688Z","shell.execute_reply.started":"2022-03-29T12:27:01.681561Z","shell.execute_reply":"2022-03-29T12:27:14.364943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_phase = 12\neffnet2_tfhub.load_weights(\"../input/flowerclass-efficientnetv2-2/training/\"+\"cp-\"+f\"{best_phase}\".rjust(4, '0')+\".ckpt\")","metadata":{"execution":{"iopub.status.busy":"2022-03-29T12:27:14.367497Z","iopub.execute_input":"2022-03-29T12:27:14.367786Z","iopub.status.idle":"2022-03-29T12:27:16.099290Z","shell.execute_reply.started":"2022-03-29T12:27:14.367751Z","shell.execute_reply":"2022-03-29T12:27:16.098526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# II. Explaining model decisions","metadata":{}},{"cell_type":"code","source":"import shap\nfrom skimage.segmentation import slic\n# make a color map\nfrom matplotlib.colors import LinearSegmentedColormap","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:08:38.467527Z","iopub.execute_input":"2022-03-29T14:08:38.468215Z","iopub.status.idle":"2022-03-29T14:08:38.472362Z","shell.execute_reply.started":"2022-03-29T14:08:38.468178Z","shell.execute_reply":"2022-03-29T14:08:38.471480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = \"../input/tpu-getting-started/tfrecords-jpeg-224x224\"\nVALIDATION_FILENAMES = tf.io.gfile.glob(data_path + '/val/*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T12:27:18.485627Z","iopub.execute_input":"2022-03-29T12:27:18.485918Z","iopub.status.idle":"2022-03-29T12:27:18.496717Z","shell.execute_reply.started":"2022-03-29T12:27:18.485884Z","shell.execute_reply":"2022-03-29T12:27:18.495902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_images_by_ids(image_ids_search):\n    \n    ds_valid = get_validation_dataset(VALIDATION_FILENAMES, 1, (image_size, image_size), None, True)\n    \n    imgs_found = []\n    imgage_ids_found = []\n    labels_found = []\n    for imgs, labels, imgs_id in tqdm(ds_valid):\n        for img, img_id, label in zip(imgs, imgs_id, labels) :\n            if img_id in image_ids_search:\n                imgage_ids_found.append(img_id)\n                imgs_found.append(img)\n                labels_found.append(tf.argmax(label))\n                \n    return (tf.stack(imgs_found, 0), tf.cast(tf.concat(labels_found, 0), tf.int64)), imgage_ids_found","metadata":{"execution":{"iopub.status.busy":"2022-03-29T12:27:18.497554Z","iopub.execute_input":"2022-03-29T12:27:18.497730Z","iopub.status.idle":"2022-03-29T12:27:18.505247Z","shell.execute_reply.started":"2022-03-29T12:27:18.497708Z","shell.execute_reply":"2022-03-29T12:27:18.504638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# IIa). globe-flower predictions\n\nHere I dive deeper to understand a prediction for the globe-flower class analyzed in flowerclass_efficientnetv2_2_analysis2_imgvis.ipynb.","metadata":{}},{"cell_type":"markdown","source":"## Image ed3a59a35\n\nThe image for analysis has the id ed3a59a35.\n\n","metadata":{}},{"cell_type":"code","source":"image_id_investigate = \"ed3a59a35\"","metadata":{"execution":{"iopub.status.busy":"2022-03-29T12:27:18.506589Z","iopub.execute_input":"2022-03-29T12:27:18.507158Z","iopub.status.idle":"2022-03-29T12:27:18.516700Z","shell.execute_reply.started":"2022-03-29T12:27:18.507100Z","shell.execute_reply":"2022-03-29T12:27:18.515974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_found, imgage_ids_found = get_images_by_ids([image_id_investigate])","metadata":{"execution":{"iopub.status.busy":"2022-03-29T12:27:18.517824Z","iopub.execute_input":"2022-03-29T12:27:18.518138Z","iopub.status.idle":"2022-03-29T12:27:24.635824Z","shell.execute_reply.started":"2022-03-29T12:27:18.518103Z","shell.execute_reply":"2022-03-29T12:27:24.634523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(batch_found[0][0].numpy())","metadata":{"execution":{"iopub.status.busy":"2022-03-29T12:27:24.637280Z","iopub.execute_input":"2022-03-29T12:27:24.637514Z","iopub.status.idle":"2022-03-29T12:27:24.893665Z","shell.execute_reply.started":"2022-03-29T12:27:24.637482Z","shell.execute_reply":"2022-03-29T12:27:24.892981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = batch_found[0][0].numpy()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T12:27:24.894615Z","iopub.execute_input":"2022-03-29T12:27:24.894869Z","iopub.status.idle":"2022-03-29T12:27:24.901404Z","shell.execute_reply.started":"2022-03-29T12:27:24.894830Z","shell.execute_reply":"2022-03-29T12:27:24.900622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-29T12:27:25.503926Z","iopub.execute_input":"2022-03-29T12:27:25.504121Z","iopub.status.idle":"2022-03-29T12:27:25.510102Z","shell.execute_reply.started":"2022-03-29T12:27:25.504097Z","shell.execute_reply":"2022-03-29T12:27:25.509462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Segment the image into superpixels","metadata":{}},{"cell_type":"code","source":"# as in the anchor method I use slice and the same parameter for segmentation. Note for LIME we used their default quickshift algorithm\nkwargs = {'n_segments': 15, 'compactness': 20, 'sigma': .5}\nsegments_slic = slic(img,  **kwargs)\nsegments_slic.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:02:57.358042Z","iopub.execute_input":"2022-03-29T14:02:57.358292Z","iopub.status.idle":"2022-03-29T14:02:57.416742Z","shell.execute_reply.started":"2022-03-29T14:02:57.358266Z","shell.execute_reply":"2022-03-29T14:02:57.415930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define a function that depends on a binary mask representing if an image region is hidden\ndef mask_image(zs, segmentation, image, background=None):\n    '''\n    zs: matrix of perturbed examples of the form (nsamples, 50)\n    segmentation: superpixel segmentation slices\n    image: original image to mask based on superpixels\n    background: contains background values for masking\n    '''\n    print('shape ', zs.shape)\n    if background is None:\n        background = image.mean((0,1)) # array of length 3, with mean value for each of the 3 channels, used as background\n    out = np.zeros((zs.shape[0], image.shape[0], image.shape[1], image.shape[2]))\n    for i in range(zs.shape[0]):\n        out[i,:,:,:] = image\n        # go over the 50 superpixels \n        for j in range(zs.shape[1]):\n            if zs[i,j] == 0:\n                out[i][segmentation == j,:] = background\n    return out\n\ndef f(z):\n    return effnet2_tfhub.predict(mask_image(z, segments_slic, img, 255)) # masking/background pixel value 255 is white for all channels","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:02:59.179463Z","iopub.execute_input":"2022-03-29T14:02:59.179731Z","iopub.status.idle":"2022-03-29T14:02:59.187232Z","shell.execute_reply.started":"2022-03-29T14:02:59.179701Z","shell.execute_reply":"2022-03-29T14:02:59.186525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Parameters:\n\n* model: black-box model to explain\n* data: representative samples from our (train) dataset to help define missingness for all our 50 features/superpixels. As we aim to represent missingness through black background for all superpixels we just need one \"sample\" which consists of a black  pixel for each superpixel (1, 50).\n* link: feature value to model output connection. Not needed in our case, hence default identity is suitable","metadata":{}},{"cell_type":"code","source":"# use Kernel SHAP to explain the network's predictions\nexplainer = shap.KernelExplainer(model=f, data=np.zeros((1,50)))","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:03:02.129700Z","iopub.execute_input":"2022-03-29T14:03:02.130292Z","iopub.status.idle":"2022-03-29T14:03:02.208778Z","shell.execute_reply.started":"2022-03-29T14:03:02.130252Z","shell.execute_reply":"2022-03-29T14:03:02.207977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Parameters:\n\n* X: sample to explain. In our special case of image explanations we use only sample of ones which gets perturbed into multiple samples by SHAP and then processed via ` mask_image` in the `predict` function into perturbed versions of the original image.\n* nsamples: We perturbe our representative sample (just the black image) ` nsample` number of times and re-evaluate our model on these. The perturbations do not affect the image?\n* l1_reg:   l1 regularization to use for feature selection of our local models. The auto option currently uses \"aic\" when less that 20% of the possible sample\n    space is enumerated, otherwise it uses no regularization.","metadata":{}},{"cell_type":"code","source":"shap_values = explainer.shap_values(X=np.ones((1,50)), nsamples=1000)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:03:02.881790Z","iopub.execute_input":"2022-03-29T14:03:02.882279Z","iopub.status.idle":"2022-03-29T14:03:10.884474Z","shell.execute_reply.started":"2022-03-29T14:03:02.882244Z","shell.execute_reply":"2022-03-29T14:03:10.883719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For each of our flower classes, we have a matrix of shap values:","metadata":{}},{"cell_type":"code","source":"len(shap_values)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:03:10.886232Z","iopub.execute_input":"2022-03-29T14:03:10.886649Z","iopub.status.idle":"2022-03-29T14:03:10.892438Z","shell.execute_reply.started":"2022-03-29T14:03:10.886609Z","shell.execute_reply":"2022-03-29T14:03:10.891714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap_values[0].shape","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:03:10.893714Z","iopub.execute_input":"2022-03-29T14:03:10.894128Z","iopub.status.idle":"2022-03-29T14:03:10.902429Z","shell.execute_reply.started":"2022-03-29T14:03:10.894079Z","shell.execute_reply":"2022-03-29T14:03:10.901705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.expand_dims(img.copy(), axis=0).shape","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:03:10.904507Z","iopub.execute_input":"2022-03-29T14:03:10.905603Z","iopub.status.idle":"2022-03-29T14:03:10.916286Z","shell.execute_reply.started":"2022-03-29T14:03:10.905559Z","shell.execute_reply":"2022-03-29T14:03:10.915592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get the top predictions from the model\npreds = effnet2_tfhub.predict(np.expand_dims(img.copy(), axis=0))\ntop_preds = np.argsort(-preds)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:03:10.917642Z","iopub.execute_input":"2022-03-29T14:03:10.920110Z","iopub.status.idle":"2022-03-29T14:03:11.018819Z","shell.execute_reply.started":"2022-03-29T14:03:10.920069Z","shell.execute_reply":"2022-03-29T14:03:11.018089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds.shape, preds[0, :5]","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:03:11.020002Z","iopub.execute_input":"2022-03-29T14:03:11.020260Z","iopub.status.idle":"2022-03-29T14:03:11.028579Z","shell.execute_reply.started":"2022-03-29T14:03:11.020225Z","shell.execute_reply":"2022-03-29T14:03:11.027707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_preds","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:03:11.029942Z","iopub.execute_input":"2022-03-29T14:03:11.030256Z","iopub.status.idle":"2022-03-29T14:03:11.036431Z","shell.execute_reply.started":"2022-03-29T14:03:11.030220Z","shell.execute_reply":"2022-03-29T14:03:11.035718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_names_mapping[15]","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:09:14.005916Z","iopub.execute_input":"2022-03-29T14:09:14.006547Z","iopub.status.idle":"2022-03-29T14:09:14.013342Z","shell.execute_reply.started":"2022-03-29T14:09:14.006504Z","shell.execute_reply":"2022-03-29T14:09:14.012582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create colormap\ncolors = []\nfor l in np.linspace(1,0,100):\n    colors.append((245/255,39/255,87/255,l)) # (red, blue, green, alpha)\nfor l in np.linspace(0,1,100):\n    colors.append((24/255,196/255,93/255,l))  # (red, blue, green, alpha)\ncm = LinearSegmentedColormap.from_list(\"shap\", colors)\ncm","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:03:11.037723Z","iopub.execute_input":"2022-03-29T14:03:11.038530Z","iopub.status.idle":"2022-03-29T14:03:11.054974Z","shell.execute_reply.started":"2022-03-29T14:03:11.038492Z","shell.execute_reply":"2022-03-29T14:03:11.054224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fill_segmentation(values, segmentation):\n    '''fill superpixels with their SHAP values and returns image\n    values: SHAP values\n    segmentation: superpixel segmentations\n    '''\n    out = np.zeros(segmentation.shape)\n    for i in range(len(values)):\n        out[segmentation == i] = values[i]\n    return out","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:03:11.056337Z","iopub.execute_input":"2022-03-29T14:03:11.056579Z","iopub.status.idle":"2022-03-29T14:03:11.061767Z","shell.execute_reply.started":"2022-03-29T14:03:11.056546Z","shell.execute_reply":"2022-03-29T14:03:11.060865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualize the shap values of the superpixels for the top 3 class with the highest class probablity:","metadata":{}},{"cell_type":"code","source":"# class indices \ninds = top_preds[0]\n\n# plot our explanations\nfig, axes = plt.subplots(nrows=1, ncols=4, figsize=(16, 8))\n\n# plot original image\naxes[0].imshow(img)\naxes[0].axis('off')\n\n# max SHAP value of all SHAP values for all classes\nmax_val = np.max([np.max(np.abs(shap_values[i][:,:-1])) for i in range(len(shap_values))])\n\n\nfor i in range(3):\n    m = fill_segmentation(shap_values[inds[i]][0], segments_slic)\n    \n    axes[i+1].set_title(class_names_mapping[inds[i]])\n    \n    axes[i+1].imshow(img, alpha=0.08, cmap='gray', vmin=0, vmax=255) # .convert('LA')\n    \n    im = axes[i+1].imshow(m, cmap=cm, vmin=-max_val, vmax=max_val)\n    \n    axes[i+1].axis('off')\n\ncb = fig.colorbar(im, ax=axes.ravel().tolist(), label=\"SHAP value\", orientation=\"horizontal\", aspect=60)\ncb.outline.set_visible(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-29T14:03:11.258751Z","iopub.execute_input":"2022-03-29T14:03:11.259255Z","iopub.status.idle":"2022-03-29T14:03:11.766213Z","shell.execute_reply.started":"2022-03-29T14:03:11.259220Z","shell.execute_reply":"2022-03-29T14:03:11.765527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Results:\n\n* For the predicted globe-flower class, according to SHAP,the model focus more on the surrounding areas rather than the relevant flower itself. This is in contrast to LIME and the anchor method which both identified the flower as the most relevant part, in alignment with reasoning.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}