{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Analysis of Model flowerclass-efficientnetv2-2 2: with XAI Anchors method\n\n### Goals\n\n* Apply Anchors method to explain decisions leading to model errors in `flowerclass-efficientnetv2-2-analysis2-imgvis` notebook\n* Leverage the `alibi` package which implements anchors for image applications\n\n\nNote: Implementation based on the anchors paper by Ribeiro et al 2018.","metadata":{}},{"cell_type":"code","source":"pip install alibi","metadata":{"execution":{"iopub.status.busy":"2022-03-28T12:37:57.116047Z","iopub.execute_input":"2022-03-28T12:37:57.116844Z","iopub.status.idle":"2022-03-28T12:38:38.562014Z","shell.execute_reply.started":"2022-03-28T12:37:57.116725Z","shell.execute_reply":"2022-03-28T12:38:38.561163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math, re, os\nimport numpy as np\nimport tensorflow as tf\n\ngpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n  try:\n    # Currently, memory growth needs to be the same across GPUs\n    for gpu in gpus:\n      tf.config.experimental.set_memory_growth(gpu, True)\n    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n  except RuntimeError as e:\n    # Memory growth must be set before GPUs have been initialized\n    print(e)\n\n\nimport tensorflow_hub as hub\n\nfrom flowerclass_read_tf_ds import get_validation_dataset\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-03-28T12:40:52.179294Z","iopub.execute_input":"2022-03-28T12:40:52.179594Z","iopub.status.idle":"2022-03-28T12:40:52.188577Z","shell.execute_reply.started":"2022-03-28T12:40:52.179555Z","shell.execute_reply":"2022-03-28T12:40:52.187814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# I. Data prep, model Loading and Predictions with EfficientNetV2","metadata":{}},{"cell_type":"code","source":"image_size = 224\nbatch_size = 1","metadata":{"execution":{"iopub.status.busy":"2022-03-28T12:40:55.483081Z","iopub.execute_input":"2022-03-28T12:40:55.483338Z","iopub.status.idle":"2022-03-28T12:40:55.488248Z","shell.execute_reply.started":"2022-03-28T12:40:55.483310Z","shell.execute_reply":"2022-03-28T12:40:55.487510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"effnet2_base = \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_s/feature_vector/2\"","metadata":{"execution":{"iopub.status.busy":"2022-03-28T12:40:56.213086Z","iopub.execute_input":"2022-03-28T12:40:56.213340Z","iopub.status.idle":"2022-03-28T12:40:56.217196Z","shell.execute_reply.started":"2022-03-28T12:40:56.213312Z","shell.execute_reply":"2022-03-28T12:40:56.216111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"effnet2_tfhub = tf.keras.Sequential([\n    tf.keras.layers.InputLayer(input_shape=(image_size, image_size,3)),\n    hub.KerasLayer(effnet2_base, trainable=False),\n    tf.keras.layers.Dropout(rate=0.2),\n    tf.keras.layers.Dense(104, activation='softmax')\n])\neffnet2_tfhub.build((None, image_size, image_size,3,))\n\n\neffnet2_tfhub.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-28T12:40:56.613223Z","iopub.execute_input":"2022-03-28T12:40:56.613481Z","iopub.status.idle":"2022-03-28T12:41:12.036049Z","shell.execute_reply.started":"2022-03-28T12:40:56.613452Z","shell.execute_reply":"2022-03-28T12:41:12.035304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_phase = 12\neffnet2_tfhub.load_weights(\"../input/flowerclass-efficientnetv2-2/training/\"+\"cp-\"+f\"{best_phase}\".rjust(4, '0')+\".ckpt\")","metadata":{"execution":{"iopub.status.busy":"2022-03-28T12:41:12.037910Z","iopub.execute_input":"2022-03-28T12:41:12.038159Z","iopub.status.idle":"2022-03-28T12:41:12.812130Z","shell.execute_reply.started":"2022-03-28T12:41:12.038122Z","shell.execute_reply":"2022-03-28T12:41:12.811404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# II. Explaining model decisions","metadata":{}},{"cell_type":"code","source":"from alibi.explainers import AnchorImage","metadata":{"execution":{"iopub.status.busy":"2022-03-28T12:41:12.813671Z","iopub.execute_input":"2022-03-28T12:41:12.814177Z","iopub.status.idle":"2022-03-28T12:41:12.818464Z","shell.execute_reply.started":"2022-03-28T12:41:12.814138Z","shell.execute_reply":"2022-03-28T12:41:12.817808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = \"../input/tpu-getting-started/tfrecords-jpeg-224x224\"\nVALIDATION_FILENAMES = tf.io.gfile.glob(data_path + '/val/*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2022-03-28T12:41:12.821000Z","iopub.execute_input":"2022-03-28T12:41:12.821597Z","iopub.status.idle":"2022-03-28T12:41:12.839778Z","shell.execute_reply.started":"2022-03-28T12:41:12.821561Z","shell.execute_reply":"2022-03-28T12:41:12.839154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_images_by_ids(image_ids_search):\n    \n    ds_valid = get_validation_dataset(VALIDATION_FILENAMES, 1, (image_size, image_size), None, True)\n    \n    imgs_found = []\n    imgage_ids_found = []\n    labels_found = []\n    for imgs, labels, imgs_id in tqdm(ds_valid):\n        for img, img_id, label in zip(imgs, imgs_id, labels) :\n            if img_id in image_ids_search:\n                imgage_ids_found.append(img_id)\n                imgs_found.append(img)\n                labels_found.append(tf.argmax(label))\n                \n    return (tf.stack(imgs_found, 0), tf.cast(tf.concat(labels_found, 0), tf.int64)), imgage_ids_found","metadata":{"execution":{"iopub.status.busy":"2022-03-28T12:41:31.519661Z","iopub.execute_input":"2022-03-28T12:41:31.520442Z","iopub.status.idle":"2022-03-28T12:41:31.528452Z","shell.execute_reply.started":"2022-03-28T12:41:31.520403Z","shell.execute_reply":"2022-03-28T12:41:31.527733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# IIa). globe-flower predictions\n\nHere I dive deeper to understand a prediction for the globe-flower class analyzed in `flowerclass_efficientnetv2_2_analysis2_imgvis.ipynb`.\n","metadata":{}},{"cell_type":"markdown","source":"## FP Image ed3a59a35\n\nThe image for analysis has the id ed3a59a35.","metadata":{}},{"cell_type":"code","source":"image_id_investigate = \"ed3a59a35\"","metadata":{"execution":{"iopub.status.busy":"2022-03-28T12:42:01.820136Z","iopub.execute_input":"2022-03-28T12:42:01.820715Z","iopub.status.idle":"2022-03-28T12:42:01.824062Z","shell.execute_reply.started":"2022-03-28T12:42:01.820674Z","shell.execute_reply":"2022-03-28T12:42:01.823265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_found, imgage_ids_found = get_images_by_ids([image_id_investigate])","metadata":{"execution":{"iopub.status.busy":"2022-03-28T12:42:02.172156Z","iopub.execute_input":"2022-03-28T12:42:02.172838Z","iopub.status.idle":"2022-03-28T12:42:09.462027Z","shell.execute_reply.started":"2022-03-28T12:42:02.172795Z","shell.execute_reply":"2022-03-28T12:42:09.460524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(batch_found[0][0].numpy())","metadata":{"execution":{"iopub.status.busy":"2022-03-28T12:42:09.463837Z","iopub.execute_input":"2022-03-28T12:42:09.464093Z","iopub.status.idle":"2022-03-28T12:42:09.721611Z","shell.execute_reply.started":"2022-03-28T12:42:09.464046Z","shell.execute_reply":"2022-03-28T12:42:09.720940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### Setup for explainer\n\nSetup for explainer. Background pixels not in the anchor have the average value of their superpixel.\n\nParameters:\n* predictor: black box predictor, in our case the keras model\n* image_shape: shape of input image\n* segmentation_fn: function for image segmentation into superpixels from skimage.segmentation. I use Alibi default method 'slic'. note that LIME uses quickshift algorithm|\n* segmentation_kwargs: arguments applied in segmentation function segmentation_fn. I use the default arguments used in the [`alibi` example](https://docs.seldon.io/projects/alibi/en/stable/examples/anchor_image_imagenet.html)\n* images_background: alternative way to calculate background (pixels), by superimposing other images. This was done in the Anchors paper by Ribeiro et al.","metadata":{}},{"cell_type":"code","source":"segmentation_fn = 'slic' \nkwargs = {'n_segments': 15, 'compactness': 20, 'sigma': .5}\nimage_shape = (image_size, image_size, 3)\nexplainer = AnchorImage(predictor= effnet2_tfhub.predict , image_shape=image_shape, segmentation_fn=segmentation_fn, \n                        segmentation_kwargs=kwargs, images_background=None, seed=42)","metadata":{"execution":{"iopub.status.busy":"2022-03-28T12:42:24.918179Z","iopub.execute_input":"2022-03-28T12:42:24.918439Z","iopub.status.idle":"2022-03-28T12:42:32.163067Z","shell.execute_reply.started":"2022-03-28T12:42:24.918411Z","shell.execute_reply":"2022-03-28T12:42:32.162277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Explain our image\n\nIdentify best anchor for image provided. Uses beam search to identify best anchor.\n\nParameters:\n\n* image: image to explain\n* p_sample: Probability for a superpixel to be represented by the average value of its pixels, as a form of background pixels. Choose default of 50% probability.\n* threshold: minimum precision for anchor (variable $\\tau$ in anchors paper), indicating the minimum amount of samples that lead to the same prediction as our image. I choose a high precision of 95%. \n* batch_size: compute 100 samples at once\n* coverage_samples: create 10 000 samples to estimate coverage of a anchor\n\n* Beam Search params:\n    * tau: Tolerance $\\epsilon$ (anchors paper, formula 5)\n    * delta: probability constraint $\\delta$, and beam search parameter. Choose default value of 15% (anchors paper uses 5%)\n    * beam_size:  beam width $B$ of the beam search to identify anchors. choose default of 1, meaning only one best anchor is taken at each step of beam search.\n    * stop_on_first: boolean to decide if stop beam search if probability constraint is satisfied. Default is false.\n    * max_anchor_size: default none, \n    * min_samples_start: use 100 inital samples to start beam search\n    * n_covered_ex: for each anchor store 10 examples where anchors apply sampled during beam search\n    *  verbose: show updates during beam anchor search\n    * verbose_every: show beam search updates every `verbose_every` iterations","metadata":{}},{"cell_type":"code","source":"np.random.seed(0)\nexplanation = explainer.explain(image=batch_found[0][0].numpy(), \n                                p_sample = 0.5,\n                                threshold=.95, \n                                delta = 0.15,\n                                tau=0.25,\n                               batch_size=100,\n                               coverage_samples = 10000,\n                                beam_size = 1,\n                                stop_on_first= False,\n                                max_anchor_size = None,\n                                min_samples_start = 100,\n                                verbose = True,\n                                verbose_every = 1\n                               )","metadata":{"execution":{"iopub.status.busy":"2022-03-28T13:25:59.511856Z","iopub.execute_input":"2022-03-28T13:25:59.512727Z","iopub.status.idle":"2022-03-28T13:26:10.272352Z","shell.execute_reply.started":"2022-03-28T13:25:59.512675Z","shell.execute_reply":"2022-03-28T13:26:10.271589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Display Superpixels created by segmentation algorithm:","metadata":{}},{"cell_type":"code","source":"def plot_colormap_explain(image, explanation):\n    '''plot segments of superpixels colored'''\n\n    heatmap = explanation.segments \n\n    fig, axes = plt.subplots(1, 2, figsize=(10,4))\n\n    axes[0].imshow(image)\n\n    img = axes[1].imshow(heatmap, cmap = 'RdBu', vmin  = -heatmap.max(), vmax = heatmap.max())\n    _ = plt.colorbar(img, ax=axes[1])\n    \nplot_colormap_explain(image=batch_found[0][0].numpy(), explanation=explanation)","metadata":{"execution":{"iopub.status.busy":"2022-03-28T13:29:00.237404Z","iopub.execute_input":"2022-03-28T13:29:00.238027Z","iopub.status.idle":"2022-03-28T13:29:00.694976Z","shell.execute_reply.started":"2022-03-28T13:29:00.237977Z","shell.execute_reply":"2022-03-28T13:29:00.693236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Display best anchor which consists of multiple superpixels:","metadata":{}},{"cell_type":"code","source":"plt.imshow(explanation.anchor)","metadata":{"execution":{"iopub.status.busy":"2022-03-28T13:25:02.801032Z","iopub.execute_input":"2022-03-28T13:25:02.801326Z","iopub.status.idle":"2022-03-28T13:25:03.007379Z","shell.execute_reply.started":"2022-03-28T13:25:02.801290Z","shell.execute_reply":"2022-03-28T13:25:03.006708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}