{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Model Interpretation for Pretrained ResNet Model","metadata":{"id":"CDbpH8nnBhdx"}},{"cell_type":"markdown","source":"This notebook demonstrates how to apply model interpretability algorithms on pretrained ResNet model using a handpicked image and visualizes the attributions for each pixel by overlaying them on the image.\n\nThe interpretation algorithms that we use in this notebook are `Integrated Gradients` (w/ and w/o noise tunnel),  `GradientShap`, and `Occlusion`. A noise tunnel allows to smoothen the attributions after adding gaussian noise to each input sample.\n  \n  **Note:** Before running this tutorial, please install the torchvision, PIL, and matplotlib packages.","metadata":{"id":"ozcCavo0Bhd0"}},{"cell_type":"code","source":"!pip install captum\n!pip install timm\n!pip install gdown\n!gdown 1dWwapuVx_aK6K2j5Jj1St2F7RTQ9DirG\n!gdown 1g-pijm1Db9cfOxBhHw5h5aqVnCEWjtTL #this is cvc-normal from train set\n!gdown 1CalZo9FPVKDHZ-66pe2CqWoZm94X4j8m #this is cvc-abnormal from train set predicted incorrectly\n!gdown 1mjBY-yXXIzZhaLCAC5dfxBshF68GjCMM #cvc-abnormal train set","metadata":{"execution":{"iopub.status.busy":"2022-08-23T10:22:11.718425Z","iopub.execute_input":"2022-08-23T10:22:11.719262Z","iopub.status.idle":"2022-08-23T10:23:08.816965Z","shell.execute_reply.started":"2022-08-23T10:22:11.719154Z","shell.execute_reply":"2022-08-23T10:23:08.815817Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Collecting captum\n  Downloading captum-0.5.0-py3-none-any.whl (1.4 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.4/1.4 MB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: torch>=1.6 in /opt/conda/lib/python3.7/site-packages (from captum) (1.11.0)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from captum) (1.21.6)\nRequirement already satisfied: matplotlib in /opt/conda/lib/python3.7/site-packages (from captum) (3.5.2)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from torch>=1.6->captum) (4.1.1)\nRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.7/site-packages (from matplotlib->captum) (2.8.2)\nRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.7/site-packages (from matplotlib->captum) (0.11.0)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib->captum) (21.3)\nRequirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib->captum) (9.1.1)\nRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib->captum) (1.4.3)\nRequirement already satisfied: pyparsing>=2.2.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib->captum) (3.0.9)\nRequirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib->captum) (4.33.3)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.7->matplotlib->captum) (1.16.0)\nInstalling collected packages: captum\nSuccessfully installed captum-0.5.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mCollecting timm\n  Downloading timm-0.6.7-py3-none-any.whl (509 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m510.0/510.0 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: torch>=1.4 in /opt/conda/lib/python3.7/site-packages (from timm) (1.11.0)\nRequirement already satisfied: torchvision in /opt/conda/lib/python3.7/site-packages (from timm) (0.12.0)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from torch>=1.4->timm) (4.1.1)\nRequirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from torchvision->timm) (2.28.1)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from torchvision->timm) (1.21.6)\nRequirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/conda/lib/python3.7/site-packages (from torchvision->timm) (9.1.1)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->torchvision->timm) (2022.6.15)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->torchvision->timm) (1.26.9)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->torchvision->timm) (3.3)\nRequirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.7/site-packages (from requests->torchvision->timm) (2.1.0)\nInstalling collected packages: timm\nSuccessfully installed timm-0.6.7\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mCollecting gdown\n  Downloading gdown-4.5.1.tar.gz (14 kB)\n  Installing build dependencies ... \u001b[?25ldone\n\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from gdown) (1.16.0)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from gdown) (3.7.1)\nRequirement already satisfied: beautifulsoup4 in /opt/conda/lib/python3.7/site-packages (from gdown) (4.11.1)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.7/site-packages (from gdown) (4.64.0)\nRequirement already satisfied: requests[socks] in /opt/conda/lib/python3.7/site-packages (from gdown) (2.28.1)\nRequirement already satisfied: soupsieve>1.2 in /opt/conda/lib/python3.7/site-packages (from beautifulsoup4->gdown) (2.3.1)\nRequirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.7/site-packages (from requests[socks]->gdown) (2.1.0)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests[socks]->gdown) (2022.6.15)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests[socks]->gdown) (3.3)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests[socks]->gdown) (1.26.9)\nRequirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /opt/conda/lib/python3.7/site-packages (from requests[socks]->gdown) (1.7.1)\nBuilding wheels for collected packages: gdown\n  Building wheel for gdown (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for gdown: filename=gdown-4.5.1-py3-none-any.whl size=14933 sha256=c03811ff921e3dca52d8b9c100e64106e883768151b660d6336f0a2ab26a5acf\n  Stored in directory: /root/.cache/pip/wheels/3d/ec/b0/a96d1d126183f98570a785e6bf8789fca559853a9260e928e1\nSuccessfully built gdown\nInstalling collected packages: gdown\nSuccessfully installed gdown-4.5.1\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mDownloading...\nFrom: https://drive.google.com/uc?id=1dWwapuVx_aK6K2j5Jj1St2F7RTQ9DirG\nTo: /kaggle/working/resnext50_32x4d_fold2_best-21aug-7hrs.pth\n100%|███████████████████████████████████████| 92.9M/92.9M [00:00<00:00, 191MB/s]\nDownloading...\nFrom: https://drive.google.com/uc?id=1g-pijm1Db9cfOxBhHw5h5aqVnCEWjtTL\nTo: /kaggle/working/1.2.826.0.1.3680043.8.498.99991581639229040366812735770671130168.jpg\n100%|████████████████████████████████████████| 308k/308k [00:00<00:00, 99.6MB/s]\nDownloading...\nFrom: https://drive.google.com/uc?id=1CalZo9FPVKDHZ-66pe2CqWoZm94X4j8m\nTo: /kaggle/working/1.2.826.0.1.3680043.8.498.70141573962967801644173304809340501120.jpg\n100%|████████████████████████████████████████| 161k/161k [00:00<00:00, 58.6MB/s]\nDownloading...\nFrom: https://drive.google.com/uc?id=1mjBY-yXXIzZhaLCAC5dfxBshF68GjCMM\nTo: /kaggle/working/1.2.826.0.1.3680043.8.498.68286643202323212801283518367144358744.jpg\n100%|█████████████████████████████████████████| 278k/278k [00:00<00:00, 100MB/s]\n","output_type":"stream"}]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom PIL import Image\n\nimport os\nimport json\nimport numpy as np\nfrom matplotlib.colors import LinearSegmentedColormap\nimport pandas as dp\nimport cv2\n\nimport torchvision\nfrom torchvision import models\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom captum.attr import IntegratedGradients\nfrom captum.attr import GradientShap\nfrom captum.attr import Occlusion\nfrom captum.attr import NoiseTunnel\nfrom captum.attr import visualization as viz\nimport timm\n\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose\n    )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport matplotlib.pyplot as plt\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n\nfrom torch.cuda.amp import autocast, GradScaler","metadata":{"id":"ashEs63bBueT","outputId":"9250c4c1-9710-4906-9b83-5fb7b4941346","execution":{"iopub.status.busy":"2022-08-23T10:47:27.248524Z","iopub.execute_input":"2022-08-23T10:47:27.24897Z","iopub.status.idle":"2022-08-23T10:47:27.266504Z","shell.execute_reply.started":"2022-08-23T10:47:27.248927Z","shell.execute_reply":"2022-08-23T10:47:27.265279Z"},"trusted":true},"execution_count":45,"outputs":[]},{"cell_type":"code","source":"import pandas as dp\n%cd jupyter\npath = os.getcwd()\ndirectory = os.listdir()\nprint (path)\nprint (directory)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T05:31:00.906639Z","iopub.execute_input":"2022-08-23T05:31:00.906988Z","iopub.status.idle":"2022-08-23T05:31:00.915524Z","shell.execute_reply.started":"2022-08-23T05:31:00.906958Z","shell.execute_reply":"2022-08-23T05:31:00.914437Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomResNext(nn.Module):\n    def __init__(self, model_name='resnext50_32x4d', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, 11)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n\nmodel = CustomResNext()\nweights = torch.load('working/resnext50_32x4d_fold2_best-21aug-7hrs.pth')\nmodel.load_state_dict(weights, strict=False)\nmodel.eval()\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel.to(device)\n\ndict_cols = {'0': ['n01440764', 'ETT - Abnormal'],\n '1': ['n01443537', 'ETT - Borderline'],\n '2': ['n01484850', 'ETT - Normal'],\n '3': ['n01491361', 'NGT - Abnormal'],\n '4': ['n01494475', 'NGT - Borderline'],\n '5': ['n01496331', 'NGT - Incompletely Imaged'],\n '6': ['n01498041', 'NGT - Normal'],\n '7': ['n01514668', 'CVC - Abnormal'],\n '8': ['n01514859', 'CVC - Borderline'],\n '9': ['n01518878', 'CVC - Normal'],\n '10': ['n01530575', 'Swan Ganz Catheter Present']\n}\n\ntarget_cols=['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',\n             'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal', \n             'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',\n             'Swan Ganz Catheter Present']","metadata":{"id":"FKfNMHtYMnZ7","execution":{"iopub.status.busy":"2022-08-23T10:23:58.241243Z","iopub.execute_input":"2022-08-23T10:23:58.241834Z","iopub.status.idle":"2022-08-23T10:24:02.370836Z","shell.execute_reply.started":"2022-08-23T10:23:58.2418Z","shell.execute_reply":"2022-08-23T10:24:02.369631Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"#cvc normal\n# img = Image.open('./1.2.826.0.1.3680043.8.498.99991581639229040366812735770671130168.jpg').convert('RGB')\n#cvc abnormal 1\nimg = Image.open('working/1.2.826.0.1.3680043.8.498.70141573962967801644173304809340501120.jpg').convert('RGB')\n#cvc abnormal 2\n# img = Image.open('1.2.826.0.1.3680043.8.498.68286643202323212801283518367144358744.jpg').convert('RGB')\n# transform = transforms.Resize(600)\n# resized_img = transform(img)\n# # transform = transforms.CenterCrop(600)\n# reresized_img = transform(resized_img)\n# reresized_img\n# transform = transforms.ToTensor()\n# tensor = transform(resized_img)\n# transform_normalize = transforms.Normalize(\n#      mean=[0.485, 0.456, 0.406],\n#      std=[0.229, 0.224, 0.225]\n#  )\nnormalize_transform = torchvision.transforms.Compose([\n    torchvision.transforms.Resize(600),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize(mean = (0.485, 0.456, 0.406), \n                                     std = (0.229, 0.224, 0.225))])\n\n# tensor = transforms(img)\ntensor = normalize_transform(img)\ninput = tensor.unsqueeze(0)\nprint(input.shape)\n\n","metadata":{"id":"5ddSbla2Bhd6","outputId":"b614cf1c-fae3-4e07-8d14-1a6d6081e6b0","execution":{"iopub.status.busy":"2022-08-23T11:57:37.495623Z","iopub.execute_input":"2022-08-23T11:57:37.495992Z","iopub.status.idle":"2022-08-23T11:57:37.581357Z","shell.execute_reply.started":"2022-08-23T11:57:37.49596Z","shell.execute_reply":"2022-08-23T11:57:37.580213Z"},"trusted":true},"execution_count":79,"outputs":[{"name":"stdout","text":"torch.Size([1, 3, 600, 732])\n","output_type":"stream"}]},{"cell_type":"code","source":"TRAIN_PATH = 'input/ranzcr-clip-catheter-line-classification/train'\nclass TrainDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.labels = df[target_cols].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TRAIN_PATH}/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).float()\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2022-08-23T10:24:06.687476Z","iopub.execute_input":"2022-08-23T10:24:06.687995Z","iopub.status.idle":"2022-08-23T10:24:06.700305Z","shell.execute_reply.started":"2022-08-23T10:24:06.687946Z","shell.execute_reply":"2022-08-23T10:24:06.697466Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"def get_transforms(*, data):\n    \n    if data == 'train':\n        return Compose([\n            #Resize(CFG.size, CFG.size),\n            RandomResizedCrop(600, 600, scale=(0.85, 1.0)),\n            HorizontalFlip(p=0.5),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n\n    elif data == 'valid':\n        return Compose([\n            Resize(600, 600),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])","metadata":{"execution":{"iopub.status.busy":"2022-08-23T10:24:10.511355Z","iopub.execute_input":"2022-08-23T10:24:10.512488Z","iopub.status.idle":"2022-08-23T10:24:10.520783Z","shell.execute_reply.started":"2022-08-23T10:24:10.51244Z","shell.execute_reply":"2022-08-23T10:24:10.519857Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"normalize_transform = torchvision.transforms.Compose([\n    torchvision.transforms.Resize(600),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize(mean = (0.485, 0.456, 0.406), \n                                     std = (0.229, 0.224, 0.225))])\n    \n#Generating data loaders from the corresponding datasets\nbatch_size = 32\n#train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size)\n# valid_dataset = torch.utils.data.DataLoader('', batch_size=batch_size)\n\ntrain = pd.read_csv('input/ranzcr-clip-catheter-line-classification/train.csv')\ntrain_dataset = TrainDataset(train, transform=get_transforms(data='valid'))\n# valid_dataset = TrainDataset('../input/ranzcr-clip-catheter-line-classification/train.csv', transform=normalize_transform)\nvalid_loader = DataLoader(train_dataset, \n                          batch_size=batch_size * 1, \n                          shuffle=False, \n                          num_workers=2, pin_memory=True, drop_last=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-23T11:06:47.344327Z","iopub.execute_input":"2022-08-23T11:06:47.344968Z","iopub.status.idle":"2022-08-23T11:06:47.415184Z","shell.execute_reply.started":"2022-08-23T11:06:47.344933Z","shell.execute_reply":"2022-08-23T11:06:47.414209Z"},"trusted":true},"execution_count":63,"outputs":[]},{"cell_type":"code","source":"def get_score(y_true, y_pred):\n    scores = []\n    for i in range(y_true.shape[1]):\n        score = roc_auc_score(y_true[:,i], y_pred[:,i])\n        scores.append(score)\n    avg_score = np.mean(scores)\n    return avg_score, scores","metadata":{"execution":{"iopub.status.busy":"2022-08-23T10:24:17.229876Z","iopub.execute_input":"2022-08-23T10:24:17.230374Z","iopub.status.idle":"2022-08-23T10:24:17.23872Z","shell.execute_reply.started":"2022-08-23T10:24:17.230333Z","shell.execute_reply":"2022-08-23T10:24:17.237652Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"    criterion = nn.BCEWithLogitsLoss()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T10:24:17.985041Z","iopub.execute_input":"2022-08-23T10:24:17.985484Z","iopub.status.idle":"2022-08-23T10:24:17.993986Z","shell.execute_reply.started":"2022-08-23T10:24:17.985448Z","shell.execute_reply":"2022-08-23T10:24:17.993036Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"sum(train['ETT - Abnormal'] == 1)\nlabels = []\ncount = []\nfor c in target_cols:\n    print(c + \": \" + str(sum(train[c] == 1)))\n    labels.append(c)\n    count.append(sum(train[c] == 1))\n\n\nplt.bar(labels, count)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T10:51:09.645696Z","iopub.execute_input":"2022-08-23T10:51:09.646059Z","iopub.status.idle":"2022-08-23T10:51:09.978933Z","shell.execute_reply.started":"2022-08-23T10:51:09.646027Z","shell.execute_reply":"2022-08-23T10:51:09.977992Z"},"trusted":true},"execution_count":50,"outputs":[{"name":"stdout","text":"ETT - Abnormal: 79\nETT - Borderline: 1138\nETT - Normal: 7240\nNGT - Abnormal: 279\nNGT - Borderline: 529\nNGT - Incompletely Imaged: 2748\nNGT - Normal: 4797\nCVC - Abnormal: 3195\nCVC - Borderline: 8460\nCVC - Normal: 21324\nSwan Ganz Catheter Present: 830\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<Figure size 432x288 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAbAAAAD4CAYAAACNMrOfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAjj0lEQVR4nO3de5xdVX338c+XBAKCSCBjDAkQ1GALaKOkgFXbKBUC9jHYx4dLW4mKRiuoVG0bL0/xRsVW1FIUHpQ0wSIXi5QUAzFEEHwkkgFCQsCQISSSGJIp4ZYQEpL8+sf6HWcznLkwM7ns8H2/Xuc1+6x9WWvtvfb67b3OnnMUEZiZmdXNbju6AGZmZn3hAGZmZrXkAGZmZrXkAGZmZrXkAGZmZrU0eEcXoK+GDRsWo0eP3tHFMDOrlbvuuuu/I6JlR5djINQ2gI0ePZrW1tYdXQwzs1qRtHxHl2GgeAjRzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqqbbfxGFm9mKMnvKTbbLdZee/a5ts13rmOzAzM6slBzAzM6slBzAzM6slBzAzM6slBzAzM6slBzAzM6slBzAzM6slBzAzM6ulHgOYpIMk3SLpfkmLJH0y0/eXNFvSkvw7NNMl6UJJbZIWSHpTZVuTcvklkiZV0o+StDDXuVCStkVlzcxs19GbO7DNwKcj4nDgWOAsSYcDU4A5ETEGmJPvAU4ExuRrMnAxlIAHnAscAxwNnNsIernMhyvrTeh/1czMbFfWYwCLiFURcXdOPw08AIwEJgLTc7HpwMk5PRG4PIq5wH6SRgAnALMjYm1EPA7MBibkvH0jYm5EBHB5ZVtmZmZNvajPwCSNBt4I/AoYHhGrctajwPCcHgk8UlltRaZ1l76iSbqZmVmXeh3AJO0DXAucExFPVeflnVMMcNmalWGypFZJre3t7ds6OzMz24n1KoBJ2p0SvK6IiB9n8uoc/iP/rsn0lcBBldVHZVp36aOapL9ARFwaEeMiYlxLS0tvim5mZruo3jyFKOAy4IGI+GZl1gyg8SThJOD6SvoZ+TTiscCTOdQ4Czhe0tB8eON4YFbOe0rSsZnXGZVtmZmZNdWb3wN7C/A+YKGk+Zn2OeB84BpJZwLLgVNy3kzgJKANeAb4AEBErJX0FWBeLvfliFib0x8DpgF7ATfmy8zMrEs9BrCI+AXQ1f9lHddk+QDO6mJbU4GpTdJbgSN7KouZmVmDv4nDzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqyQHMzMxqqccAJmmqpDWS7qukXS1pfr6WNX6pWdJoSRsq8y6prHOUpIWS2iRdKEmZvr+k2ZKW5N+h26CeZma2i+nNHdg0YEI1ISJOjYixETEWuBb4cWX2Q415EfHRSvrFwIeBMflqbHMKMCcixgBz8r2ZmVm3egxgEXEbsLbZvLyLOgW4srttSBoB7BsRcyMigMuBk3P2RGB6Tk+vpJuZmXWpv5+BvQ1YHRFLKmmHSrpH0s8lvS3TRgIrKsusyDSA4RGxKqcfBYZ3lZmkyZJaJbW2t7f3s+hmZlZn/Q1gp/P8u69VwMER8UbgU8APJe3b243l3Vl0M//SiBgXEeNaWlr6WmYzM9sFDO7ripIGA38OHNVIi4iNwMacvkvSQ8BhwEpgVGX1UZkGsFrSiIhYlUONa/paJjMze+nozx3YnwK/jojfDQ1KapE0KKdfTXlYY2kOET4l6dj83OwM4PpcbQYwKacnVdLNzMy61JvH6K8E7gBeJ2mFpDNz1mm88OGNPwYW5GP1/wF8NCIaD4B8DPg+0AY8BNyY6ecD75S0hBIUz+97dczM7KWixyHEiDi9i/T3N0m7lvJYfbPlW4Ejm6Q/BhzXUznMzMyq/E0cZmZWSw5gZmZWSw5gZmZWSw5gZmZWSw5gZmZWSw5gZmZWSw5gZmZWSw5gZmZWSw5gZmZWSw5gZmZWSw5gZmZWSw5gZmZWSw5gZmZWSw5gZmZWSw5gZmZWS735QcupktZIuq+S9kVJKyXNz9dJlXmfldQmabGkEyrpEzKtTdKUSvqhkn6V6VdL2mMgK2hmZrum3tyBTQMmNEn/VkSMzddMAEmHU36p+Yhc57uSBkkaBHwHOBE4HDg9lwX4em7rtcDjwJmdMzIzM+usxwAWEbcBa3u5vYnAVRGxMSIeBtqAo/PVFhFLI2ITcBUwUZKAdwD/ketPB05+cVUwM7OXov58Bna2pAU5xDg000YCj1SWWZFpXaUfADwREZs7pTclabKkVkmt7e3t/Si6mZnVXV8D2MXAa4CxwCrggoEqUHci4tKIGBcR41paWrZHlmZmtpMa3JeVImJ1Y1rS94Ab8u1K4KDKoqMyjS7SHwP2kzQ478Kqy5uZmXWpT3dgkkZU3r4HaDyhOAM4TdIQSYcCY4A7gXnAmHzicA/Kgx4zIiKAW4D35vqTgOv7UiYzM3tp6fEOTNKVwHhgmKQVwLnAeEljgQCWAR8BiIhFkq4B7gc2A2dFxJbcztnALGAQMDUiFmUWfw9cJemrwD3AZQNVOTMz23X1GMAi4vQmyV0GmYg4DzivSfpMYGaT9KWUpxTNzMx6zd/EYWZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmtdRjAJM0VdIaSfdV0v5Z0q8lLZB0naT9Mn20pA2S5ufrkso6R0laKKlN0oWSlOn7S5otaUn+HboN6mlmZruY3tyBTQMmdEqbDRwZEW8AHgQ+W5n3UESMzddHK+kXAx8GxuSrsc0pwJyIGAPMyfdmZmbd6jGARcRtwNpOaT+NiM35di4wqrttSBoB7BsRcyMigMuBk3P2RGB6Tk+vpJuZmXVpID4D+yBwY+X9oZLukfRzSW/LtJHAisoyKzINYHhErMrpR4HhXWUkabKkVkmt7e3tA1B0MzOrq34FMEmfBzYDV2TSKuDgiHgj8Cngh5L27e328u4supl/aUSMi4hxLS0t/Si5mZnV3eC+rijp/cCfAcdl4CEiNgIbc/ouSQ8BhwEref4w46hMA1gtaURErMqhxjV9LZOZmb109OkOTNIE4O+Ad0fEM5X0FkmDcvrVlIc1luYQ4VOSjs2nD88Ars/VZgCTcnpSJd3MzKxLPd6BSboSGA8Mk7QCOJfy1OEQYHY+DT83nzj8Y+DLkp4DtgIfjYjGAyAfozzRuBflM7PG52bnA9dIOhNYDpwyIDUzM7NdWo8BLCJOb5J8WRfLXgtc28W8VuDIJumPAcf1VA4zM7MqfxOHmZnVkgOYmZnVkgOYmZnVkgOYmZnVkgOYmZnVkgOYmZnVkgOYmZnVkgOYmZnVkgOYmZnVkgOYmZnVkgOYmZnVkgOYmZnVkgOYmZnVkgOYmZnVkgOYmZnVkgOYmZnVUq8CmKSpktZIuq+Str+k2ZKW5N+hmS5JF0pqk7RA0psq60zK5ZdImlRJP0rSwlznQuXPPJuZmXWlt3dg04AJndKmAHMiYgwwJ98DnAiMyddk4GIoAQ84FzgGOBo4txH0cpkPV9brnJeZmdnz9CqARcRtwNpOyROB6Tk9HTi5kn55FHOB/SSNAE4AZkfE2oh4HJgNTMh5+0bE3IgI4PLKtszMzJrqz2dgwyNiVU4/CgzP6ZHAI5XlVmRad+krmqS/gKTJkloltba3t/ej6GZmVncD8hBH3jnFQGyrh3wujYhxETGupaVlW2dnZmY7sf4EsNU5/Ef+XZPpK4GDKsuNyrTu0kc1STczM+tSfwLYDKDxJOEk4PpK+hn5NOKxwJM51DgLOF7S0Hx443hgVs57StKx+fThGZVtmZmZNTW4NwtJuhIYDwyTtILyNOH5wDWSzgSWA6fk4jOBk4A24BngAwARsVbSV4B5udyXI6LxYMjHKE867gXcmC8zM7Mu9SqARcTpXcw6rsmyAZzVxXamAlObpLcCR/amLGZmZuBv4jAzs5pyADMzs1pyADMzs1pyADMzs1pyADMzs1pyADMzs1pyADMzs1pyADMzs1pyADMzs1rq1TdxmJkNpNFTfrLNtr3s/Hdts23bzsV3YGZmVksOYGZmVksOYGZmVksOYGZmVksOYGZmVkt9DmCSXidpfuX1lKRzJH1R0spK+kmVdT4rqU3SYkknVNInZFqbpCn9rZSZme36+vwYfUQsBsYCSBoErASuo/wC87ci4hvV5SUdDpwGHAEcCNws6bCc/R3gncAKYJ6kGRFxf1/LZmZmu76B+j+w44CHImK5pK6WmQhcFREbgYcltQFH57y2iFgKIOmqXNYBzMzMujRQn4GdBlxZeX+2pAWSpkoammkjgUcqy6zItK7SX0DSZEmtklrb29sHqOhmZlZH/Q5gkvYA3g38KJMuBl5DGV5cBVzQ3zwaIuLSiBgXEeNaWloGarNmZlZDAzGEeCJwd0SsBmj8BZD0PeCGfLsSOKiy3qhMo5t06wN/TY+ZvRQMxBDi6VSGDyWNqMx7D3BfTs8ATpM0RNKhwBjgTmAeMEbSoXk3d1oua2Zm1qV+3YFJ2pvy9OBHKsn/JGksEMCyxryIWCTpGsrDGZuBsyJiS27nbGAWMAiYGhGL+lMuMzPb9fUrgEXEeuCATmnv62b584DzmqTPBGb2pyxmZvbS4m/iMDOzWnIAMzOzWnIAMzOzWnIAMzOzWnIAMzOzWnIAMzOzWnIAMzOzWnIAMzOzWnIAMzOzWnIAMzOzWnIAMzOzWnIAMzOzWnIAMzOzWnIAMzOzWnIAMzOzWup3AJO0TNJCSfMltWba/pJmS1qSf4dmuiRdKKlN0gJJb6psZ1Iuv0TSpP6Wy8zMdm0DdQf29ogYGxHj8v0UYE5EjAHm5HuAE4Ex+ZoMXAwl4AHnAscARwPnNoKemZlZM9tqCHEiMD2npwMnV9Ivj2IusJ+kEcAJwOyIWBsRjwOzgQnbqGxmZrYLGIgAFsBPJd0laXKmDY+IVTn9KDA8p0cCj1TWXZFpXaU/j6TJkloltba3tw9A0c3MrK4GD8A23hoRKyW9Epgt6dfVmRERkmIA8iEiLgUuBRg3btyAbNPMzOqp3wEsIlbm3zWSrqN8hrVa0oiIWJVDhGty8ZXAQZXVR2XaSmB8p/Rb+1s2s7oaPeUn22zby85/1zbbttn21K8hREl7S3p5Yxo4HrgPmAE0niScBFyf0zOAM/JpxGOBJ3OocRZwvKSh+fDG8ZlmZmbWVH/vwIYD10lqbOuHEXGTpHnANZLOBJYDp+TyM4GTgDbgGeADABGxVtJXgHm53JcjYm0/y2ZmZruwfgWwiFgK/EGT9MeA45qkB3BWF9uaCkztT3nMzOylw9/EYWZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmteQAZmZmtTQQP2hpZjXn3x+zOvIdmJmZ1ZIDmJmZ1VKfA5ikgyTdIul+SYskfTLTvyhppaT5+Tqpss5nJbVJWizphEr6hExrkzSlf1UyM7OXgv58BrYZ+HRE3C3p5cBdkmbnvG9FxDeqC0s6HDgNOAI4ELhZ0mE5+zvAO4EVwDxJMyLi/n6UzWxAbavPiPz5kFnf9TmARcQqYFVOPy3pAWBkN6tMBK6KiI3Aw5LagKNzXlv+ujOSrsplHcDMzKxLA/IZmKTRwBuBX2XS2ZIWSJoqaWimjQQeqay2ItO6SjczM+tSvwOYpH2Aa4FzIuIp4GLgNcBYyh3aBf3No5LXZEmtklrb29sHarNmZlZD/QpgknanBK8rIuLHABGxOiK2RMRW4Ht0DBOuBA6qrD4q07pKf4GIuDQixkXEuJaWlv4U3czMaq4/TyEKuAx4ICK+WUkfUVnsPcB9OT0DOE3SEEmHAmOAO4F5wBhJh0rag/Kgx4y+lsvMzF4a+vMU4luA9wELJc3PtM8Bp0saCwSwDPgIQEQsknQN5eGMzcBZEbEFQNLZwCxgEDA1Ihb1o1xmZvYS0J+nEH8BqMmsmd2scx5wXpP0md2tZ2Zm1pm/icPMzGrJAczMzGrJAczMzGrJAczMzGrJAczMzGrJAczMzGrJAczMzGrJAczMzGqpP9/EYb20rX5LCvx7Umb20uU7MDMzqyUHMDMzqyUPIZqZbQPb6qMDf2zQwXdgZmZWSw5gZmZWSx5CtFry8IyZ+Q7MzMxqaae5A5M0AfgXyq8yfz8izt/BRbIXwXdEZra97RR3YJIGAd8BTgQOB06XdPiOLZWZme3MdpY7sKOBtohYCiDpKmAicP+2yMzfjGFmVn+KiB1dBiS9F5gQER/K9+8DjomIszstNxmYnG9fByzeDsUbBvz3dshnR+W3I/J0fvXOb0fk6fwGziER0bKd8tqmdpY7sF6JiEuBS7dnnpJaI2LcrprfjsjT+dU7vx2Rp/OzZnaKz8CAlcBBlfejMs3MzKypnSWAzQPGSDpU0h7AacCMHVwmMzPbie0UQ4gRsVnS2cAsymP0UyNi0Q4uVsN2HbLcAfntiDydX73z2xF5Oj97gZ3iIQ4zM7MXa2cZQjQzM3tRHMDMzKyeImKneQFbgPmV1xTgupxuA56szHsu//4GaK+kj36ReT0MBHBBJa+Vle33O69O+d0L3A38UR/rvg7YDKzNsjQtT6NOlfefAW4FPlOZ/wSwKMv0K2BqZVtrc9/MB27uoW5RzS/zWg48RblI+qvcd5tzXz4G/BfwDPBspm8BNgJP95RfJd/xuT+eBBYANwOv7GPbW5d/lwHDXuS6Afx7Ts/M43tNZf5fZfka+/r7wD8C93W3r4FXZfqjwF257cOApZT/gxyfef8v4NvA3+e+/RpwG+X/JO/J/F7WQx1eBVwFPFTJ64Hcv6+rLHcr8EPg7/P90XnsnpcXMBq4r0k+n8/9sCDr+z3K+TYf+DVwcbaZZuX5Xd07bfPbwP8DbqB8AcL6JuUZnuVemtu7A3hPk+N/AaU9vrLzvD60qTPyGC/Msnymh+VPpnwT0fisy63AuBeR32jgL/pS1k7bWZZlXgD8FHhVf7fZhzKMBU7qcbntXbAeCt1lQ2kc1Cbp7wcu6mtewNXA7cCXKvPOAVYPVF6N/IDBOX0C8PMXUffjGnVvNGpgGnBRV+WhBIaHyc4Y+Ltc9zPABGArJagMozw4czXwr5X1pwHv7aZMgzvltZUStIYBfwtsAFZkvkvyZLgg550JfI4SNBt1uaW7/LppE3dU9s3XqsexF+tX69CnAJb7biulE27s63lkAMt9fRcwsrL8Bxt172pfA8q63dGYB/wB8DZK8Ds36/8IMDf39SGUYPgY8ObKtt4LDO+m/I28PlpJP5kSPJ4GvlpJvxVYnXkNz2P+TOe8aBLAgDdnPkPy/bBsE5+hBK3dgF8Ab29SnufVvZK+e9b9VEpnuwJ4qEl5Om/vEODjdJyTjeP/Rcp58fXenJtN9qeyHidSLmQOzPQhwId7WHdalnc8fQtg42nSR/b2HKikLaOjLf8jcGGzOr6YfF7si172tdusAH0s9HYNYMA+lI7nMGBxZd45lE7gJ5QruUuyUb4f2AScR7mSnkt2DHnC/ozSUc8BDq40yksoV6nfzPezKB3N0qzXQsqV9OPAqbne9ZTO4ynKHcYNlID1DKXzm5nvLwKOAn5O6ShnASOyfkuBO4FWyr8l3ErpLG6nXGVekXVZRjn5V2VZjsvp5cBvKVfH92S9Lsn8G3W5OOsWlBN2ftbjWcod5Krcl4uBNbnf7qAEtcWUAHZTrnMH5Sr8CjoeMDou815IuUNsdH7LKFfU6zLfZVnW9bnffpv7bmPuh89TroY35mt91uO2XD6AK7POV2a92rIebVmuB7LO6ygd7xZK4A9KIPlWluOePF4PUL5dYUOWZXYem7Ny/z+b27qeEuirx/C9WbZpOf0O4D+z7q/Pct6e+dxJBgtK+2gE9A2UYPMspZ2/hdIGngOuzf32cUqb3pTrPgC8iRK8NuTr2SzTglxmc6Y/l/VeT+n4r8/6bKS0lUaZPpX7/jeVtNGU439v7qPGHeNTuT9X537amttbnOVdnvPX0NHG27M8kcs18hgE/HMeu3XARyp9ye2Uc+LBLMuW3P4ayrm5jHJn155l2ET5artPUdrDpixXI98Hgcspd5d/kvvppky/AvhT4P9T2v15lIucBynnxnxK+36CctHZRulLbgW+Tjm+DwJv61SveXlMGvWaS8dIzd90s9zz6t9DAJtA6Wsax6tRx0MoF6ONbX8pl9+b0m/eSznmjf7sBX1Upr+gjsAePH+069S6BLDOw2inVuaNZ2AD2BbKydBoQIuBoyoBbAvw6mwEsymdyPvJIZtc7p+AL+T0fwGTcvqDdHQ20yidTKNuT1Ia/1GU4Y5nKB33Fjo6hfvyoK4HXpPrLcpy3JqN6glK8Pou8EugJfM7ldLRr8tG+hTwCipDiJQT9DngfZTGupwSbL4IfIyOzvHHlMd7/xPYL8t+IzCoUrerMq/GXdgqSqN+Brgs057IMnwpl/1DytX3s1mXm7I8kykXCncAbwX2pHSQh2V+lwPnVE6yS+gYQtyc078CLqQc12coAePJzGs45S5tPuWf5ddTAvc/5Pw76AhkMynH9M9zH14FTKKcgAGckn//Mo/dZbnccjoC2OZ8vRq4BvjXPDZz6Lj7PCb32eJOx3AuJSBOo7Q9UTrixjJPUH694QZK4FiS6RspbWLPLN+HMv03uf7ulGD220x/GaU9zAHGUALHgizT7ZQLi8aF1PsoFwZPUwLHzymdzHOUtrOecsExjNKW76O084WUjm14pi8HfkBpG9/P8mzK8i7N+Wszz5sp58mDlICyby57NyUoPJ3b/GSWZR0dAWwy8AXgE5Q20QocSulL1gOHVvbBukqb3kRpE7/IMq4H9sr05cDBOe9+YHoey63AsZXgHJQ7zt0obWZqHsOJwE9yuX0pwezjWZdlvPAOrDEsfxI5vNyoV04P6VSvGyp9XHfL/a7+PQSwiyh90ehOdTye0jc07jhvAP4Y+N/A9yrbegWlzb2gj6oEsGZ1fD+96Nd3iv8Dq9gQEWO3V16UE+tfImK2pE8Ap1MaG8AT0fHlwldSOtT5lIN4Qy5zF/DOnH4zpbODcnL+UyWvHwHjI2KspGmUq6zLKZ9dbKR8DvB6SsA4mHJ3Mz7zui7TNlGu+s+gXLH+LLe9H3AkMFsSlIC7KudtyfU/kfVttg9uAj4EvJzS2d5I6fRvzTpvAH4v3w+iDH1uqWzjv4A/y7wep3Q0x+T03rnNPSmd3UGUjuCXlADzLHBEbudpYG1EbJU0n3LCPA08HBEP5jLTKVfl3873t1CGlh7LffAbyok6H/gLSmc7KMu2idLRHkf5fOUGSsd1IPDvlDugMVluKFfix1MuIIYAb6QEgEWU8flrKcfnasqxnAG8i9Iprc1trKQcu/+kdOp75Lb2zvJckft3H8qJXj2Gu1f2MRERkn4A/JWkf8u8WygdyDHAUklvrazyOkonelm+v5VygfacpOeAoZm+e5ZnLKWd7kc5PsMone6XsmztlGA9IvMeDRyQ+1CUY/xcRMwBkLSK8n+mbwWui4j1wHpJF+Q+2pqvYZRAPZHSPvbNffz7ua9+TDlP96C00wm53SNyvb0px+xeSlur9mnHA2/IMu5B6bTHAH+d++YaysXU7sAQSQspbWN3SsD7NOXzx30jYoOkdZTO/Q2Uc2IwpTN+BFgfEXMreW8F7s/2vAiYk8dwIXCYpNvz+B1E6QNWVo5J1Y/z7125z39Xr/wOWfL4jKG0qarulrszIh5ukl/DLZK2UC5mvkBpF8srdTw+X/fk+31y27cDF0j6OiWY3i7pSLruo7qqY6/s0k8hSjpG0vx8vbvJIu8Avi9pGeV2+BTlHm4i8u+WyEsESsfYOGEGA/Oy8/2zTuuu7/R+EeXEHZrbaNhK6QxeSTkxfhkRb6BcjXdVLoBFETE2X6+PiOMpx3Yc8EfA2ZQTvZp/Y3ufy78PU07cLZ3yOpUSzMcC/wG0STor6/luSufQsIaOjqeFEqCV2/2/lDuQhZRgdl/Oa+zLrZXtVPfrCEkrJG2gdMZ7VpZ7ttN+aAwHbc3tNrYTmdcpWd5vZn220vU5sIUSME+i3N28LiK+mOts6hTEq/V/eaXsmyhB+JOUi5KLgJGUoLUM+MssxxzgkeoxpATqoygd5T/n/p5JeSDkdMqx+BNKO1pAuQP6Qpb74Mw/Km01eP4+fllus5US2B+ktJfBuQ+HUoafDqR0OIdQOtunKIH8N7l/Gg/iNPKo7r9GO3p3noMzc5k2ykXIbyntBcoQ0hDKxdkzdFy5357lHknH531rKcHqG3QEQnJfD6qUQZS7m0nAAxFxaET8lNKGttJx8fQ3Wa4/oAxLExFPUM6TansLOj4zvDP325coF7HLeb5nKccPOoZBG9OHUM7JuZQLzBmUc2UQL9RYr3pOCPh4pb006tVZd8t17pM6e3uuc0bui87rCPhaZduvjYjL8mLzTZTz/KuS/iGXbdZHdVfHXtmlA1hE/Kqy0zp/NdVg4AcRcUhEjI6Igyid+Nty/n751Va7UTrxX/SQ3W2UW+GxlJPv9m6WbXQKj1NO1lMzfQjlKaSl5HCipOGUJ73WVZY7gPJBN5ShpBZJbwaQtLukIygnSiuls5tOeXCi4WuZ//4RsZbSMH+vMv9JylDDvZRhqkZd9geIiO9kPWfQcccCpQE2PvPZShkL/ztKO/saZQjpCEqn8LJ8dfeTOYtzn4yPiL0odxGzull+FB3f6P1bylUjlI5nD0qnvJ5ytf32LNd/U+7WBlM68cZV8ELKcM4BAJL2l3RIp/x2y2UgH8zJ7b860zZRjtvFlDaxGyUIrMt6vTwvmAYB+1aPIeXKfwhlP/5t7u8tlI7xC5S7jycpx+3K7JiGZh5vptzJSNJrJf05pTN9qFL2jbnNG7Kce1CG83YDXksZQXiQ8lDEWbl/DqJcOY+mdML3Uu6oB2e5Nkj6XLbZxr6/Pbf5R5SLpVPpaE97Zh2gDDsfQAkKN+c6gyrbGURpD/vlepsoQ1ubKefQ6ykXBYOzTFDaypcpAXdPSedK2puOz1zW5HKvoAT7rXQE1EaZqhckz1I+L3+YMuTX+Jxrr9x/Ve2UC49X5fvBkj6U07vlftwvyw1lyGxr5teTWcBfZztB0mFZr6c7rd/VcgNhFvBBSfvktkdKeqWkAykP9fw75QLoTeTweJM+qjud69LUzjaEuFdeFTbcFBFTtlFeQ4A/reR3E2VY6HTKDn+CcsX8WspQ1XWU4buufBz4N0l/S2m8H+g0v1G3gynDg5MoDfYpyhX08cD/oVxdfpPSyMdTrgjvo3RISyifr32Z8nkNuY33AhdKegXlmH67U94XUK74lgJExMwcHvi8pHMoHcIgypUxmc8YSic1GHg6h0H2pQyXdUWUISZyPSLiW5I+T+loGz9Seizl5F9ONz8hERHPSvoA8CNJgykdyiWdFns9pWPfK+twRabfSeksb6ScCN+gHL/X5DpXZ333pDx+PoTSAY7M9VdQAsUPKAFpNqUjr1pPubjYjTIUdRJlKOzARhUy7WrKv0WIEix/RhlSuTGX+zXlGH290zF8D+WD/4skfSnL9yPgtIh4QNLPKMNhjSGY8yifh/0L5ekxKG1nPeXupdFmqr6b9dqbclzIfTKOcnz+kNKWt1CGiUZRguuhlB+hbacc87Ozfp+m40KLiLg7h83vpKOdXZHLDqFclPwN5TzYndIevkv5LHYYZb837tT/kBLwhuR2Xk4JKt+h3OW2UALaqyQtppwbGyid7eDc5gcyrYWO4/ld4BOS7qXjszhyuWco7Z6c92tKXzE46/3LLN/qTvt1HaX/uJnSpt5Bx3m5hvK52YbcxijKRc56ykjQ7pTj1ZXvUy4i7s4LoHbKU6MLgC1Zj2mUdtBsuX6LiJ9K+n3gjhy0WkcZHXgtJXBvpVyM/HVEbMphzM59VHdfF3gLMCX7zK9FxNXNFvJXSZn1kaR1EbHPds7zIuCeiLisx4X7n9c+EbFO0gGUAPSWiHh0W+dr1ls72x2YmXVB0l2Uq/RPb6csb5C0H2V47CsOXraz8R2YmZnV0i79EIeZme26HMDMzKyWHMDMzKyWHMDMzKyWHMDMzKyW/gfN8B5YmdHl0AAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":"with torch.no_grad():\n    #Iterating over the training dataset in batches\n    for i, (images, labels) in enumerate(valid_loader):\n          \n        images = images.to(device)\n        y_true = labels.to(device)\n                                     \n        y_preds = model(images)\n        loss = criterion(y_preds, y_true)\n        \n#         print(loss)\n#         print(y_preds.squeeze())\n        for i in range (0, 31):\n#             print('(Prediction, True): (' + str(y_preds.squeeze()[i].argmax().item()) + ', ' + str(y_true[i].argmax().item()) + ')')\n#         print(y_preds.squeeze().argmax())\n\n\n#         score, scores = get_score(labels, y_preds)\n#         print(score)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-23T11:11:54.133621Z","iopub.execute_input":"2022-08-23T11:11:54.134101Z","iopub.status.idle":"2022-08-23T11:11:59.556921Z","shell.execute_reply.started":"2022-08-23T11:11:54.134036Z","shell.execute_reply":"2022-08-23T11:11:59.555056Z"},"trusted":true},"execution_count":74,"outputs":[{"name":"stdout","text":"(Prediction, True): (2, 6)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 8)\n(Prediction, True): (2, 7)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 8)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 5)\n(Prediction, True): (2, 1)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 8)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 8)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 8)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 8)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 5)\n(Prediction, True): (2, 8)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 7)\n(Prediction, True): (2, 1)\n(Prediction, True): (2, 7)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 8)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 2)\n(Prediction, True): (2, 8)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 8)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 7)\n(Prediction, True): (2, 9)\n(Prediction, True): (2, 8)\n(Prediction, True): (2, 7)\n(Prediction, True): (2, 9)\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)","\u001b[0;32m/tmp/ipykernel_17/2550206284.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0;31m#Iterating over the training dataset in batches\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalid_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mimages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    528\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    529\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    531\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    532\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1205\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1206\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shutdown\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tasks_outstanding\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1207\u001b[0;31m             \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1208\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tasks_outstanding\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1209\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_get_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1161\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1162\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory_thread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_alive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1163\u001b[0;31m                 \u001b[0msuccess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_try_get_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1164\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0msuccess\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1165\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_try_get_data\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1009\u001b[0m         \u001b[0;31m#   (bool: whether successfully get data, any: data if successful else None)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1010\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1011\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data_queue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1012\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1013\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/queue.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m    177\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mremaining\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;36m0.0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    178\u001b[0m                         \u001b[0;32mraise\u001b[0m \u001b[0mEmpty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 179\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnot_empty\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mremaining\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    180\u001b[0m             \u001b[0mitem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    181\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnot_full\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnotify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    298\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    299\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 300\u001b[0;31m                     \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    301\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    302\u001b[0m                     \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "],"ename":"KeyboardInterrupt","evalue":"","output_type":"error"}]},{"cell_type":"code","source":"# random input for testing ONLY\ninput = torch.randn(1, 3, 600, 600)","metadata":{"id":"g5NP8TeAaRmg","execution":{"iopub.status.busy":"2022-08-23T06:07:56.392667Z","iopub.execute_input":"2022-08-23T06:07:56.393072Z","iopub.status.idle":"2022-08-23T06:07:56.40835Z","shell.execute_reply.started":"2022-08-23T06:07:56.393038Z","shell.execute_reply":"2022-08-23T06:07:56.407392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = model(input)\noutput = F.softmax(output, dim=1)\nprediction_score, pred_label_idx = torch.topk(output, 1)\npredicted_label = dict_cols[str(pred_label_idx.item())][1]\nprint('Predicted:', predicted_label, '(', prediction_score.squeeze().item(), ')')\n","metadata":{"id":"Xyi2vMLwBhd7","outputId":"c0efac88-3004-477f-f484-a06a929aed1d","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2- Gradient-based attribution","metadata":{"id":"zHUp39vVBhd7"}},{"cell_type":"raw","source":"Let's compute attributions using Integrated Gradients and visualize them on the image. Integrated gradients computes the integral of the gradients of the output of the model for the predicted class `pred_label_idx` with respect to the input image pixels along the path from the black image to our input image.","metadata":{"id":"AsrsPLJyBhd7"}},{"cell_type":"code","source":"print('Predicted:', predicted_label, '(', prediction_score.squeeze().item(), ')')\n\nintegrated_gradients = IntegratedGradients(model)\nattributions_ig = integrated_gradients.attribute(input, target=pred_label_idx, n_steps=20)","metadata":{"id":"HdZ-YXBABhd8","outputId":"7f7553c4-75c9-497b-9c86-068d9e239143","execution":{"iopub.status.busy":"2022-08-20T08:49:13.260546Z","iopub.execute_input":"2022-08-20T08:49:13.260982Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's visualize the image and corresponding attributions by overlaying the latter on the image.","metadata":{"id":"JFoWqrHYBhd8"}},{"cell_type":"code","source":"default_cmap = LinearSegmentedColormap.from_list('custom blue', \n                                                 [(0, '#ffffff'),\n                                                  (0.25, '#000000'),\n                                                  (1, '#000000')], N=256)\n\n_ = viz.visualize_image_attr(np.transpose(attributions_ig.squeeze().cpu().detach().numpy(), (1,2,0)),\n                             np.transpose(input.squeeze().cpu().detach().numpy(), (1,2,0)),\n                             method='heat_map',\n                             cmap=default_cmap,\n                             show_colorbar=True,\n                             sign='positive',\n                             outlier_perc=1)","metadata":{"id":"Txn0S72qBhd8","outputId":"6422bbbe-a6bf-4c7b-a13c-d60393f36e59"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us compute attributions using Integrated Gradients and smoothens them across multiple images generated by a <em>noise tunnel</em>. The latter adds gaussian noise with a std equals to one, 10 times (nt_samples=10) to the input. Ultimately, noise tunnel smoothens the attributions across `nt_samples` noisy samples using `smoothgrad_sq` technique. `smoothgrad_sq` represents the mean of the squared attributions across `nt_samples` samples.","metadata":{"id":"cdEb1SMNBhd8"}},{"cell_type":"code","source":"noise_tunnel = NoiseTunnel(integrated_gradients)\n\nattributions_ig_nt = noise_tunnel.attribute(input, nt_samples=10, nt_type='smoothgrad_sq', target=pred_label_idx)\n_ = viz.visualize_image_attr_multiple(np.transpose(attributions_ig_nt.squeeze().cpu().detach().numpy(), (1,2,0)),\n                                      np.transpose(input.squeeze().cpu().detach().numpy(), (1,2,0)),\n                                      [\"original_image\", \"heat_map\"],\n                                      [\"all\", \"positive\"],\n                                      cmap=default_cmap,\n                                      show_colorbar=True)\n","metadata":{"id":"DVkF1uBDBhd9"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, let us use `GradientShap`, a linear explanation model which uses a distribution of reference samples (in this case two images) to explain predictions of the model. It computes the expectation of gradients for an input which was chosen randomly between the input and a baseline. The baseline is also chosen randomly from given baseline distribution.","metadata":{"id":"-jYJHa6pBhd9"}},{"cell_type":"code","source":"torch.manual_seed(0)\nnp.random.seed(0)\n\ngradient_shap = GradientShap(model)\n\n# Defining baseline distribution of images\nrand_img_dist = torch.cat([input * 0, input * 1])\n\nattributions_gs = gradient_shap.attribute(input,\n                                          n_samples=5,\n                                          stdevs=0.0001,\n                                          baselines=rand_img_dist,\n                                          target=pred_label_idx)\n_ = viz.visualize_image_attr_multiple(np.transpose(attributions_gs.squeeze().cpu().detach().numpy(), (1,2,0)),\n                                      np.transpose(input.squeeze().cpu().detach().numpy(), (1,2,0)),\n                                      [\"original_image\", \"heat_map\"],\n                                      [\"all\", \"absolute_value\"],\n                                      cmap=default_cmap,\n                                      show_colorbar=True)\n","metadata":{"id":"atw48Bg8Bhd9","outputId":"2a5a2054-f620-4006-dd3b-4c65ef9f9a00","execution":{"iopub.status.busy":"2022-08-20T13:30:40.288177Z","iopub.execute_input":"2022-08-20T13:30:40.28925Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3- Occlusion-based attribution","metadata":{"id":"332TTwN2Bhd9"}},{"cell_type":"markdown","source":"Now let us try a different approach to attribution. We can estimate which areas of the image are critical for the classifier's decision by occluding them and quantifying how the decision changes.\n\nWe run a sliding window of size 15x15 (defined via `sliding_window_shapes`) with a stride of 8 along both image dimensions (a defined via `strides`). At each location, we occlude the image with a baseline value of 0 which correspondes to a gray patch (defined via `baselines`).\n\n**Note:** this computation might take more than one minute to complete, as the model is evaluated at every position of the sliding window.","metadata":{"id":"JXtvRb91Bhd9"}},{"cell_type":"code","source":"occlusion = Occlusion(model)\n\nattributions_occ = occlusion.attribute(input,\n                                       strides = (3, 30, 30),\n                                       target=pred_label_idx,\n                                       sliding_window_shapes=(3,45, 45),\n                                       baselines=0,\n                                       show_progress=True)\n\n_ = viz.visualize_image_attr_multiple(np.transpose(attributions_occ.squeeze().cpu().detach().numpy(), (1,2,0)),\n                                      np.transpose(input.squeeze().cpu().detach().numpy(), (1,2,0)),\n                                      [\"original_image\", \"heat_map\"],\n                                      [\"all\", \"positive\"],\n                                      show_colorbar=True,\n                                      outlier_perc=2,\n                                     )\n","metadata":{"id":"8_lODYz0Bhd-","execution":{"iopub.status.busy":"2022-08-23T06:04:55.786466Z","iopub.execute_input":"2022-08-23T06:04:55.787556Z","iopub.status.idle":"2022-08-23T06:04:55.850345Z","shell.execute_reply.started":"2022-08-23T06:04:55.787512Z","shell.execute_reply":"2022-08-23T06:04:55.848691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us visualize the attribution, focusing on the areas with positive attribution (those that are critical for the classifier's decision):","metadata":{"id":"yoT8aA19Bhd-"}},{"cell_type":"markdown","source":"The upper part of the goose, especially the beak, seems to be the most critical for the model to predict this class.\n\nWe can verify this further by occluding the image using a larger sliding window:","metadata":{"id":"iRyWGcYtBhd-"}},{"cell_type":"code","source":"occlusion = Occlusion(model)\n\nattributions_occ = occlusion.attribute(input,\n                                       strides = (3, 50, 50),\n                                       target=pred_label_idx,\n                                       sliding_window_shapes=(3,60, 60),\n                                       baselines=0)\n\n_ = viz.visualize_image_attr_multiple(np.transpose(attributions_occ.squeeze().cpu().detach().numpy(), (1,2,0)),\n                                      np.transpose(input.squeeze().cpu().detach().numpy(), (1,2,0)),\n                                      [\"original_image\", \"heat_map\"],\n                                      [\"all\", \"positive\"],\n                                      show_colorbar=True,\n                                      outlier_perc=2,\n                                     )","metadata":{"id":"eeCJrvzyBhd-","outputId":"0b7ae3c7-9b10-47ab-da41-fcf7abbe367d","execution":{"iopub.status.busy":"2022-08-20T12:42:40.798748Z","iopub.execute_input":"2022-08-20T12:42:40.799449Z","iopub.status.idle":"2022-08-20T12:46:05.831865Z","shell.execute_reply.started":"2022-08-20T12:42:40.799412Z","shell.execute_reply":"2022-08-20T12:46:05.830865Z"}},"execution_count":null,"outputs":[]}]}