{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":13836,"databundleVersionId":1718836,"sourceType":"competition"},{"sourceId":1715591,"sourceType":"datasetVersion","datasetId":1017454},{"sourceId":1715611,"sourceType":"datasetVersion","datasetId":1017466},{"sourceId":9300100,"sourceType":"datasetVersion","datasetId":5631007}],"dockerImageVersionId":30039,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About this notebook  \n- Visualize ViT Attention Map\n- ViT github is [here](https://github.com/tczhangzhi/VisionTransformer-Pytorch).\n(I modified a little for attention map. please see this [issue](https://github.com/tczhangzhi/VisionTransformer-Pytorch/issues/1#issuecomment-739138519).)\n\n\nI want to show that Attention Map for cassava.\n- I just show a few sample in 2019 train dataset.\n\nYou can check my pretrained ViT weight in [here](https://www.kaggle.com/piantic/cassava-vit-b-16).\n\n### If this kernel is useful, feel free to upvote:)","metadata":{}},{"cell_type":"markdown","source":"# Vision Transformer (ViT) : Attention Map","metadata":{}},{"cell_type":"markdown","source":"This is the Attention Map example.\n- Reference is [here](https://github.com/jeonsworld/ViT-pytorch/blob/main/visualize_attention_map.ipynb).","metadata":{}},{"cell_type":"markdown","source":"<img src='https://user-images.githubusercontent.com/6073256/101206904-2a338f00-36b3-11eb-8920-f617abab1604.png'>","metadata":{}},{"cell_type":"markdown","source":"Next, we will see the attention map for cassava leaf!","metadata":{}},{"cell_type":"markdown","source":"# Import libraries","metadata":{}},{"cell_type":"code","source":"import sys\n\npackage_path = '../input/visiontransformerpytorch121/VisionTransformer-Pytorch'\nsys.path.append(package_path)\n\nimport os\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nimport torchvision.models as models\n\nimport cv2\nfrom PIL import Image\nfrom torchvision import transforms\n\nfrom vision_transformer_pytorch import VisionTransformer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-18T04:08:44.050528Z","iopub.execute_input":"2024-10-18T04:08:44.050887Z","iopub.status.idle":"2024-10-18T04:08:45.747285Z","shell.execute_reply.started":"2024-10-18T04:08:44.050857Z","shell.execute_reply":"2024-10-18T04:08:45.746540Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transform = transforms.Compose([\n    transforms.Resize((384, 384)),\n    transforms.ToTensor(),\n    transforms.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],\n    ),\n])","metadata":{"execution":{"iopub.status.busy":"2024-10-18T04:08:45.749578Z","iopub.execute_input":"2024-10-18T04:08:45.749903Z","iopub.status.idle":"2024-10-18T04:08:45.755307Z","shell.execute_reply.started":"2024-10-18T04:08:45.749873Z","shell.execute_reply":"2024-10-18T04:08:45.754448Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Helper function","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\ndef load_state(model_path):\n    state_dict = torch.load(model_path)['model']\n    state_dict = {k[7:] if k.startswith('module.') else k: state_dict[k] for k in state_dict.keys()}\n    state_dict = {k[6:] if k.startswith('model.') else k: state_dict[k] for k in state_dict.keys()}\n\n    return state_dict","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-10-18T04:08:45.756360Z","iopub.execute_input":"2024-10-18T04:08:45.756634Z","iopub.status.idle":"2024-10-18T04:08:45.766927Z","shell.execute_reply.started":"2024-10-18T04:08:45.756607Z","shell.execute_reply":"2024-10-18T04:08:45.766265Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_attention_map(img, get_mask=False):\n    x = transform(img)\n    x.size()\n\n    logits, att_mat = model(x.unsqueeze(0))\n\n    att_mat = torch.stack(att_mat).squeeze(1)\n\n    # Average the attention weights across all heads.\n    att_mat = torch.mean(att_mat, dim=1)\n\n    # To account for residual connections, we add an identity matrix to the\n    # attention matrix and re-normalize the weights.\n    residual_att = torch.eye(att_mat.size(1))\n    aug_att_mat = att_mat + residual_att\n    aug_att_mat = aug_att_mat / aug_att_mat.sum(dim=-1).unsqueeze(-1)\n\n    # Recursively multiply the weight matrices\n    joint_attentions = torch.zeros(aug_att_mat.size())\n    joint_attentions[0] = aug_att_mat[0]\n\n    for n in range(1, aug_att_mat.size(0)):\n        joint_attentions[n] = torch.matmul(aug_att_mat[n], joint_attentions[n-1])\n\n    v = joint_attentions[-1]\n    grid_size = int(np.sqrt(aug_att_mat.size(-1)))\n    mask = v[0, 1:].reshape(grid_size, grid_size).detach().numpy()\n    if get_mask:\n        result = cv2.resize(mask / mask.max(), img.size)\n    else:        \n        mask = cv2.resize(mask / mask.max(), img.size)[..., np.newaxis]\n        result = (mask * img).astype(\"uint8\")\n    \n    return result\n\ndef plot_attention_map(original_img, att_map):\n    fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(16, 16))\n    ax1.set_title('Original')\n    ax2.set_title('Attention Map Last Layer')\n    _ = ax1.imshow(original_img)\n    _ = ax2.imshow(att_map)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-10-18T04:08:45.767979Z","iopub.execute_input":"2024-10-18T04:08:45.768272Z","iopub.status.idle":"2024-10-18T04:08:45.783239Z","shell.execute_reply.started":"2024-10-18T04:08:45.768246Z","shell.execute_reply":"2024-10-18T04:08:45.782551Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load ViT Model","metadata":{}},{"cell_type":"code","source":"model = VisionTransformer.from_name('ViT-B_16', num_classes=5)\nstate = load_state('../input/cassava-vit-b-16/ViT-B_16_fold0.pth')\nmodel.load_state_dict(state)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T04:08:45.785637Z","iopub.execute_input":"2024-10-18T04:08:45.785953Z","iopub.status.idle":"2024-10-18T04:08:53.747887Z","shell.execute_reply.started":"2024-10-18T04:08:45.785927Z","shell.execute_reply":"2024-10-18T04:08:53.747029Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Visualize Attention Map","metadata":{}},{"cell_type":"code","source":"label_map = pd.read_json('../input/cassava-leaf-disease-classification/label_num_to_disease_map.json', \n                         orient='index')\n\ndisplay(label_map)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T04:08:53.749345Z","iopub.execute_input":"2024-10-18T04:08:53.749633Z","iopub.status.idle":"2024-10-18T04:08:54.159553Z","shell.execute_reply.started":"2024-10-18T04:08:53.749605Z","shell.execute_reply":"2024-10-18T04:08:54.158822Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CBB - Class0","metadata":{}},{"cell_type":"code","source":"img1 = Image.open(\"../input/cassava-vit-b-16/train-cbb-114.jpg\")\nimg2 = Image.open(\"../input/cassava-vit-b-16/train-cbb-44.jpg\")\n\nresult1 = get_attention_map(img1)\nresult2 = get_attention_map(img2)","metadata":{"execution":{"iopub.status.busy":"2024-10-18T04:08:54.161200Z","iopub.execute_input":"2024-10-18T04:08:54.161592Z","iopub.status.idle":"2024-10-18T04:08:56.463121Z","shell.execute_reply.started":"2024-10-18T04:08:54.161543Z","shell.execute_reply":"2024-10-18T04:08:56.462222Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img1, result1)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:14:13.614033Z","iopub.execute_input":"2024-10-17T15:14:13.614386Z","iopub.status.idle":"2024-10-17T15:14:14.270272Z","shell.execute_reply.started":"2024-10-17T15:14:13.614352Z","shell.execute_reply":"2024-10-17T15:14:14.269283Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img2, result2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:14:45.791390Z","iopub.execute_input":"2024-10-17T15:14:45.791712Z","iopub.status.idle":"2024-10-17T15:14:46.294334Z","shell.execute_reply.started":"2024-10-17T15:14:45.791683Z","shell.execute_reply":"2024-10-17T15:14:46.293518Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Check mask for Attention Map","metadata":{}},{"cell_type":"code","source":"result1 = get_attention_map(img1, True)\nresult2 = get_attention_map(img2, True)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:14:52.494884Z","iopub.execute_input":"2024-10-17T15:14:52.495261Z","iopub.status.idle":"2024-10-17T15:14:54.382800Z","shell.execute_reply.started":"2024-10-17T15:14:52.495225Z","shell.execute_reply":"2024-10-17T15:14:54.381778Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img1, result1)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:14:54.384699Z","iopub.execute_input":"2024-10-17T15:14:54.385117Z","iopub.status.idle":"2024-10-17T15:14:54.869409Z","shell.execute_reply.started":"2024-10-17T15:14:54.385074Z","shell.execute_reply":"2024-10-17T15:14:54.868694Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img2, result2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:14:55.844299Z","iopub.execute_input":"2024-10-17T15:14:55.844626Z","iopub.status.idle":"2024-10-17T15:14:56.291665Z","shell.execute_reply.started":"2024-10-17T15:14:55.844597Z","shell.execute_reply":"2024-10-17T15:14:56.290658Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CBSD - Class1","metadata":{}},{"cell_type":"code","source":"img1 = Image.open(\"/kaggle/input/reid-9-full/content/Compress/bedroom/00000003.jpg\")\nimg2 = Image.open(\"../input/cassava-vit-b-16/train-cbsd-821.jpg\")","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:19:19.276895Z","iopub.execute_input":"2024-10-17T15:19:19.277218Z","iopub.status.idle":"2024-10-17T15:19:19.290068Z","shell.execute_reply.started":"2024-10-17T15:19:19.277189Z","shell.execute_reply":"2024-10-17T15:19:19.289245Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"result1 = get_attention_map(img1)\nresult2 = get_attention_map(img2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:19:19.751825Z","iopub.execute_input":"2024-10-17T15:19:19.752147Z","iopub.status.idle":"2024-10-17T15:19:21.550739Z","shell.execute_reply.started":"2024-10-17T15:19:19.752118Z","shell.execute_reply":"2024-10-17T15:19:21.549967Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img1, result1)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:19:21.552878Z","iopub.execute_input":"2024-10-17T15:19:21.553258Z","iopub.status.idle":"2024-10-17T15:19:22.022880Z","shell.execute_reply.started":"2024-10-17T15:19:21.553218Z","shell.execute_reply":"2024-10-17T15:19:22.022106Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img2, result2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:19:22.024005Z","iopub.execute_input":"2024-10-17T15:19:22.024258Z","iopub.status.idle":"2024-10-17T15:19:22.667699Z","shell.execute_reply.started":"2024-10-17T15:19:22.024232Z","shell.execute_reply":"2024-10-17T15:19:22.666849Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Check mask for Attention Map","metadata":{}},{"cell_type":"code","source":"result1 = get_attention_map(img1, True)\nresult2 = get_attention_map(img2, True)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:19:35.128645Z","iopub.execute_input":"2024-10-17T15:19:35.129020Z","iopub.status.idle":"2024-10-17T15:19:36.953538Z","shell.execute_reply.started":"2024-10-17T15:19:35.128982Z","shell.execute_reply":"2024-10-17T15:19:36.952666Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img1, result1)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:19:36.955454Z","iopub.execute_input":"2024-10-17T15:19:36.955762Z","iopub.status.idle":"2024-10-17T15:19:37.536254Z","shell.execute_reply.started":"2024-10-17T15:19:36.955707Z","shell.execute_reply":"2024-10-17T15:19:37.535304Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img2, result2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:15:08.169354Z","iopub.execute_input":"2024-10-17T15:15:08.169632Z","iopub.status.idle":"2024-10-17T15:15:08.708558Z","shell.execute_reply.started":"2024-10-17T15:15:08.169603Z","shell.execute_reply":"2024-10-17T15:15:08.707753Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CGM - Class2","metadata":{}},{"cell_type":"code","source":"img1 = Image.open(\"../input/cassava-vit-b-16/train-cgm-498.jpg\")\nimg2 = Image.open(\"../input/cassava-vit-b-16/train-cgm-6.jpg\")\n\nresult1 = get_attention_map(img1)\nresult2 = get_attention_map(img2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:15:08.710042Z","iopub.execute_input":"2024-10-17T15:15:08.710374Z","iopub.status.idle":"2024-10-17T15:15:10.953644Z","shell.execute_reply.started":"2024-10-17T15:15:08.710340Z","shell.execute_reply":"2024-10-17T15:15:10.952799Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img1, result1)\nplot_attention_map(img2, result2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:15:10.955050Z","iopub.execute_input":"2024-10-17T15:15:10.955455Z","iopub.status.idle":"2024-10-17T15:15:12.245708Z","shell.execute_reply.started":"2024-10-17T15:15:10.955416Z","shell.execute_reply":"2024-10-17T15:15:12.244670Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Check mask for Attention Map","metadata":{}},{"cell_type":"code","source":"result1 = get_attention_map(img1, True)\nresult2 = get_attention_map(img2, True)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:15:12.247120Z","iopub.execute_input":"2024-10-17T15:15:12.247475Z","iopub.status.idle":"2024-10-17T15:15:14.165708Z","shell.execute_reply.started":"2024-10-17T15:15:12.247439Z","shell.execute_reply":"2024-10-17T15:15:14.164722Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img1, result1)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:15:14.169148Z","iopub.execute_input":"2024-10-17T15:15:14.169529Z","iopub.status.idle":"2024-10-17T15:15:14.704811Z","shell.execute_reply.started":"2024-10-17T15:15:14.169489Z","shell.execute_reply":"2024-10-17T15:15:14.704065Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img2, result2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:15:14.706220Z","iopub.execute_input":"2024-10-17T15:15:14.706480Z","iopub.status.idle":"2024-10-17T15:15:15.405388Z","shell.execute_reply.started":"2024-10-17T15:15:14.706454Z","shell.execute_reply":"2024-10-17T15:15:15.404617Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CMD - Class3","metadata":{}},{"cell_type":"code","source":"img1 = Image.open(\"/kaggle/input/reid-9-full/content/Compress/bathroom/00000004.jpg\")\nimg2 = Image.open(\"../input/cassava-vit-b-16/train-cbsd-821.jpg\")","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:17:44.389674Z","iopub.execute_input":"2024-10-17T15:17:44.390022Z","iopub.status.idle":"2024-10-17T15:17:44.406390Z","shell.execute_reply.started":"2024-10-17T15:17:44.389987Z","shell.execute_reply":"2024-10-17T15:17:44.405683Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"result1 = get_attention_map(img1)\nresult2 = get_attention_map(img2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:17:45.523326Z","iopub.execute_input":"2024-10-17T15:17:45.523661Z","iopub.status.idle":"2024-10-17T15:17:47.284852Z","shell.execute_reply.started":"2024-10-17T15:17:45.523632Z","shell.execute_reply":"2024-10-17T15:17:47.283975Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img1, result1)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:17:47.286712Z","iopub.execute_input":"2024-10-17T15:17:47.287107Z","iopub.status.idle":"2024-10-17T15:17:47.793082Z","shell.execute_reply.started":"2024-10-17T15:17:47.287065Z","shell.execute_reply":"2024-10-17T15:17:47.792354Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img2, result2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:17:47.794197Z","iopub.execute_input":"2024-10-17T15:17:47.794456Z","iopub.status.idle":"2024-10-17T15:17:48.442360Z","shell.execute_reply.started":"2024-10-17T15:17:47.794430Z","shell.execute_reply":"2024-10-17T15:17:48.441433Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Check mask for Attention Map","metadata":{}},{"cell_type":"code","source":"result1 = get_attention_map(img1, True)\nresult2 = get_attention_map(img2, True)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:17:56.191395Z","iopub.execute_input":"2024-10-17T15:17:56.191719Z","iopub.status.idle":"2024-10-17T15:17:57.957873Z","shell.execute_reply.started":"2024-10-17T15:17:56.191691Z","shell.execute_reply":"2024-10-17T15:17:57.957103Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img1, result1)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:17:57.959975Z","iopub.execute_input":"2024-10-17T15:17:57.960346Z","iopub.status.idle":"2024-10-17T15:17:58.460506Z","shell.execute_reply.started":"2024-10-17T15:17:57.960306Z","shell.execute_reply":"2024-10-17T15:17:58.459658Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img2, result2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:17:15.440330Z","iopub.execute_input":"2024-10-17T15:17:15.440610Z","iopub.status.idle":"2024-10-17T15:17:15.980672Z","shell.execute_reply.started":"2024-10-17T15:17:15.440582Z","shell.execute_reply":"2024-10-17T15:17:15.979967Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Healthy - Class4","metadata":{}},{"cell_type":"code","source":"img1 = Image.open(\"../input/cassava-vit-b-16/train-healthy-105.jpg\")\nimg2 = Image.open(\"../input/cassava-vit-b-16/train-healthy-236.jpg\")\n\nresult1 = get_attention_map(img1)\nresult2 = get_attention_map(img2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:15:28.859269Z","iopub.execute_input":"2024-10-17T15:15:28.859609Z","iopub.status.idle":"2024-10-17T15:15:30.850736Z","shell.execute_reply.started":"2024-10-17T15:15:28.859575Z","shell.execute_reply":"2024-10-17T15:15:30.849916Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img1, result1)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:15:30.852872Z","iopub.execute_input":"2024-10-17T15:15:30.853245Z","iopub.status.idle":"2024-10-17T15:15:31.388646Z","shell.execute_reply.started":"2024-10-17T15:15:30.853206Z","shell.execute_reply":"2024-10-17T15:15:31.387668Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img2, result2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:15:31.390077Z","iopub.execute_input":"2024-10-17T15:15:31.390428Z","iopub.status.idle":"2024-10-17T15:15:31.877371Z","shell.execute_reply.started":"2024-10-17T15:15:31.390393Z","shell.execute_reply":"2024-10-17T15:15:31.876607Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Check mask for Attention Map","metadata":{}},{"cell_type":"code","source":"result1 = get_attention_map(img1, True)\nresult2 = get_attention_map(img2, True)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:15:31.878735Z","iopub.execute_input":"2024-10-17T15:15:31.879042Z","iopub.status.idle":"2024-10-17T15:15:33.852465Z","shell.execute_reply.started":"2024-10-17T15:15:31.879007Z","shell.execute_reply":"2024-10-17T15:15:33.847768Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img1, result1)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:15:33.857277Z","iopub.execute_input":"2024-10-17T15:15:33.858104Z","iopub.status.idle":"2024-10-17T15:15:34.344489Z","shell.execute_reply.started":"2024-10-17T15:15:33.858026Z","shell.execute_reply":"2024-10-17T15:15:34.343660Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_attention_map(img2, result2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:15:34.346151Z","iopub.execute_input":"2024-10-17T15:15:34.346544Z","iopub.status.idle":"2024-10-17T15:15:34.828473Z","shell.execute_reply.started":"2024-10-17T15:15:34.346501Z","shell.execute_reply":"2024-10-17T15:15:34.827700Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Visualize Attention Maps","metadata":{}},{"cell_type":"markdown","source":"For example, I will use cbsd images.","metadata":{}},{"cell_type":"code","source":"def get_attention_info(img):\n    x = transform(img)\n    x.size()\n\n    logits, att_mat = model(x.unsqueeze(0))\n\n    att_mat = torch.stack(att_mat).squeeze(1)\n\n    # Average the attention weights across all heads.\n    att_mat = torch.mean(att_mat, dim=1)\n\n    # To account for residual connections, we add an identity matrix to the\n    # attention matrix and re-normalize the weights.\n    residual_att = torch.eye(att_mat.size(1))\n    aug_att_mat = att_mat + residual_att\n    aug_att_mat = aug_att_mat / aug_att_mat.sum(dim=-1).unsqueeze(-1)\n\n    # Recursively multiply the weight matrices\n    joint_attentions = torch.zeros(aug_att_mat.size())\n    joint_attentions[0] = aug_att_mat[0]\n\n    for n in range(1, aug_att_mat.size(0)):\n        joint_attentions[n] = torch.matmul(aug_att_mat[n], joint_attentions[n-1])\n\n    grid_size = int(np.sqrt(aug_att_mat.size(-1)))\n    \n    return joint_attentions, grid_size\n\n# def plot_attention_map(original_img, att_map):\n#     fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(16, 16))\n#     ax1.set_title('Original')\n#     ax2.set_title('Attention Map Last Layer')\n#     _ = ax1.imshow(original_img)\n#     _ = ax2.imshow(att_map)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-10-17T15:15:34.829795Z","iopub.execute_input":"2024-10-17T15:15:34.830084Z","iopub.status.idle":"2024-10-17T15:15:34.840625Z","shell.execute_reply.started":"2024-10-17T15:15:34.830056Z","shell.execute_reply":"2024-10-17T15:15:34.839548Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img1 = Image.open(\"/kaggle/input/reid-9-full/content/Compress/bathroom/00000001.jpg\")\nimg2 = Image.open(\"/kaggle/input/reid-9-full/content/Compress/bathroom/00000002.jpg\")","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:16:29.376549Z","iopub.execute_input":"2024-10-17T15:16:29.376939Z","iopub.status.idle":"2024-10-17T15:16:29.393423Z","shell.execute_reply.started":"2024-10-17T15:16:29.376901Z","shell.execute_reply":"2024-10-17T15:16:29.392819Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"joint_att1, grid_size1 = get_attention_info(img1)\njoint_att2, grid_size2 = get_attention_info(img2)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:16:29.924445Z","iopub.execute_input":"2024-10-17T15:16:29.924773Z","iopub.status.idle":"2024-10-17T15:16:32.121030Z","shell.execute_reply.started":"2024-10-17T15:16:29.924724Z","shell.execute_reply":"2024-10-17T15:16:32.120157Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i, v in enumerate(joint_att1):\n    v = joint_att1[-1]\n    mask = v[0, 1:].reshape(grid_size1, grid_size1).detach().numpy()\n    mask = cv2.resize(mask / mask.max(), img1.size)[..., np.newaxis]\n    result = (mask * img1).astype(\"uint8\")\n\n    fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(16, 16))\n    ax1.set_title('Original')\n    ax2.set_title('Attention Map_%d Layer' % (i+1))\n    _ = ax1.imshow(img1)\n    _ = ax2.imshow(result)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:16:32.122869Z","iopub.execute_input":"2024-10-17T15:16:32.123166Z","iopub.status.idle":"2024-10-17T15:16:38.235826Z","shell.execute_reply.started":"2024-10-17T15:16:32.123136Z","shell.execute_reply":"2024-10-17T15:16:38.235107Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i, v in enumerate(joint_att2):\n    v = joint_att2[-1]\n    mask = v[0, 1:].reshape(grid_size2, grid_size2).detach().numpy()\n    mask = cv2.resize(mask / mask.max(), img1.size)[..., np.newaxis]\n    result = (mask * img2).astype(\"uint8\")\n\n    fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(16, 16))\n    ax1.set_title('Original')\n    ax2.set_title('Attention Map_%d Layer' % (i+1))\n    _ = ax1.imshow(img2)\n    _ = ax2.imshow(result)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T15:16:38.236844Z","iopub.execute_input":"2024-10-17T15:16:38.237102Z","iopub.status.idle":"2024-10-17T15:16:44.256976Z","shell.execute_reply.started":"2024-10-17T15:16:38.237076Z","shell.execute_reply":"2024-10-17T15:16:44.256162Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## If this kernel is useful, <font color='orange'>please upvote</font>!","metadata":{}}]}