{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\ncount=0\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        count+=1\n        if count>20:\n            break\n\n    if count>20:\n        break\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Use the kagglehub client library to attach Kaggle resources like competitions, datasets, and models to your session\n# Learn more about kagglehub: https://github.com/Kaggle/kagglehub/blob/main/README.md\n\nimport kagglehub\n# kagglehub.dataset_download('<owner>/<dataset-slug>')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-06-29T12:59:58.021293Z","iopub.execute_input":"2026-06-29T12:59:58.021689Z","iopub.status.idle":"2026-06-29T12:59:59.064664Z","shell.execute_reply.started":"2026-06-29T12:59:58.021651Z","shell.execute_reply":"2026-06-29T12:59:59.063769Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Plot할 이미지 고르기","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom glob import glob\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport numpy as np\nimport cv2\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport torch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T12:59:59.066692Z","iopub.execute_input":"2026-06-29T12:59:59.067097Z","iopub.status.idle":"2026-06-29T12:59:59.072759Z","shell.execute_reply.started":"2026-06-29T12:59:59.067066Z","shell.execute_reply":"2026-06-29T12:59:59.071668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df=pd.read_csv('../input/competitions/aptos2019-blindness-detection/test.csv')\n\n#test_df에 이미지 path 만들기\nall_image_path={os.path.basename(x).replace('.png',''):x for x in\n               glob(os.path.join('..','input','competitions','aptos2019-blindness-detection','test_images','*.png'))}\nprint('Scans Found:', len(all_image_path), 'Total image Index:', len(test_df))\n\ntest_df['path']=test_df['id_code'].map(all_image_path.get)\ntest_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T12:59:59.074021Z","iopub.execute_input":"2026-06-29T12:59:59.074399Z","iopub.status.idle":"2026-06-29T12:59:59.107568Z","shell.execute_reply.started":"2026-06-29T12:59:59.074372Z","shell.execute_reply":"2026-06-29T12:59:59.106791Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"diagnosis_df=pd.read_csv('../input/notebooks/ryuminhyuk/aptos-vit-baseline/submission.csv')\n\ndf=pd.merge(test_df, diagnosis_df, on='id_code', how='inner')\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T12:59:59.109290Z","iopub.execute_input":"2026-06-29T12:59:59.109533Z","iopub.status.idle":"2026-06-29T12:59:59.126864Z","shell.execute_reply.started":"2026-06-29T12:59:59.109511Z","shell.execute_reply":"2026-06-29T12:59:59.125965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#image plot 함수\ndef plot_images(df, rows, columns, figsize):\n    fig, axes=plt.subplots(rows, columns, figsize=figsize)\n    idx=0\n    for i in range(rows):\n        for j in range(columns):\n            image=np.array(Image.open(df['path'].values[idx]).convert('RGB'))\n            axes[i,j].imshow(image)\n            axes[i,j].set_title(f'Label: {df['diagnosis'].values[idx]}, id: {df['id_code'].values[idx]}')\n\n            idx+=1\n\n    plt.show()\n\n\nplot_images(df, 3,3, (10,10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T12:59:59.128192Z","iopub.execute_input":"2026-06-29T12:59:59.128550Z","iopub.status.idle":"2026-06-29T13:00:00.874444Z","shell.execute_reply.started":"2026-06-29T12:59:59.128510Z","shell.execute_reply":"2026-06-29T13:00:00.873350Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 8번째 이미지로 attention rollout","metadata":{}},{"cell_type":"code","source":"#AutoCrop 코드\ndef AutoCrop(img, tol=7):   #tol=tolerance(허용 오차)-밝기 7 이하인 픽셀은 크롭\n    gray_image=cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n\n    mask=gray_image>tol\n    row_mask=mask.any(1)  #열을 흝으며 해당 행에 true를 찾음-그래서 1\n    col_mask=mask.any(0)\n\n    check_shape=img[:,:,0][np.ix_(row_mask, col_mask)].shape   \n    #np.ix_(mask.any(1), mask.any(0)) 브로드케스팅 계산이 가능한 형태로 튜플 반환((3,1),(1,3) 형태의 array)\n\n    if check_shape[0]==0:\n        return img\n\n    else:\n        img1=img[:,:,0][np.ix_(row_mask, col_mask)]\n        img2=img[:,:,1][np.ix_(row_mask, col_mask)]\n        img3=img[:,:,2][np.ix_(row_mask, col_mask)]\n\n        img=np.stack([img1, img2, img3], axis=-1)\n\n        return img\n\n\n#Ben Graham 전처리 코드\ndef ben_color(img):\n    blur=cv2.GaussianBlur(img, (0,0), sigmaX=20)  #가우시안 커널 (0,0)으로 두면 시그마 값에 맞춰서 자동 설정\n\n    #조도 보정 공식 구현 (4*original-4*blur+128)\n    result=cv2.addWeighted(img, 4, blur, -4, 128)\n    return result","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T13:00:00.875921Z","iopub.execute_input":"2026-06-29T13:00:00.876312Z","iopub.status.idle":"2026-06-29T13:00:00.886192Z","shell.execute_reply.started":"2026-06-29T13:00:00.876273Z","shell.execute_reply":"2026-06-29T13:00:00.885286Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#시각화 해보기\nimage=np.array(Image.open(df['path'].values[7]).convert('RGB'))\nimg1=AutoCrop(image)\nimg2=ben_color(img1)\n\nfig, axes=plt.subplots(1,3, figsize=(10,4))\naxes[0].imshow(image)\naxes[0].set_title('Original')\n\naxes[1].imshow(img1)\naxes[1].set_title('Autocrop')\n\naxes[2].imshow(img2)\naxes[2].set_title('Autocrop+Ben Color')\n\nfor ax in axes:\n    ax.axis('off')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T13:00:00.887439Z","iopub.execute_input":"2026-06-29T13:00:00.887852Z","iopub.status.idle":"2026-06-29T13:00:01.589456Z","shell.execute_reply.started":"2026-06-29T13:00:00.887780Z","shell.execute_reply":"2026-06-29T13:00:01.588332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#전처리 코\ntest_transform=A.Compose([\n    A.Resize(384,384),\n    A.Normalize(),\n    ToTensorV2()]\n)\n\nimg=test_transform(image=img2)['image']   #AutoCrop, ben 전처리 완료 후 리사이즈, 정규화\n\ninput=img.unsqueeze(0) #배치 축 생성(albumentation은 배치 축 없는걸 받음)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T13:00:01.590724Z","iopub.execute_input":"2026-06-29T13:00:01.591127Z","iopub.status.idle":"2026-06-29T13:00:01.601682Z","shell.execute_reply.started":"2026-06-29T13:00:01.591098Z","shell.execute_reply":"2026-06-29T13:00:01.600852Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 모델 생성","metadata":{}},{"cell_type":"code","source":"!pip install timm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T13:00:01.603229Z","iopub.execute_input":"2026-06-29T13:00:01.603636Z","iopub.status.idle":"2026-06-29T13:00:05.649521Z","shell.execute_reply.started":"2026-06-29T13:00:01.603607Z","shell.execute_reply":"2026-06-29T13:00:05.648528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import timm\n\ndevice='cuda' if torch.cuda.is_available() else 'cpu'\n\nmodel=timm.create_model('vit_base_patch16_384', pretrained=True, num_classes=1)  #224모델을 384로 파인튜닝 한 모델, 헤드는 초기화\n\ninput=input.to(device)\nmodel.to(device)\noutput=model(input)\nprint(output.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T13:00:05.653350Z","iopub.execute_input":"2026-06-29T13:00:05.653818Z","iopub.status.idle":"2026-06-29T13:00:08.591376Z","shell.execute_reply.started":"2026-06-29T13:00:05.653779Z","shell.execute_reply":"2026-06-29T13:00:08.590368Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Rollout 구현\n\n**[Code Reference]**\n* GitHub Repository: [jacobgil/vit-explain](https://github.com/jacobgil/vit-explain)\n* Author: Jacob Gildenblat","metadata":{}},{"cell_type":"code","source":"#timm 모델의 레이어 이름 출력해서 확인하기\nfor name, module in model.named_modules():\n    if 'attn_drop' in name:\n        print(name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T13:00:08.592649Z","iopub.execute_input":"2026-06-29T13:00:08.592982Z","iopub.status.idle":"2026-06-29T13:00:08.600028Z","shell.execute_reply.started":"2026-06-29T13:00:08.592955Z","shell.execute_reply":"2026-06-29T13:00:08.598949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def rollout(attentions, discard_ratio, head_fusion):\n    result = torch.eye(attentions[0].size(-1))   #어텐션 맵 하나의 크기는 [1, 12, 577, 577](배치, 헤드, 토큰, 토큰)\n    #처음 곱셈을 시작할 단위행렬 생성\n    with torch.no_grad():\n        for attention in attentions:    #헤드를 섞는 방식. 논문에서는 mean\n            if head_fusion == \"mean\":\n                attention_heads_fused = attention.mean(axis=1)\n            elif head_fusion == \"max\":\n                attention_heads_fused = attention.max(axis=1)[0]  #[0]을 붙인 이유는 max() 함수가 (최대값, 인덱스) 쌍을 뱉기 때문\n            elif head_fusion == \"min\":\n                attention_heads_fused = attention.min(axis=1)[0]\n            else:\n                raise \"Attention head fusion type Not supported\"\n\n            # Drop the lowest attentions, but\n            # don't drop the class token    #자잘한 노이즈 쳐내기 위해 하위 90퍼센트 데이터는 0으로 만들기\n            #작은 값이어도 행렬 곱셈을 계속 누적하다보면 배경이 하얗게 번지는 블러 생김\n            flat = attention_heads_fused.view(attention_heads_fused.size(0), -1)\n            _, indices = flat.topk(int(flat.size(-1)*discard_ratio), -1, False)\n            indices = indices[indices != 0]  #cls 토큰은 드롭 안하기\n            flat[0, indices] = 0\n\n            I = torch.eye(attention_heads_fused.size(-1))  #잔차연결 계산을 위한 단위행렬\n            a = (attention_heads_fused + 1.0*I)/2\n            a = a / a.sum(dim=-1)  #이론상으로 a의 총합은 1이지만 컴퓨터의 소수점 연산 오차의 누적을 막기 위해 레이어 생성 때마다 정규화\n\n            result = torch.matmul(a, result)\n    \n    # Look at the total attention between the class token,\n    # and the image patches\n    mask = result[0, 0 , 1:]  #cls 벡터에서 cls토큰 제외하기\n    # 이미지 크기 384, 패치 크기 16 모델이다. 576개의 이미지 패치(24*24)\n    width = int(mask.size(-1)**0.5)\n    mask = mask.reshape(width, width).numpy()\n    mask = mask / np.max(mask)   #시각적 품질 향상을 위해 최댓값으로 정규화\n    return mask  #(24,24)\n\nclass VITAttentionRollout:\n    def __init__(self, model, attention_layer_name='attn_drop', head_fusion=\"mean\",\n        discard_ratio=0.7):\n        self.model = model\n        self.head_fusion = head_fusion\n        self.discard_ratio = discard_ratio\n        for name, module in self.model.named_modules():\n            if attention_layer_name in name:\n                module.register_forward_hook(self.get_attention)   #forward에서 어텐션맵 빼오는 예약 걸어둠. model에 input흘리면 실행\n\n        self.attentions = []\n\n    def get_attention(self, module, input, output):\n        self.attentions.append(output.cpu())\n\n    def __call__(self, input_tensor):\n        self.attentions = []\n        with torch.no_grad():\n            output = self.model(input_tensor)\n\n        return rollout(self.attentions, self.discard_ratio, self.head_fusion)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T13:00:08.601379Z","iopub.execute_input":"2026-06-29T13:00:08.601925Z","iopub.status.idle":"2026-06-29T13:00:08.618768Z","shell.execute_reply.started":"2026-06-29T13:00:08.601883Z","shell.execute_reply":"2026-06-29T13:00:08.617690Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# hook 직접 걸어서 호출되는지 테스트(훅 안걸리는 문제 발생)\nhandles = []\ntest_outputs = []\n\ndef test_hook(module, input, output):\n    test_outputs.append(output)\n    print(\"hook 호출됨!\", output.shape)\n\nfor name, module in model.named_modules():\n    if 'attn_drop' in name:\n        handles.append(module.register_forward_hook(test_hook))\n\nwith torch.no_grad():\n    output = model(input)\n\nprint(f\"총 호출 횟수: {len(test_outputs)}\")\n\n# hook 제거\nfor h in handles:\n    h.remove()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T13:00:08.620247Z","iopub.execute_input":"2026-06-29T13:00:08.621115Z","iopub.status.idle":"2026-06-29T13:00:09.648045Z","shell.execute_reply.started":"2026-06-29T13:00:08.621072Z","shell.execute_reply":"2026-06-29T13:00:09.647105Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#모델 forward에서 attn_drop계층이 호출되는지 확인\n#inspect는 소스 코드 자체를 텍스트로 보여주는 라이브러리\nimport inspect\nprint(inspect.getsource(model.blocks[0].attn.forward))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T13:00:09.649184Z","iopub.execute_input":"2026-06-29T13:00:09.650203Z","iopub.status.idle":"2026-06-29T13:00:09.657525Z","shell.execute_reply.started":"2026-06-29T13:00:09.650160Z","shell.execute_reply":"2026-06-29T13:00:09.656578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#F.scaled_dot_product_attention은 PyTorch 내장 커널이라 attention 행렬을 중간에 메모리에 올리지 않고 한 번에 처리. (FlashAttention 방식)\n#attention map을 잡을 수가 없다.\n#fused_attn을 false로 해주기\nfor block in model.blocks:\n    block.attn.fused_attn = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T13:00:09.658680Z","iopub.execute_input":"2026-06-29T13:00:09.659097Z","iopub.status.idle":"2026-06-29T13:00:09.672240Z","shell.execute_reply.started":"2026-06-29T13:00:09.659067Z","shell.execute_reply":"2026-06-29T13:00:09.671302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rollout_output=VITAttentionRollout(model)\nmask=rollout_output(input)\nprint(mask.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T13:00:09.673460Z","iopub.execute_input":"2026-06-29T13:00:09.673725Z","iopub.status.idle":"2026-06-29T13:00:11.146617Z","shell.execute_reply.started":"2026-06-29T13:00:09.673701Z","shell.execute_reply":"2026-06-29T13:00:11.145674Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Attention Map 시각화","metadata":{}},{"cell_type":"code","source":"from torch.nn import functional as F\n\nattn_map=F.interpolate(\n    input=torch.tensor(mask).unsqueeze(0).unsqueeze(0),   #F.interpolate는 [배치, 채널, H, W] 텐서 입력받음\n    size=(image.shape[0], image.shape[1]),\n    mode='bilinear'\n).squeeze().detach().numpy()\n\n\nplt.figure(figsize=(5,5))\nplt.imshow(image)\nplt.imshow(attn_map, cmap='jet', alpha=0.5)\n\nplt.axis('off')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T13:00:11.147818Z","iopub.execute_input":"2026-06-29T13:00:11.148237Z","iopub.status.idle":"2026-06-29T13:00:11.516154Z","shell.execute_reply.started":"2026-06-29T13:00:11.148196Z","shell.execute_reply":"2026-06-29T13:00:11.514690Z"}},"outputs":[],"execution_count":null}]}