{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%pip install timm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-16T13:05:52.868020Z","iopub.execute_input":"2023-06-16T13:05:52.868437Z","iopub.status.idle":"2023-06-16T13:06:06.464229Z","shell.execute_reply.started":"2023-06-16T13:05:52.868411Z","shell.execute_reply":"2023-06-16T13:06:06.462911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport timm\nimport random\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nfrom torch.autograd import Function\nfrom torchvision import transforms\n\nplt.rcParams['figure.figsize'] = (20, 20)","metadata":{"execution":{"iopub.status.busy":"2023-06-16T13:07:38.093337Z","iopub.execute_input":"2023-06-16T13:07:38.093698Z","iopub.status.idle":"2023-06-16T13:07:39.686136Z","shell.execute_reply.started":"2023-06-16T13:07:38.093669Z","shell.execute_reply":"2023-06-16T13:07:39.685199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeatureExtractor():\n    '''Class for extracting activations and \n    registering gradients from targeted intermediate layers\n    '''\n    \n    def __init__(self, model, target_layers):\n        self.model = model\n        self.target_layers = target_layers\n        self.gradients = []\n        \n    def save_gradient(self, gradient):\n        self.gradients.append(gradient)\n        \n    def __call__(self, x):\n        outputs = []\n        self.gradients = []\n        for name, module in self.model._modules.items():\n            x = module(x)\n            if name in self.target_layers:\n                x.register_hook(self.save_gradient)\n                outputs += [x]\n        return outputs, x","metadata":{"execution":{"iopub.status.busy":"2023-06-16T13:12:42.649247Z","iopub.execute_input":"2023-06-16T13:12:42.649612Z","iopub.status.idle":"2023-06-16T13:12:42.657436Z","shell.execute_reply.started":"2023-06-16T13:12:42.649581Z","shell.execute_reply":"2023-06-16T13:12:42.656272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ModelOutputs():\n    '''Class for making a forward pass, and getting:\n    1. The network output\n    2. Activations from intermediate targeted layers\n    3. Gradients from intermediate targeted layers. '''\n    \n    def __init__(self, model, feature_model, target_layers):\n        self.model = model\n        self.feature_model = feature_model\n        self.feature_extractor = FeatureExtractor(self.feature_model, target_layers)\n        \n    def get_gradients(self):\n        return self.feature_extractor.gradients\n    \n    def __call__(self, x):\n        target_activations = []\n        for name, module in self.model._modules.items():\n            if module == self.feature_model:\n                target_activations, x = self.feature_extractor(x)\n            elif 'avgpool' in name.lower():\n                x = module(x)\n                x = x.view(x.size(0), -1)\n            else:\n                x = module(x)\n                \n        return target_activations, x","metadata":{"execution":{"iopub.status.busy":"2023-06-16T13:19:05.912263Z","iopub.execute_input":"2023-06-16T13:19:05.912619Z","iopub.status.idle":"2023-06-16T13:19:05.921847Z","shell.execute_reply.started":"2023-06-16T13:19:05.912590Z","shell.execute_reply":"2023-06-16T13:19:05.920910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_image(img):\n    normalize = transforms.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225]\n    )\n    preprocessing = transforms.Compose([\n        transforms.ToTensor(),\n        normalize\n    ])\n    return preprocessing(img.copy()).unsqueeze(0)","metadata":{"execution":{"iopub.status.busy":"2023-06-16T13:20:50.521427Z","iopub.execute_input":"2023-06-16T13:20:50.521787Z","iopub.status.idle":"2023-06-16T13:20:50.527346Z","shell.execute_reply.started":"2023-06-16T13:20:50.521757Z","shell.execute_reply":"2023-06-16T13:20:50.526403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_cam_on_image(img, mask):\n    heatmap = cv2.applyColorMap(np.uint8(255*mask), cv2.COLORMAP_JET)\n    heatmap = np.float32(heatmap) / 255\n    cam = heatmap + np.float32(img)\n    cam = cam / np.max(cam)\n    return np.uint8(255*cam)","metadata":{"execution":{"iopub.status.busy":"2023-06-16T13:22:26.186619Z","iopub.execute_input":"2023-06-16T13:22:26.186999Z","iopub.status.idle":"2023-06-16T13:22:26.192910Z","shell.execute_reply.started":"2023-06-16T13:22:26.186948Z","shell.execute_reply":"2023-06-16T13:22:26.191938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GradCam:\n    def __init__(self, model, feature_module, target_layer_names, use_cuda):\n        self.model = model\n        self.feature_module = feature_module\n        self.model.eval()\n        self.cuda = use_cuda\n        if self.cuda:\n            self.model = model.cuda()\n            \n        self.extractor = ModelOutputs(self.model, self.feature_module, target_layer_names)\n        \n    def forward(self, input_img):\n        return self.model(input_img)\n    \n    def __call__(self, input_img, target_category=None):\n        if self.cuda:\n            input_img = input_img.cuda()\n        \n        features, output = self.extractor(input_img)\n        \n        if target_category is None:\n            target_category = np.argmax(output.cpu().data.numpy())\n        \n        one_hot = np.zeros((1, output.size()[-1]), dtype=np.float32)\n        one_hot[0][target_category] = 1\n        one_hot = torch.from_numpy(one_hot).requires_grad_(True)\n        if self.cuda:\n            one_hot = one_hot.cuda()\n            \n        one_hot = torch.sum(one_hot*output)\n        \n        self.feature_module.zero_grad()\n        self.model.zero_grad()\n        one_hot.backward(retain_graph=True)\n        \n        grads_val = self.extractor.get_gradients()[-1].cpu().data.numpy()\n        \n        target = features[-1]\n        target = target.cpu().data.numpy()[0, :]\n        \n        weights = np.mean(grads_val, axis=(2, 3))[0, :]\n        cam = np.zeros(target.shape[1:], dtype=np.float32)\n        \n        for i, w in enumerate(weights):\n            cam += w*target[i, :, :]\n            \n        cam = np.maximum(cam, 0)\n        cam = cv2.resize(cam, input_img.shape[2:])\n        cam = cam - np.min(cam)\n        cam = cam / np.max(cam)\n        return cam","metadata":{"execution":{"iopub.status.busy":"2023-06-16T14:30:31.809843Z","iopub.execute_input":"2023-06-16T14:30:31.810444Z","iopub.status.idle":"2023-06-16T14:30:31.823901Z","shell.execute_reply.started":"2023-06-16T14:30:31.810411Z","shell.execute_reply":"2023-06-16T14:30:31.822916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GuidedBackPropReLU(Function):\n    @staticmethod\n    def forward(self, input_img):\n        positive_mask = (input_img > 0).type_as(input_img)\n        output = torch.addcmul(torch.zeros(input_img.size()).type_as(input_img), input_img, positive_mask)\n        self.save_for_backward(input_img, output)\n        return output\n    \n    @staticmethod\n    def backward(self, grad_output):\n        input_img, output = self.saved_tensors\n        grad_input = None\n        \n        positive_mask_1 = (input_img > 0).type_as(grad_output)\n        positive_mask_2 = (grad_output > 0).type_as(grad_output)\n        grad_input = torch.addcmul(\n            torch.zeros(input_img.size()).type_as(input_img),\n            torch.addcmul(\n                torch.zeros(input_img.size()).type_as(input_img), \n                grad_output, positive_mask_2\n            ), \n            positive_mask_2\n        )\n        return grad_input","metadata":{"execution":{"iopub.status.busy":"2023-06-16T14:30:33.186672Z","iopub.execute_input":"2023-06-16T14:30:33.187046Z","iopub.status.idle":"2023-06-16T14:30:33.195072Z","shell.execute_reply.started":"2023-06-16T14:30:33.187016Z","shell.execute_reply":"2023-06-16T14:30:33.193887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GuidedBackPropReLUModel:\n    def __init__(self, model, use_cuda):\n        self.model = model\n        self.model.eval()\n        self.cuda = use_cuda\n        if self.cuda:\n            self.model = model.cuda()\n        \n        def recursive_relu_apply(module_top):\n            for idx, module in module_top._modules.items():\n                recursive_relu_apply(module)\n                if module.__class__.__name__ == 'ReLU':\n                    module_top._modules[idx] = GuidedBackPropReLU.apply\n        \n        recursive_relu_apply(self.model)\n        \n    def forward(self, input_img):\n        return self.model(input_img)\n    \n    def __call__(self, input_img, target_category=None):\n        if self.cuda:\n            input_img = input_img.cuda()\n        \n        input_img = input_img.requires_grad_(True)\n        \n        output = self.forward(input_img)\n        \n        if target_category == None:\n            target_category = np.argmax(output.cpu().data.numpy())\n            \n        one_hot = np.zeros((1, output.size()[-1]), dtype=np.float32)\n        one_hot[0][target_category] = 1\n        one_hot = torch.from_numpy(one_hot).requires_grad_(True)\n        if self.cuda:\n            one_hot = one_hot.cuda()\n            \n        one_hot = torch.sum(one_hot*output)\n        one_hot.backward(retain_graph=True)\n        \n        output = input_img.grad.cpu().data.numpy()\n        output = output[0, :, :, :]\n        \n        return output","metadata":{"execution":{"iopub.status.busy":"2023-06-16T14:30:33.768698Z","iopub.execute_input":"2023-06-16T14:30:33.769073Z","iopub.status.idle":"2023-06-16T14:30:33.780540Z","shell.execute_reply.started":"2023-06-16T14:30:33.769042Z","shell.execute_reply":"2023-06-16T14:30:33.779294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def deprocess_image(img):\n    img = img - np.mean(img)\n    img = img / (np.std(img) + 1e-5)\n    img = img * 0.1\n    img = img + 0.5\n    img = np.clip(img, 0, 1)\n    return np.uint8(img*255)","metadata":{"execution":{"iopub.status.busy":"2023-06-16T14:30:34.622430Z","iopub.execute_input":"2023-06-16T14:30:34.622787Z","iopub.status.idle":"2023-06-16T14:30:34.628640Z","shell.execute_reply.started":"2023-06-16T14:30:34.622756Z","shell.execute_reply":"2023-06-16T14:30:34.627433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"use_cuda = torch.cuda.is_available()","metadata":{"execution":{"iopub.status.busy":"2023-06-16T14:30:35.573669Z","iopub.execute_input":"2023-06-16T14:30:35.574088Z","iopub.status.idle":"2023-06-16T14:30:35.581085Z","shell.execute_reply.started":"2023-06-16T14:30:35.574054Z","shell.execute_reply":"2023-06-16T14:30:35.580019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = timm.create_model('tf_efficientnet_b4.ns_jft_in1k', pretrained=False)\nnum_features = model.classifier.in_features\nmodel.classifier = nn.Linear(num_features, 5)\nif use_cuda:\n    model = model.cuda()","metadata":{"execution":{"iopub.status.busy":"2023-06-16T14:30:36.422678Z","iopub.execute_input":"2023-06-16T14:30:36.423055Z","iopub.status.idle":"2023-06-16T14:30:36.841830Z","shell.execute_reply.started":"2023-06-16T14:30:36.423024Z","shell.execute_reply":"2023-06-16T14:30:36.840851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model","metadata":{"execution":{"iopub.status.busy":"2023-06-16T14:30:36.881196Z","iopub.execute_input":"2023-06-16T14:30:36.881492Z","iopub.status.idle":"2023-06-16T14:30:36.900040Z","shell.execute_reply.started":"2023-06-16T14:30:36.881468Z","shell.execute_reply":"2023-06-16T14:30:36.897940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize the 6th Convolutional Black\n\nIMAGE_PATH = '/kaggle/input/cassava-leaf-disease-classification/train_images/1005138819.jpg'\n\nimg = cv2.imread(IMAGE_PATH, 1)\nimg = np.float32(img) / 255\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\ninput_img = preprocess_image(img)\n\ngrad_cam = GradCam(model=model, feature_module=model.blocks, target_layer_names=['6'], use_cuda=use_cuda)\n\ntarget_category = None\ngrayscale_cam = grad_cam(input_img, target_category)\n\ngrayscale_cam = cv2.resize(grayscale_cam, (img.shape[1], img.shape[0]))\ncam = show_cam_on_image(img, grayscale_cam)\n\ngb_model = GuidedBackPropReLUModel(model=model, use_cuda=use_cuda)\ngb = gb_model(input_img, target_category=target_category)\ngb = gb.transpose((1, 2, 0))\n\ncam_mask = cv2.merge([grayscale_cam, grayscale_cam, grayscale_cam])\ncam_gb = deprocess_image(cam_mask*gb)\ngb = deprocess_image(gb)\n\ncv2.imwrite('cam.jpg', cam)\ncv2.imwrite('gb.jpg', gb)\ncv2.imwrite('cam_gb.jpg', cam_gb)","metadata":{"execution":{"iopub.status.busy":"2023-06-16T14:38:17.194406Z","iopub.execute_input":"2023-06-16T14:38:17.194767Z","iopub.status.idle":"2023-06-16T14:38:17.550403Z","shell.execute_reply.started":"2023-06-16T14:38:17.194740Z","shell.execute_reply":"2023-06-16T14:38:17.549480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# GradCAM\n\nfig, ax = plt.subplots(1, 2)\nimg = cv2.imread(IMAGE_PATH)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\ncam = cv2.imread('./cam.jpg')\ncam = cv2.cvtColor(cam, cv2.COLOR_BGR2RGB)\nax[0].imshow(img)\nax[1].imshow(cam)","metadata":{"execution":{"iopub.status.busy":"2023-06-16T14:38:18.108652Z","iopub.execute_input":"2023-06-16T14:38:18.109042Z","iopub.status.idle":"2023-06-16T14:38:19.224540Z","shell.execute_reply.started":"2023-06-16T14:38:18.109009Z","shell.execute_reply":"2023-06-16T14:38:19.223685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,2)\nimg = cv2.imread(IMAGE_PATH)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\ngb = cv2.imread(\"./gb.jpg\")\ngb = cv2.cvtColor(gb, cv2.COLOR_BGR2RGB)\nax[0].imshow(img)\nax[1].imshow(gb)","metadata":{"execution":{"iopub.status.busy":"2023-06-16T14:38:22.893282Z","iopub.execute_input":"2023-06-16T14:38:22.893637Z","iopub.status.idle":"2023-06-16T14:38:23.765735Z","shell.execute_reply.started":"2023-06-16T14:38:22.893608Z","shell.execute_reply":"2023-06-16T14:38:23.764966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Combined Grad-CAM with Guided Backpropagation\n\nfig, ax = plt.subplots(1,2)\nimg = cv2.imread(IMAGE_PATH)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\ncam_gb = cv2.imread(\"./cam_gb.jpg\")\ncam_gb = cv2.cvtColor(cam_gb, cv2.COLOR_BGR2RGB)\nax[0].imshow(img)\nax[1].imshow(cam_gb)","metadata":{"execution":{"iopub.status.busy":"2023-06-16T14:38:26.103624Z","iopub.execute_input":"2023-06-16T14:38:26.104451Z","iopub.status.idle":"2023-06-16T14:38:27.262444Z","shell.execute_reply.started":"2023-06-16T14:38:26.104404Z","shell.execute_reply":"2023-06-16T14:38:27.261626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}