{"cells":[{"metadata":{},"cell_type":"markdown","source":"![](https://glassboxmedicine.files.wordpress.com/2020/05/example-cbam.png?w=966)"},{"metadata":{},"cell_type":"markdown","source":"<link rel=\"preconnect\" href=\"https://fonts.gstatic.com\">\n<link href=\"https://fonts.googleapis.com/css2?family=Kanit&display=swap\" rel=\"stylesheet\">\n\n<h1 style=\"text-align:center; font-family: 'Kanit', sans-serif;\"> Visualize what your CNN Learns</h1>"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport timm\nimport random\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nfrom torch.autograd import Function\nfrom torchvision import transforms\n\nplt.rcParams[\"figure.figsize\"] = (20,20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### The following code for Grad-CAM and Guided Backpropagation is borrowed from this amazing repository [https://github.com/jacobgil/pytorch-grad-cam](https://github.com/jacobgil/pytorch-grad-cam)"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FeatureExtractor():\n    \"\"\" Class for extracting activations and\n    registering gradients from targetted intermediate layers \"\"\"\n\n    def __init__(self, model, target_layers):\n        self.model = model\n        self.target_layers = target_layers\n        self.gradients = []\n\n    def save_gradient(self, grad):\n        self.gradients.append(grad)\n\n    def __call__(self, x):\n        outputs = []\n        self.gradients = []\n        for name, module in self.model._modules.items():\n            x = module(x)\n            if name in self.target_layers:\n                x.register_hook(self.save_gradient)\n                outputs += [x]\n        return outputs, x\n\nclass ModelOutputs():\n    \"\"\" Class for making a forward pass, and getting:\n    1. The network output.\n    2. Activations from intermeddiate targetted layers.\n    3. Gradients from intermeddiate targetted layers. \"\"\"\n\n    def __init__(self, model, feature_module, target_layers):\n        self.model = model\n        self.feature_module = feature_module\n        self.feature_extractor = FeatureExtractor(self.feature_module, target_layers)\n\n    def get_gradients(self):\n        return self.feature_extractor.gradients\n\n    def __call__(self, x):\n        target_activations = []\n        for name, module in self.model._modules.items():\n            if module == self.feature_module:\n                target_activations, x = self.feature_extractor(x)\n            elif \"avgpool\" in name.lower():\n                x = module(x)\n                x = x.view(x.size(0),-1)\n            else:\n                x = module(x)\n\n        return target_activations, x\n\ndef preprocess_image(img):\n    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225])\n    preprocessing = transforms.Compose([\n        transforms.ToTensor(),\n        normalize,\n    ])\n    return preprocessing(img.copy()).unsqueeze(0)\n\ndef show_cam_on_image(img, mask):\n    heatmap = cv2.applyColorMap(np.uint8(255 * mask), cv2.COLORMAP_JET)\n    heatmap = np.float32(heatmap) / 255\n    cam = heatmap + np.float32(img)\n    cam = cam / np.max(cam)\n    return np.uint8(255 * cam)\n\nclass GradCam:\n    def __init__(self, model, feature_module, target_layer_names, use_cuda):\n        self.model = model\n        self.feature_module = feature_module\n        self.model.eval()\n        self.cuda = use_cuda\n        if self.cuda:\n            self.model = model.cuda()\n\n        self.extractor = ModelOutputs(self.model, self.feature_module, target_layer_names)\n\n    def forward(self, input_img):\n        return self.model(input_img)\n\n    def __call__(self, input_img, target_category=None):\n        if self.cuda:\n            input_img = input_img.cuda()\n\n        features, output = self.extractor(input_img)\n\n        if target_category == None:\n            target_category = np.argmax(output.cpu().data.numpy())\n\n        one_hot = np.zeros((1, output.size()[-1]), dtype=np.float32)\n        one_hot[0][target_category] = 1\n        one_hot = torch.from_numpy(one_hot).requires_grad_(True)\n        if self.cuda:\n            one_hot = one_hot.cuda()\n        \n        one_hot = torch.sum(one_hot * output)\n\n        self.feature_module.zero_grad()\n        self.model.zero_grad()\n        one_hot.backward(retain_graph=True)\n\n        grads_val = self.extractor.get_gradients()[-1].cpu().data.numpy()\n\n        target = features[-1]\n        target = target.cpu().data.numpy()[0, :]\n\n        weights = np.mean(grads_val, axis=(2, 3))[0, :]\n        cam = np.zeros(target.shape[1:], dtype=np.float32)\n\n        for i, w in enumerate(weights):\n            cam += w * target[i, :, :]\n\n        cam = np.maximum(cam, 0)\n        cam = cv2.resize(cam, input_img.shape[2:])\n        cam = cam - np.min(cam)\n        cam = cam / np.max(cam)\n        return cam\n\n\nclass GuidedBackpropReLU(Function):\n    @staticmethod\n    def forward(self, input_img):\n        positive_mask = (input_img > 0).type_as(input_img)\n        output = torch.addcmul(torch.zeros(input_img.size()).type_as(input_img), input_img, positive_mask)\n        self.save_for_backward(input_img, output)\n        return output\n\n    @staticmethod\n    def backward(self, grad_output):\n        input_img, output = self.saved_tensors\n        grad_input = None\n\n        positive_mask_1 = (input_img > 0).type_as(grad_output)\n        positive_mask_2 = (grad_output > 0).type_as(grad_output)\n        grad_input = torch.addcmul(torch.zeros(input_img.size()).type_as(input_img),\n                                   torch.addcmul(torch.zeros(input_img.size()).type_as(input_img), grad_output,\n                                                 positive_mask_1), positive_mask_2)\n        return grad_input\n\n\nclass GuidedBackpropReLUModel:\n    def __init__(self, model, use_cuda):\n        self.model = model\n        self.model.eval()\n        self.cuda = use_cuda\n        if self.cuda:\n            self.model = model.cuda()\n\n        def recursive_relu_apply(module_top):\n            for idx, module in module_top._modules.items():\n                recursive_relu_apply(module)\n                if module.__class__.__name__ == 'ReLU':\n                    module_top._modules[idx] = GuidedBackpropReLU.apply\n\n        # replace ReLU with GuidedBackpropReLU\n        recursive_relu_apply(self.model)\n\n    def forward(self, input_img):\n        return self.model(input_img)\n\n    def __call__(self, input_img, target_category=None):\n        if self.cuda:\n            input_img = input_img.cuda()\n\n        input_img = input_img.requires_grad_(True)\n\n        output = self.forward(input_img)\n\n        if target_category == None:\n            target_category = np.argmax(output.cpu().data.numpy())\n\n        one_hot = np.zeros((1, output.size()[-1]), dtype=np.float32)\n        one_hot[0][target_category] = 1\n        one_hot = torch.from_numpy(one_hot).requires_grad_(True)\n        if self.cuda:\n            one_hot = one_hot.cuda()\n\n        one_hot = torch.sum(one_hot * output)\n        one_hot.backward(retain_graph=True)\n\n        output = input_img.grad.cpu().data.numpy()\n        output = output[0, :, :, :]\n\n        return output\n\ndef deprocess_image(img):\n    \"\"\" see https://github.com/jacobgil/keras-grad-cam/blob/master/grad-cam.py#L65 \"\"\"\n    img = img - np.mean(img)\n    img = img / (np.std(img) + 1e-5)\n    img = img * 0.1\n    img = img + 0.5\n    img = np.clip(img, 0, 1)\n    return np.uint8(img*255)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use_cuda = torch.cuda.is_available()\nMODEL_PATH = \"../input/tf-effnet-b4-ns/Fold4_0.8920308483290489_epoch7.bin\"\nIMAGE_PATH = \"../input/cassava-leaf-disease-classification/train_images/100042118.jpg\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Uncomment the following Code to get random images"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TRAIN_DIR = \"../input/cassava-leaf-disease-classification/train_images\"\n# train_images = os.listdir(TRAIN_DIR)\n# IMG = random.choice(train_images)\n# IMAGE_PATH = os.path.join(TRAIN_DIR, IMG)\n# print(IMAGE_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = timm.create_model(\"tf_efficientnet_b4_ns\", pretrained=False)\nnum_features = model.classifier.in_features\nmodel.classifier = nn.Linear(num_features, 5)\nif use_cuda:\n    model.load_state_dict(torch.load(MODEL_PATH))\nelse:\n    model.load_state_dict(torch.load(MODEL_PATH, map_location=torch.device('cpu')));","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Let's Visualize the 6th Convolutional Block"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" \n1. Loads an image with opencv.\n2. Preprocesses it and converts to a pytorch variable.\n3. Makes a forward pass to find the category index with the highest score,\nand computes intermediate activations.\nMakes the visualization. \n\n\"\"\"\n\nimg = cv2.imread(IMAGE_PATH, 1)\nimg = np.float32(img) / 255\n# Opencv loads as BGR:\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\ninput_img = preprocess_image(img)\n\ngrad_cam = GradCam(model=model, feature_module=model.blocks,\n                   target_layer_names=[\"6\"], use_cuda=use_cuda)\n\n# If None, returns the map for the highest scoring category.\n# Otherwise, targets the requested category.\ntarget_category = None\ngrayscale_cam = grad_cam(input_img, target_category)\n\ngrayscale_cam = cv2.resize(grayscale_cam, (img.shape[1], img.shape[0]))\ncam = show_cam_on_image(img, grayscale_cam)\n\ngb_model = GuidedBackpropReLUModel(model=model, use_cuda=use_cuda)\ngb = gb_model(input_img, target_category=target_category)\ngb = gb.transpose((1, 2, 0))\n\ncam_mask = cv2.merge([grayscale_cam, grayscale_cam, grayscale_cam])\ncam_gb = deprocess_image(cam_mask*gb)\ngb = deprocess_image(gb)\n\ncv2.imwrite(\"cam.jpg\", cam)\ncv2.imwrite('gb.jpg', gb)\ncv2.imwrite('cam_gb.jpg', cam_gb);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Grad-CAM"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2)\nimg = cv2.imread(IMAGE_PATH)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\ncam = cv2.imread(\"./cam.jpg\")\ncam = cv2.cvtColor(cam, cv2.COLOR_BGR2RGB)\nax[0].imshow(img)\nax[1].imshow(cam);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Guided Backpropagation"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2)\nimg = cv2.imread(IMAGE_PATH)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\ngb = cv2.imread(\"./gb.jpg\")\ngb = cv2.cvtColor(gb, cv2.COLOR_BGR2RGB)\nax[0].imshow(img)\nax[1].imshow(gb);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Combined Grad-CAM with Guided Backpropagation"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2)\nimg = cv2.imread(IMAGE_PATH)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\ncam_gb = cv2.imread(\"./cam_gb.jpg\")\ncam_gb = cv2.cvtColor(cam_gb, cv2.COLOR_BGR2RGB)\nax[0].imshow(img)\nax[1].imshow(cam_gb);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h3> If you liked my kernel, consider <span style=\"color:green\">UPVOTING</span> it :)</h3>"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}