{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Class Activation Map\n\nHi everyone,\n\nhere I implemented the paper that introduced Class Action Map (CAM) https://arxiv.org/pdf/1512.04150.pdf. \n\nThis is a simple method that computes the heatmap, at the pixel level, where the network in paying attention in order to make the prediction. I think it can be useful for this challenge, since most of the solutions I'm seeing here use the spectrogram (and variants such as mel-spectrograms) that can be considered as images. In particular this tool can be helpful for understanding the data and debugging the model.\n\nThis is a PyTorch implementation, hope this can help!\n\nGuglielmo"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"##################################################\n# Imports\n##################################################\n\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport torch\nfrom torchvision import models, transforms\nimport torch.nn.functional as F\nimport numpy as np\nimport skimage.transform\n\n# Download an image\n!wget https://img.webmd.com/dtmcms/live/webmd/consumer_assets/site_images/article_thumbnails/other/cat_relaxing_on_patio_other/1800x1200_cat_relaxing_on_patio_other.jpg -O cat.jpg >/dev/null 2>&1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Show the image\nimage = Image.open(\"cat.jpg\")\nplt.figure(figsize=(15, 10))\nplt.imshow(image)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##################################################\n# Preprocessing\n##################################################\n\n# Imagenet mean/std\nnormalize = transforms.Normalize(\n   mean=[0.485, 0.456, 0.406],\n   std=[0.229, 0.224, 0.225]\n)\n\n# Scale to 224x224, convert to tensor, and normalize with mean/std for ImageNet\npreprocess = transforms.Compose([\n   transforms.Resize((224, 224)),\n   transforms.ToTensor(),\n   normalize,\n])\n\nx_img = preprocess(image).unsqueeze(0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Using CAM on your model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model\nmodel = models.resnet18(pretrained=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get the features from a model\nclass SaveFeatures():\n    features = None\n    def __init__(self, module): \n        self.hook = module.register_forward_hook(self.hook_fn)\n\n    def hook_fn(self, module, input, output): \n        self.features = output.data.numpy()\n\n    def remove(self): \n        self.hook.remove()\n\ndef getCAM(feature_conv, weight_fc, class_idx):\n    _, nc, h, w = feature_conv.shape\n    cam = weight_fc[class_idx].dot(feature_conv.reshape((nc, h * w)))\n    cam = cam.reshape(h, w)\n    cam = cam - np.min(cam)\n    cam_img = cam / np.max(cam)\n    return [cam_img]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get features from last conv layer\nfinal_layer = model._modules.get('layer4')\nactivated_features = SaveFeatures(final_layer)\n\n# Inference\n_ = model.eval()\nprediction = model(x_img)\npred_probabilities = F.softmax(prediction).data.squeeze()\nactivated_features.remove()\nprint('Top-1 prediction:', torch.topk(pred_probabilities, 1))\n\n# Take weights from the first linear layer\nweight_softmax_params = list(model._modules.get('fc').parameters())\nweight_softmax = np.squeeze(weight_softmax_params[0].data.numpy())\n\n# Get the top-1 prediction and get CAM\nclass_idx = torch.topk(pred_probabilities, 1)[1].int()\noverlay = getCAM(activated_features.features, weight_softmax, class_idx )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show CAM\nplt.figure(figsize=(5, 5))\nplt.title('Class Activation Map', fontweight='bold')\nplt.imshow(overlay[0], alpha=0.5, cmap='jet')\n\n# Show CAM on the image\nplt.figure(figsize=(15, 10))\nplt.title('Class Activation Map on the Image', fontweight='bold')\nplt.imshow(image)\nplt.imshow(skimage.transform.resize(overlay[0], (image.size[1], image.size[0])), alpha=0.5, cmap='jet');\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}