{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Inspired from 1st team ([github](https://github.com/alipay/cvpr2020-plant-pathology)) in [Plant Pathology 2020 - FGVC7](http://www.kaggle.com/c/plant-pathology-2020-fgvc7), it is interesting to display the CAM to track model performance.\n\n### Thank my teammate @toxu for sharing me with this repo."},{"metadata":{},"cell_type":"markdown","source":"# What is Class Activation Map (CAM) ? \n\n### A Class Activation map for a particular category indicates the particular region used by CNN to identify the output class.\n\n### The CNN model is composed of numerous convolutionary layers and we perform global average pooling just before the final output layer. To get the desired output, the resulting features are fed to a fully connected layer with softmax activation. By projecting the output layer weights back into the convolutionary maps derived from the last Convolution Layer the importance of the image regions is identifiable.\n\nref: \n\nhttps://medium.com/intelligentmachines/implementation-of-class-activation-map-cam-with-pytorch-c32f7e414923\n\nhttps://arxiv.org/abs/1512.04150"},{"metadata":{},"cell_type":"markdown","source":"# Using general structure at this competition for easily understanding the pipeline"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n\nimport os\nimport math\nimport time\nimport random\nimport shutil\nimport albumentations\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nfrom scipy.special import softmax\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\n## replace it with your model path\nMODEL_DIR = '../input/cassava-resnext/' \n\nTRAIN_PATH = '../input/cassava-leaf-disease-classification/train_images'\nTEST_PATH = '../input/cassava-leaf-disease-classification/test_images'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\ntest['filepath'] = test.image_id.apply(lambda x: os.path.join('../input/cassava-leaf-disease-classification/test_images', f'{x}'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain['filepath'] = train.image_id.apply(lambda x: os.path.join('../input/cassava-leaf-disease-classification/train_images', f'{x}'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    num_workers=4\n    model_name='seresnext50_32x4d'\n    size=512\n    batch_size=1\n    target_size=5\n    target_col='label'\n    n_fold=5\n    trn_fold=[0, 1, 2, 3, 4]\n    seed=123","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TRAIN_PATH}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_transforms(*, data):\n    if data == 'no-tta':\n        return A.Compose([\n            A.Resize(CFG.size, CFG.size),\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n    \n    if data == 'tta':\n        return A.Compose([\n            RandomResizedCrop(CFG.size, CFG.size),\n#             A.Resize(CFG.size, CFG.size),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomResNext(nn.Module):\n    def __init__(self, model_name='resnext50_32x4d', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        if 'legacy' in model_name:\n            n_features = self.model.last_linear.in_features\n            self.model.last_linear = nn.Linear(n_features,CFG.target_size)\n        elif 'res' in model_name:\n            n_features = self.model.fc.in_features\n            self.model.fc = nn.Linear(n_features, CFG.target_size)\n        elif 'eff' in model_name:\n            n_features = self.model.classifier.in_features\n            self.model.classifier = nn.Linear(n_features, CFG.target_size)\n            \n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \nseed_everything(CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = TestDataset(test, transform=get_transforms(data='no-tta'))\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, \n                         num_workers=CFG.num_workers, pin_memory=True)\n\ntrain_dataset = TestDataset(train, transform=get_transforms(data='no-tta'))\ntrain_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=False, \n                         num_workers=CFG.num_workers, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = CustomResNext(CFG.model_name, pretrained=False)\nmodel.to(device)\n\n## load one of your model\nstate = torch.load(MODEL_DIR+f'{CFG.model_name}_fold_{0}')\nmodel.load_state_dict(state)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## remove gap and final linear layer\n## if you use eff or other models, this line would be different\n## try to change by yourself, and you will learn and understand\nmod = nn.Sequential(*list(model.model.children())[:-2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self):\n        super(Net,self).__init__() \n        self.fc=nn.Linear(2048,5)\n\n    \n    def forward(self,x):     \n        x=x.view(2048,16*16).mean(1).view(1,-1) ## for image size 512 x 512, the feature map size is 16 x 16\n        x=self.fc(x)\n        return  F.softmax(x,dim=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## concat feature map layer and linear layer\nmodel=nn.Sequential(mod,Net())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = list(Net().parameters())\nweight = np.squeeze(params[-2].data.numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params[-2].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def return_CAM(feature_conv, weight, class_idx):\n    size_upsample = (CFG.size,CFG.size)\n    bz, nc, h, w = feature_conv.shape\n    output_cam = []\n    for idx in class_idx:\n        beforeDot =  feature_conv.reshape((nc, h*w))\n        cam = np.matmul(weight[idx], beforeDot)\n        cam = cam.reshape(h, w)\n        cam = cam - np.min(cam)\n        cam_img = cam / np.max(cam)\n        cam_img = np.uint8(255 * cam_img)\n        output_cam.append(cv2.resize(cam_img, size_upsample))\n    return output_cam","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## for example, get some of images to show\nimgs = []\nfor i, (images) in enumerate(train_loader):\n    if i >= 10:\n        break\n    imgs.append(images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i = 7\nimg = imgs[i]\nmodel.to(device)\nimg = img.to(device)\nlogit = model(img)\n\nh_x = F.softmax(logit, dim=1).data.squeeze()\n \nprobs, idx = h_x.sort(0, True)\nprobs = probs.cpu().detach().numpy()\nidx = idx.cpu().numpy()\n\n\nfeatures_blobs = mod(img)\nfeatures_blobs1 = features_blobs.cpu().detach().numpy()\nCAMs = return_CAM(features_blobs1, weight, [idx[0]])\n\n\nimg = cv2.imread(train.loc[i, 'filepath'])\nheight, width, _ = img.shape\nheatmap = cv2.applyColorMap(cv2.resize(CAMs[0],(width, height)), cv2.COLORMAP_JET)\n\nresult = heatmap * 0.3 + img * 0.7\n# cv2.imwrite(\"sample.jpg\", result)\n\nplt.figure(figsize=(16,8))\nplt.imshow(result.astype('uint8'))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}