{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Installing necessary libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"! /opt/conda/bin/python3.7 -m pip install -q --upgrade pip\n! pip install -q timm catalyst\n! pip install -q --upgrade wandb\n! pip install -q pytorch-gradcam","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! wandb login e1da498db2dd649a76a04c6e4743e5a4f95a2ae0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ! cp -r ../input/leaf-disease-classification-with-resnet34-train/model_dir ./\n# ! cp -r ../input/leaf-disease-classification-with-resnet34-train/history_dir ./","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Config\nThis section contains configuration parameters for my classification pipeline."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import cv2\nimport pandas as pd\nimport albumentations as A\nfrom albumentations.augmentations.transforms import Equalize, Posterize, Downscale\nfrom albumentations import (\n    PadIfNeeded, HorizontalFlip, VerticalFlip, CenterCrop,    \n    RandomCrop, Resize, Crop, Compose, HueSaturationValue,\n    Transpose, RandomRotate90, ElasticTransform, GridDistortion, \n    OpticalDistortion, RandomSizedCrop, Resize, CenterCrop,\n    VerticalFlip, HorizontalFlip, OneOf, CLAHE, Normalize,\n    RandomBrightnessContrast, Cutout, RandomGamma, ShiftScaleRotate ,\n    GaussNoise, Blur, MotionBlur, GaussianBlur, \n)\n\nSEED = 24\nn_epochs = 30\ndevice = 'cuda:0'\ndata_dir = '../input/cassava-leaf-disease-merged/'\nloss_thr = 1e6\nimg_path = f'{data_dir}/train'\ndf = pd.read_csv(f'{data_dir}/merged.csv')\ndf['path'] = df['image_id'].map(lambda x: f\"{img_path}/{x}\")\n# encoder_model = 'tf_efficientnet_b4_ns'\nencoder_model = 'gluon_resnext101_32x4d'\nfold = 0\nmodel_name= f'Resnext101_fold{fold}' # Will come up with a better name later\nmodel_dir = 'model_dir'\nhistory_dir = 'history_dir'\nload_model = False\nimg_dim = 384\nbatch_size = 32\naccum_step = 1\nlearning_rate = 3.50e-3\nnum_workers = 4\nmixed_precision = True\npatience = 3\nbalanced_sampler = False\ntrain_aug = A.Compose([A.CenterCrop(p=0.3, height=int(0.7*img_dim), width=int(0.7*img_dim)),\nA.augmentations.transforms.RandomCrop(int(0.7*img_dim), int(0.7*img_dim), p=0.3),\nA.augmentations.transforms.Rotate(limit=30, interpolation=1, border_mode=4, value=None, mask_value=None, always_apply=False, p=0.5),\nA.augmentations.transforms.Resize(img_dim, img_dim, interpolation=1, always_apply=True, p=0.6),\nCutout(num_holes=8, max_h_size=20, max_w_size=20, fill_value=0, always_apply=False, p=0.2),\nA.RandomBrightnessContrast(brightness_limit=0.3, contrast_limit=0.3, brightness_by_max=True, always_apply=False, p=0.3),\nA.augmentations.transforms.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=20, always_apply=False, p=0.4),\n# A.HorizontalFlip(p=0.5),\nA.VerticalFlip(p=0.5),                    \nOneOf([\n        GaussNoise(var_limit=0.1),\n        Blur(),\n        GaussianBlur(blur_limit=3),\n        # RandomGamma(p=0.7),\n        ], p=0.3),\nA.HorizontalFlip(p=0.3), Normalize(always_apply=True)],)\nval_aug = Compose([Normalize(always_apply=True)])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.label.hist()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fixing Seed"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport torch\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Sample Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib import pyplot as plt\n\n%matplotlib inline\nfig = plt.figure(figsize=(60, 60))\nfor class_id in sorted(df['label'].unique()):\n    for i, (idx, row) in enumerate(df.loc[df['label'] == class_id].sample(3, random_state=SEED).iterrows()):\n        ax = fig.add_subplot(5, 5, class_id * 5 + i + 1, xticks=[], yticks=[])\n        path= row['path']\n        image = cv2.imread(path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (320, 320))\n        plt.imshow(image)\n        ax.set_title('Label: %s Name: %s' % (row['label'], row['image_id']), fontsize=30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1 = df[df['source']==2020]\ndf2 = df[df['source']==2019]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Stratification"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split, KFold, StratifiedKFold\nskf = StratifiedKFold(n_splits=5, random_state=SEED)\nX = df1['path']\ny = df1['label']\ntrain_idx = []\nval_idx = []\n# for i, (train_index, val_index) in enumerate(skf.split(X, y)):\n#     train_idx = train_index\n#     val_idx = val_index\n#     break\n\ndf1['fold'] = np.nan\ndf2['fold'] = np.nan\ndf2['fold'] = df2['fold'].map(lambda x: fold)\n\ndf1= df1.sample(frac=1, random_state=SEED).reset_index(drop=True)\n#split data\n# mskf = MultilabelStratifiedKFold(n_splits=n_fold, random_state=SEED)\nfor i, (_, test_index) in enumerate(skf.split(X, y)):\n    df1.loc[test_index, 'fold'] = i\n    \ndf1['fold'] = df1['fold'].astype('int')\n\n# train_df = df.loc[train_idx]\n# valid_df = df.loc[val_idx]\nvalid_df = df1[df1['fold']==fold]\ntrain_df1 = df1[df1['fold']!=fold]\ntrain_df = pd.concat([train_df1, df2])\nprint(len(train_df))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset,DataLoader\n\ndef onehot(size, target, one_hot=False):\n    if not one_hot: return target\n    else:\n        vec = torch.zeros(size, dtype=torch.float32)\n        vec[target] = 1.\n        return vec\n\n\nclass LeafDataset(Dataset):\n    def __init__(self, df, dim=256, transforms=None):\n        super().__init__()\n        self.image_ids = df.path.tolist()\n        try:\n            self.labels = df.label.tolist()\n        except:\n            self.labels = None\n        self.transforms = transforms\n        self.dim = dim\n        \n    def __getitem__(self, idx):\n        image_id = self.image_ids[idx]\n        image = cv2.imread(image_id, cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (self.dim, self.dim))\n        \n        if self.transforms is not None:\n            aug = self.transforms(image=image)\n            image = aug['image'].reshape(self.dim, self.dim, 3).transpose(2, 0, 1)\n        else:\n            image = image.reshape(self.dim, self.dim, 3).transpose(2, 0, 1)\n        if self.labels is not None:\n            target = self.labels[idx]\n            return image_id, image, onehot(5, target, False)\n        else:\n            return image_id, image\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def get_labels(self):\n        return list(self.labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nimport torch\nfrom torch import nn\nfrom torch.nn import *\nfrom torch.nn import functional as F\nfrom torchvision import models\nfrom typing import Optional\n\n\nclass SwishImplementation(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, i):\n        result = i * torch.sigmoid(i)\n        ctx.save_for_backward(i)\n        return result\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        i = ctx.saved_variables[0]\n        sigmoid_i = torch.sigmoid(i)\n        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))\n\n\nclass MemoryEfficientSwish(nn.Module):\n    def forward(self, x):\n        return SwishImplementation.apply(x)\n\nclass Swish(nn.Module):\n    def forward(self, x):\n        return x * torch.sigmoid(x)\n\nclass AdaptiveConcatPool2d(Module):\n    \"Layer that concats `AdaptiveAvgPool2d` and `AdaptiveMaxPool2d`.\"\n    def __init__(self, sz=None):\n        super(AdaptiveConcatPool2d, self).__init__()\n        \"Output will be 2*sz or 2 if sz is None\"\n        self.output_size = sz or 1\n        self.ap = nn.AdaptiveAvgPool2d(self.output_size)\n        self.mp = nn.AdaptiveMaxPool2d(self.output_size)\n\n    def forward(self, x):\n        return torch.cat([self.mp(x), self.ap(x)], 1)\n\ndef bn_drop_lin(n_in:int, n_out:int, bn:bool=True, p:float=0., actn:Optional[nn.Module]=None):\n    \"Sequence of batchnorm (if `bn`), dropout (with `p`) and linear (`n_in`,`n_out`) layers followed by `actn`.\"\n    layers = [nn.BatchNorm1d(n_in)] if bn else []\n    if p != 0: layers.append(nn.Dropout(p))\n    layers.append(nn.Linear(n_in, n_out))\n    if actn is not None: layers.append(actn)\n    return layers\n\nclass Flatten(Module):\n    \"Flatten `x` to a single dimension, often used at the end of a model. `full` for rank-1 tensor\"\n    def __init__(self, full:bool=False): \n        super(Flatten,self).__init__()\n        self.full = full\n    def forward(self, x):\n        return x.view(-1) if self.full else x.view(x.size(0), -1)\n\ndef gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM,self).__init__()\n        self.p = Parameter(torch.ones(1)*p)\n        self.eps = eps\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)       \n    def __repr__(self):\n        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'\n\n\nclass MishFunction(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, x):\n        ctx.save_for_backward(x)\n        return x * torch.tanh(F.softplus(x))   # x * tanh(ln(1 + exp(x)))\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        x = ctx.saved_variables[0]\n        sigmoid = torch.sigmoid(x)\n        tanh_sp = torch.tanh(F.softplus(x)) \n        return grad_output * (tanh_sp + x * sigmoid * (1 - tanh_sp * tanh_sp))\n\nclass Mish(nn.Module):\n    def forward(self, x):\n        return MishFunction.apply(x)\n\ndef to_Mish(model):\n    for child_name, child in model.named_children():\n        if isinstance(child, nn.ReLU):\n            setattr(model, child_name, Mish())\n        # if isinstance(child, utils.MemoryEfficientSwish):\n        #     setattr(model, child_name, Mish())\n        else:\n            to_Mish(child)\n\ndef to_GeM(model):\n    for child_name, child in model.named_children():\n        if isinstance(child, nn.ReLU):\n            setattr(model, child_name, GeM())\n\nclass Head(nn.Module):\n    def __init__(self, nc, n, ps=0.5, activation='swish'):\n        super().__init__()\n        if activation=='mish':\n            layers = [AdaptiveConcatPool2d(), Mish(), Flatten()]\n        else:\n            layers = [GeM(), Swish(), Flatten()] \n            \n        layers += \\\n        bn_drop_lin(nc, 256, True, ps, Swish()) + \\\n        bn_drop_lin(256, n, True, ps)\n\n        self.fc = nn.Sequential(*layers)\n        \n        self._init_weight()\n        \n    def _init_weight(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                torch.nn.init.kaiming_normal_(m.weight)\n            elif isinstance(m, nn.BatchNorm2d):\n                m.weight.data.fill_(1.0)\n                m.bias.data.zero_()\n        \n    def forward(self, x, meta_data=None):\n        return self.fc(x)\n\nclass Conv2dStaticSamePadding(nn.Conv2d):\n    \"\"\" 2D Convolutions like TensorFlow, for a fixed image size\"\"\"\n\n    def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):\n        super().__init__(in_channels, out_channels, kernel_size, **kwargs)\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2\n\n        # Calculate padding based on image size and save it\n        assert image_size is not None\n        ih, iw = image_size if type(image_size) == list else [image_size, image_size]\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            self.static_padding = nn.ZeroPad2d((pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))\n        else:\n            self.static_padding = Identity()\n\n    def forward(self, x):\n        x = self.static_padding(x)\n        x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n        return x\n\nclass AttentionBlock(nn.Module):\n    def __init__(self, in_features_l, in_features_g, attn_features, up_factor, normalize_attn=True):\n        super(AttentionBlock, self).__init__()\n        self.up_factor = up_factor\n        self.normalize_attn = normalize_attn\n        self.W_l = nn.Conv2d(in_channels=in_features_l, out_channels=attn_features, kernel_size=1, padding=0, bias=False)\n        self.W_g = nn.Conv2d(in_channels=in_features_g, out_channels=attn_features, kernel_size=1, padding=0, bias=False)\n        self.phi = nn.Conv2d(in_channels=attn_features, out_channels=1, kernel_size=1, padding=0, bias=True)\n    \n    def forward(self, l, g):\n        N, C, W, H = l.size()\n        l_ = self.W_l(l)\n        g_ = self.W_g(g)\n        if self.up_factor > 1:\n            g_ = F.interpolate(g_, scale_factor=self.up_factor, mode='bilinear', align_corners=False)\n        c = self.phi(F.relu(l_ + g_)) # batch_sizex1xWxH\n        \n        # compute attn map\n        if self.normalize_attn:\n            a = F.softmax(c.view(N,1,-1), dim=2).view(N,1,W,H)\n        else:\n            a = torch.sigmoid(c)\n        # re-weight the local feature\n        f = torch.mul(a.expand_as(l), l) # batch_sizexCxWxH\n        if self.normalize_attn:\n            output = f.view(N,C,-1).sum(dim=2) # weighted sum\n        else:\n            output = F.adaptive_avg_pool2d(f, (1,1)).view(N,C) # global average pooling\n        return a, output","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Resnest"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch import nn\nfrom torch.nn import *\nfrom torch.nn import functional as F\nfrom torchvision import models\n# import timm\n\nclass Resnest(nn.Module):\n\n    def __init__(self, model_name='resnest50_fast_1s1x64d'):\n        super().__init__()\n        try:\n            self.backbone = timm.create_model(model_name, pretrained=True)\n        except:\n            self.backbone = torch.hub.load('zhanghang1989/ResNeSt', model_name, pretrained=True)\n        self.in_features = self.backbone.fc.in_features\n        self.output = Head(self.in_features, 5)\n\n    def forward(self, x, meta_data=None):\n        x = self.backbone.conv1(x)\n        x = self.backbone.bn1(x)\n        try:\n            x = self.backbone.act1(x)\n        except:\n            x = self.backbone.relu(x)\n        x = self.backbone.maxpool(x)\n\n        x = self.backbone.layer1(x)\n        x = self.backbone.layer2(x)\n        \n        x = self.backbone.layer3(x)\n        x = self.backbone.layer4(x)\n#         x = self.backbone.avgpool(x)\n#         x = x.view(x.size(0), -1)\n        x = self.output(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Attention Resnet"},{"metadata":{},"cell_type":"markdown","source":"#### Attention Block"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AttentionBlock(nn.Module):\n    def __init__(self, in_features_l, in_features_g, attn_features, up_factor, normalize_attn=True):\n        super(AttentionBlock, self).__init__()\n        self.up_factor = up_factor\n        self.normalize_attn = normalize_attn\n        self.W_l = nn.Conv2d(in_channels=in_features_l, out_channels=attn_features, kernel_size=1, padding=0, bias=False)\n        self.W_g = nn.Conv2d(in_channels=in_features_g, out_channels=attn_features, kernel_size=1, padding=0, bias=False)\n        self.phi = nn.Conv2d(in_channels=attn_features, out_channels=1, kernel_size=1, padding=0, bias=True)\n    \n    def forward(self, l, g):\n        N, C, W, H = l.size()\n        l_ = self.W_l(l)\n        g_ = self.W_g(g)\n        if self.up_factor > 1:\n            g_ = F.interpolate(g_, scale_factor=self.up_factor, mode='bilinear', align_corners=False)\n        c = self.phi(F.relu(l_ + g_)) # batch_sizex1xWxH\n        \n        # compute attn map\n        if self.normalize_attn:\n            a = F.softmax(c.view(N,1,-1), dim=2).view(N,1,W,H)\n        else:\n            a = torch.sigmoid(c)\n        # re-weight the local feature\n        f = torch.mul(a.expand_as(l), l) # batch_sizexCxWxH\n        if self.normalize_attn:\n            output = f.view(N,C,-1).sum(dim=2) # weighted sum\n        else:\n            output = F.adaptive_avg_pool2d(f, (1,1)).view(N,C) # global average pooling\n        return a, output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Attn_Resnest(nn.Module):\n\n    def __init__(self, model_name, normalize_attn=False):\n        super().__init__()\n        try:\n            self.backbone = timm.create_model(model_name, pretrained=True)\n        except:\n            self.backbone = torch.hub.load('zhanghang1989/ResNeSt', model_name, pretrained=True)\n        self.attn1 = AttentionBlock(256, 1024, 512, 4, normalize_attn=normalize_attn)\n        self.attn2 = AttentionBlock(512, 1024, 512, 2, normalize_attn=normalize_attn)\n        self.output = nn.Sequential(nn.Linear(768, 48), nn.Linear(48, 5))\n\n    def forward(self, x):\n        x = self.backbone.conv1(x)\n        x = self.backbone.bn1(x)\n        try:\n            x = self.backbone.act1(x)\n        except:\n            x = self.backbone.relu(x)\n        x = self.backbone.maxpool(x)\n\n        layer1 = self.backbone.layer1(x)\n        layer2 = self.backbone.layer2(layer1)\n        layer3 = self.backbone.layer3(layer2)\n        layer4 = self.backbone.layer4(layer3)\n        a1, g1 = self.attn1(layer1, layer3)\n        a2, g2 = self.attn2(layer2, layer3)\n        g_hat = torch.cat((g1,g2), dim=1) # batch_size x C\n        out = self.output(g_hat)\n        return out","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Efficientnet"},{"metadata":{"trusted":true},"cell_type":"code","source":"import timm\n\nclass Effnet(nn.Module):\n    def __init__(self, model_name):\n        super().__init__()\n        self.backbone = timm.create_model(model_name, pretrained=True)\n        in_feature = self.backbone.classifier.in_features\n        self.output = nn.Sequential(nn.Linear(in_feature, 256), nn.Linear(256, 5))\n    \n    def forward(self, x):\n        x = self.backbone.conv_stem(x)\n        x = self.backbone.bn1(x)\n        x = self.backbone.act1(x)\n        x = self.backbone.blocks(x)\n        x = self.backbone.conv_head(x)\n        x = self.backbone.bn2(x)\n        x = self.backbone.act2(x)\n        x = self.backbone.global_pool(x)\n        x = x.view(x.size(0), -1)\n        x = self.output(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss\n\nI am using focal loss as it seems to work better with data imbalance issue."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Courtesy: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/155201\ndef criterion_margin_focal_binary_cross_entropy(logit, truth):\n    weight_pos=2\n    weight_neg=1\n    gamma=2\n    margin=0.2\n    em = np.exp(margin)\n\n    logit = logit.view(-1)\n    truth = truth.view(-1)\n    log_pos = -F.logsigmoid( logit)\n    log_neg = -F.logsigmoid(-logit)\n\n    log_prob = truth*log_pos + (1-truth)*log_neg\n    prob = torch.exp(-log_prob)\n    margin = torch.log(em +(1-em)*prob)\n\n    weight = truth*weight_pos + (1-truth)*weight_neg\n    loss = margin + weight*(1 - prob) ** gamma * log_prob\n#     loss = loss.mean()\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SymmetricCrossEntropy(nn.Module):\n\n    def __init__(self, alpha=0.1, beta=1.0, num_classes= 5):\n        super(SymmetricCrossEntropy, self).__init__()\n        self.alpha = alpha\n        self.beta = beta\n        self.num_classes = num_classes\n\n    def forward(self, logits, targets, reduction='mean'):\n        onehot_targets = torch.eye(self.num_classes)[targets].cuda()\n        ce_loss = F.cross_entropy(logits, targets, reduction=reduction)\n        rce_loss = (-onehot_targets*logits.softmax(1).clamp(1e-7, 1.0).log()).sum(1)\n        if reduction == 'mean':\n            rce_loss = rce_loss.mean()\n        elif reduction == 'sum':\n            rce_loss = rce_loss.sum()\n        return self.alpha * ce_loss + self.beta * rce_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import logging\nlogging.basicConfig(level=logging.ERROR)\nimport wandb\nfrom functools import partial\nfrom collections import Counter\nimport gc\nimport time\nimport pandas as pd\nfrom torch import optim\nfrom catalyst.data.sampler import BalanceClassSampler\n\nrun = wandb.init(project=\"leaf_disease\")\nwandb.run.name= model_name\n\nm_p = mixed_precision\nif m_p:\n  scaler = torch.cuda.amp.GradScaler() \n\nnp.random.seed(SEED)\n\ntrain_ds = LeafDataset(train_df, img_dim, train_aug)\nif balanced_sampler:\n  print('Using Balanced Sampler....')\n  train_loader = torch.utils.data.DataLoader(train_ds,batch_size=batch_size, sampler=BalanceClassSampler(labels=train_ds.get_labels(), mode=\"upsampling\"), shuffle=False, num_workers=4)\nelse:\n  train_loader = torch.utils.data.DataLoader(train_ds, batch_size=batch_size, shuffle=True, num_workers=num_workers, pin_memory=True)\n\nval_ds = LeafDataset(valid_df, img_dim, val_aug)\nvalid_loader = torch.utils.data.DataLoader(\nval_ds, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True)\n\nos.makedirs(model_dir, exist_ok=True)\nos.makedirs(history_dir, exist_ok=True)\n\nresult = pd.DataFrame(columns=['name', 'prediction', 'label', 'difference'])\nif os.path.exists(f'{history_dir}/history_{model_name}_{img_dim}.csv'):\n    history = pd.read_csv(f'{history_dir}/history_{model_name}_{img_dim}.csv')\nelse:\n    history = pd.DataFrame(columns=['train_loss','train_time','val_loss','val_cat_acc', 'val_time'])\n\n# model = Attn_Resnest(encoder_model).to(device)\nmodel = Resnest(encoder_model).to(device)\n# model = Effnet(model_name=encoder_model).to(device)\nwandb.watch(model)\n# criterion = criterion_margin_focal_binary_cross_entropy\n# criterion = nn.CrossEntropyLoss()\ncriterion = SymmetricCrossEntropy()\n# criterion = nn.BCEWithLogitsLoss()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_model(valid_loss, valid_acc, best_valid_loss, best_valid_acc, best_state, savepath):\n    if valid_loss<best_valid_loss:\n        print(f'Validation loss has decreased from:  {best_valid_loss:.4f} to: {valid_loss:.4f}. Saving checkpoint')\n        torch.save(best_state, savepath+'_loss.pth')\n        best_valid_loss = valid_loss\n    if valid_acc>best_valid_acc:\n        print(f'Validation Accuracy score has increased from:  {best_valid_acc:.4f} to: {valid_acc:.4f}. Saving checkpoint')\n        torch.save(best_state, savepath + '_acc.pth')\n        best_valid_acc = valid_acc\n    else:\n        torch.save(best_state, savepath + '_last.pth')\n    return best_valid_loss, best_valid_acc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import itertools\nfrom sklearn.metrics import confusion_matrix\nimport seaborn as sns\n\ndef plot_confusion_matrix(y_true=None, y_pred=None, labels=None, true_labels=None,\n                          pred_labels=None, normalize=False):\n    \"\"\"                   \n    Computes the confusion matrix to evaluate the accuracy of a classification.\n    \"\"\"\n    y_true = np.asarray(y_true)\n    y_pred = np.asarray(y_pred)\n    cm = confusion_matrix(y_true, y_pred)\n    if labels is None:\n        classes = unique_labels(y_true, y_pred)\n    else:\n        classes = np.asarray(labels)\n            \n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        cm = np.around(cm, decimals=2)\n        cm[np.isnan(cm)] = 0.0\n            \n    if true_labels is None:\n        true_classes = classes\n    else:\n        true_label_indexes = np.in1d(classes, true_labels)\n        true_classes = classes[true_label_indexes]\n        cm = cm[true_label_indexes]\n            \n    if pred_labels is None:\n        pred_classes = classes\n    else:\n        pred_label_indexes = np.in1d(classes, pred_labels)\n        pred_classes = classes[pred_label_indexes]\n        cm = cm[:, pred_label_indexes]\n            \n    data=[]\n    count = 0\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        if labels is not None and (isinstance(pred_classes[i], int)\n                                    or isinstance(pred_classes[0], np.integer)):\n            pred_dict = labels[pred_classes[i]]\n            true_dict = labels[true_classes[j]]\n        else:\n            pred_dict = pred_classes[i]\n            true_dict = true_classes[j]\n        data.append([pred_dict, true_dict, cm[i,j]])\n        count+=1\n    wandb.log({\"confusion_matrix\" : wandb.Table(\n                columns=['Predicted', 'Actual', 'Count'],\n                data=data)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import itertools\nfrom sklearn.metrics import confusion_matrix\nimport seaborn as sns\n\nfrom sklearn.metrics import confusion_matrix\nimport seaborn as sns\n\ndef plot_confusion_matrix(predictions, actual_labels, labels):\n    cm = confusion_matrix(predictions, actual_labels, labels)\n    # Normalise\n    cmn = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n    fig, ax = plt.subplots(figsize=(12,12))\n    sns.heatmap(cmn, annot=True, fmt='.2f', xticklabels=labels, yticklabels=labels)\n    plt.ylabel('Actual')\n    plt.xlabel('Predicted')\n    plt.savefig('heatmap.png')\n#     plt.show(block=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from gradcam import GradCAM, GradCAMpp\nfrom gradcam.utils import visualize_cam\n\ndef grad_cam_gen(model, img, mixed_precision = False, device = 'cuda'):\n    if mixed_precision:\n        model, optim = amp.initialize(model, optim, opt_level='O1')\n         \n    configs = [dict(model_type='resnet', arch=model, layer_name='layer4')]\n    for config in configs:\n        config['arch'].to(device).eval()\n\n    cams = [\n    [cls.from_config(**config) for cls in (GradCAM, GradCAMpp)]\n        for config in configs]\n\n    for _, gradcam_pp in cams:\n        mask_pp, _ = gradcam_pp(img)\n        heatmap_pp, result_pp = visualize_cam(mask_pp, img)\n        result_pp = result_pp.cpu().numpy()\n        #convert image back to Height,Width,Channels\n        result_pp = np.transpose(result_pp, (1,2,0))\n        return result_pp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\ndef plot_heatmap(model):\n    fig = plt.figure(figsize=(70, 56))\n    for class_id in sorted(valid_df['label'].unique()):\n        for i, (idx, row) in enumerate(valid_df.loc[valid_df['label'] == class_id].sample(3, random_state=SEED).iterrows()):\n            ax = fig.add_subplot(5, 5, class_id * 5 + i + 1, xticks=[], yticks=[])\n            path=f\"{row['path']}\"\n            image = cv2.imread(path, cv2.IMREAD_COLOR)\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            image = cv2.resize(image, (img_dim, img_dim))\n            aug = val_aug(image=image)\n            image = aug['image'].reshape(img_dim, img_dim, 3).transpose(2, 0, 1)\n            image = torch.FloatTensor(image)\n            prediction = torch.argmax(model(torch.unsqueeze(image.to(device), dim=0)))\n            prediction = prediction.data.cpu().numpy()\n            image = grad_cam_gen(model.backbone, torch.unsqueeze(image, dim=0).cuda())\n            plt.imshow(image, cmap='bone')\n            ax.set_title('Label: %s Prediction: %s' % (row['Label'], pred_label))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_heatmap(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_val(epoch, dataloader, optimizer, pretrained=None, train=True, mode='train', record=True):\n    global m_p\n    global result\n    global batch_size\n    global accum_step\n    t1 = time.time()\n    running_loss = 0\n    epoch_samples = 0\n    pred = []\n    lab = []\n    if pretrained:\n        model.load_state_dict(pretrained)\n    if train:\n        model.train()\n        print(\"Initiating train phase ...\")\n    else:\n        model.eval()\n        print(\"Initiating val phase ...\")\n    for idx, (_, img, labels) in enumerate(dataloader):\n        with torch.set_grad_enabled(train):\n            img = img.to(device, dtype=torch.float32)\n#             labels = labels.to(device, dtype=torch.float32)\n            labels = torch.LongTensor(labels).to(device)\n            epoch_samples += len(img)\n            optimizer.zero_grad()\n            with torch.cuda.amp.autocast(m_p):\n                if m_p:\n                    img = img.half()\n                else:\n                    img = img.float()\n                outputs = model(img)\n\n                loss = criterion(outputs, labels).sum()\n                running_loss += loss.item()*len(img)\n                loss = loss/accum_step\n      \n                if train:\n                     if m_p:\n                         scaler.scale(loss).backward()\n                         if (idx+1) % accum_step == 0:\n                             scaler.step(optimizer)\n                             scaler.update() \n                             optimizer.zero_grad()\n#                              cyclic_scheduler.step()\n                     else:\n                         loss.backward()\n                         if (idx+1) % accum_step == 0:\n                             optimizer.step()\n                             optimizer.zero_grad()\n#                              cyclic_scheduler.step()\n\n        elapsed = int(time.time() - t1)\n        eta = int(elapsed / (idx+1) * (len(dataloader)-(idx+1)))\n        pred.append(torch.argmax(outputs, dim=1).detach().cpu().numpy())\n#         lab.append(torch.argmax(labels, dim=1).cpu().numpy())\n        lab.append(labels.cpu().numpy())\n        if train:\n            msg = f\"Epoch: {epoch} Progress: [{idx}/{len(dataloader)}] loss: {(running_loss/epoch_samples):.4f} Time: {elapsed}s ETA: {eta} s\"\n        else:\n            msg = f'Epoch {epoch} Progress: [{idx}/{len(dataloader)}] loss: {(running_loss/epoch_samples):.4f} Time: {elapsed}s ETA: {eta} s'\n        wandb.log({\"Train Loss\": running_loss/epoch_samples})\n        print(msg, end= '\\r')\n    cat_acc = (np.concatenate(pred)==np.concatenate(lab)).mean()\n    history.loc[epoch, f'{mode}_loss'] = running_loss/epoch_samples\n    history.loc[epoch, f'{mode}_time'] = elapsed\n    if mode=='val' or mode=='test':\n        lr_reduce_scheduler.step(cat_acc)\n        msg = f'{mode} Loss: {running_loss/epoch_samples:.4f} \\n {mode} Categorical Accuracy: {cat_acc:.4f}'\n        print(msg)\n        wandb.log({f\"{mode} Loss\": running_loss/epoch_samples, f\"{mode} Categorical Accuracy\":cat_acc})\n        plot_confusion_matrix(np.concatenate(lab), np.concatenate(pred), [i for i in range(5)])\n#         wandb.log({\"conf_mat\" : wandb.plot.confusion_matrix(\n#                         np.concatenate(pred), np.concatenate(lab), [i for i in range(5)])})\n        hist = cv2.imread('./heatmap.png', cv2.IMREAD_COLOR)\n        hist = cv2.cvtColor(hist, cv2.COLOR_BGR2RGB)\n        wandb.log({\"histogram\": [wandb.Image(hist, caption=\"Histogram\")]})\n        history.loc[epoch, f'{mode}_loss'] = running_loss/epoch_samples\n        history.loc[epoch, f'{mode}_cat_acc'] = cat_acc\n        # NaN check\n        if running_loss/epoch_samples > loss_thr or running_loss!=running_loss:\n            print('\\033[91mMixed Precision\\033[0m rendering nan value. Forcing \\033[91mMixed Precision\\033[0m to be False ...')\n            m_p = False\n            batch_size = batch_size//2\n            accum_step = accum_step*2\n            print('Loading last best model ...')\n            tmp = torch.load(os.path.join(model_dir, model_name+'_loss.pth'))\n            model.load_state_dict(tmp['model'])\n            optimizer.load_state_dict(tmp['optim'])\n            lr_reduce_scheduler.load_state_dict(tmp['scheduler'])\n#             cyclic_scheduler.load_state_dict(tmp['cyclic_scheduler'])\n            del tmp\n            \n        if record:\n            history.to_csv(f'{history_dir}/history_{model_name}_{img_dim}.csv', index=False)\n        return running_loss/epoch_samples, cat_acc\n\n\nplist = [ \n        {'params': model.backbone.parameters(),  'lr': learning_rate/100},\n#         {'params': model.attn1.parameters(),  'lr': learning_rate},\n#         {'params': model.attn2.parameters(),  'lr': learning_rate},\n        {'params': model.output.parameters(),  'lr': learning_rate}\n    ]\noptimizer = optim.Adam(plist, lr=learning_rate)\n# cyclic_scheduler = optim.lr_scheduler.OneCycleLR(optimizer, max_lr=[0.1*learning_rate, 10*learning_rate, 10*learning_rate, 10*learning_rate], steps_per_epoch=len(train_loader), epochs=n_epochs)\nlr_reduce_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=patience, verbose=True, threshold=1e-4, threshold_mode='rel', cooldown=0, min_lr=1e-7, eps=1e-08)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def main():\n  prev_epoch_num = 0\n  best_valid_loss = np.inf\n  best_valid_acc = 0.0\n\n  if load_model:\n    tmp = torch.load(os.path.join(model_dir, model_name+'_acc.pth'))\n    model.load_state_dict(tmp['model'])\n    optimizer.load_state_dict(tmp['optim'])\n    lr_reduce_scheduler.load_state_dict(tmp['scheduler'])\n#     cyclic_scheduler.load_state_dict(tmp['cyclic_scheduler'])\n    scaler.load_state_dict(tmp['scaler'])\n    prev_epoch_num = tmp['epoch']\n    best_valid_loss = tmp['best_loss']\n    best_valid_loss, best_valid_acc = train_val(prev_epoch_num+1, valid_loader, optimizer=optimizer, train=False, mode='val')\n    del tmp\n    print('Model Loaded!')\n  \n  for epoch in range(prev_epoch_num, n_epochs):\n    torch.cuda.empty_cache()\n    print(gc.collect())\n\n    train_val(epoch, train_loader, optimizer=optimizer, train=True, mode='train')\n    valid_loss, valid_acc = train_val(epoch, valid_loader, optimizer=optimizer, train=False, mode='val')\n    print(\"#\"*20)\n    print(f\"Epoch {epoch} Report:\")\n    print(f\"Validation Loss: {valid_loss :.4f} Validation ACC: {valid_acc :.4f}\")\n    best_state = {'model': model.state_dict(), 'optim': optimizer.state_dict(), 'scheduler':lr_reduce_scheduler.state_dict(), \n    \n#     'cyclic_scheduler':cyclic_scheduler.state_dict(), \n          'scaler': scaler.state_dict(),\n    'best_loss':valid_loss, 'best_acc':valid_acc, 'epoch':epoch}\n    best_valid_loss, best_valid_acc = save_model(valid_loss, valid_acc, best_valid_loss, best_valid_acc, best_state, os.path.join(model_dir, model_name))\n    print(\"#\"*20)\n   \nif __name__== '__main__':\n  main()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"artifact = wandb.Artifact('model', type='model')\nartifact.add_file(os.path.join(model_dir, model_name) + '_acc.pth')\nrun.log_artifact(artifact)\nrun.join()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}