{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install torchsummary\n!pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"base_dir = '/kaggle/input/cassava-leaf-disease-classification/'\ntrain_pth = base_dir + 'train_images/'\ntest_pth = base_dir + 'test_images'\ntrain_csv = base_dir + 'train.csv'\nlbl_map = base_dir + 'label_num_to_disease_map.json'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nimport torch.nn as nn\n\nfile = open(lbl_map)\nmap_lbl = json.load(file)\nmap_lbl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport pandas as pd\ndf = pd.read_csv(train_csv)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_df, valid_df = train_test_split(df, test_size=0.20, random_state=42, stratify=df.label.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nplt.title('Total')\nsns.countplot(df['label'])\nplt.show()\nplt.title('Validation Split')\nsns.countplot(valid_df['label'])\nplt.show()\nplt.title('Train Split')\nsns.countplot(train_df['label'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\nfrom torchvision import datasets, transforms\nfrom PIL import Image\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n\nclass Cassava_leaf_disease_dataset():\n    def __init__(self, img_pth, csv, transform=None):\n        self.img_pth = img_pth\n        self.transform = transform\n        self.all_imgs = list(csv['image_id'])\n        self.csv = csv\n  \n    def __len__(self):\n        return len(self.all_imgs)\n\n    def __getitem__(self, idx):\n        img_loc = os.path.join(self.img_pth, self.all_imgs[idx])\n        name = self.all_imgs[idx]\n        label = self.csv.loc[self.csv['image_id'] == name]\n        label = int(label['label'])\n        image = Image.open(img_loc).convert(\"RGB\")\n        image = np.array(image, dtype=np.uint8)\n        if self.transform and (label == 0 or label == 1 or label == 2, label == 4) : \n            out_img = self.transform(image=image)\n            return out_img, label\n        else:\n            return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nfrom sklearn.utils import class_weight\nclass_weights = class_weight.compute_class_weight(\n           'balanced',\n            np.unique(train_df.label), \n            train_df.label)\nclass_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations as albu\nfrom albumentations.pytorch import ToTensorV2\n\ntrain_augs = albu.Compose([\n    albu.RandomResizedCrop(height=256, width=256, p=1.0),\n    albu.HorizontalFlip(p=0.5),\n    albu.VerticalFlip(p=0.5),\n    albu.RandomBrightnessContrast(p=0.5),\n    albu.ShiftScaleRotate(p=0.5),\n    albu.Normalize(    \n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],),\n    ToTensorV2(),\n])\n\nvalid_augs = albu.Compose([\n    albu.Resize(height=384, width=384, p=1.0),\n    albu.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],),\n    ToTensorV2(),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 16\n# train_transforms = transforms.Compose([transforms.RandomRotation(25),\n#                                        transforms.Resize(384),\n#                                        transforms.CenterCrop(384),\n#                                        transforms.RandomHorizontalFlip(),\n#                                        transforms.RandomVerticalFlip(),\n#                                        transforms.ColorJitter(),\n#                                        transforms.ToTensor()\n#                                        ])\n\n# val_transforms = transforms.Compose([transforms.Resize(384),\n#                                     transforms.CenterCrop(384),\n#                                      transforms.ToTensor()])\n\n\ntrain_data = Cassava_leaf_disease_dataset(train_pth, train_df, train_augs)\ntrainloader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True, num_workers=4)\n\nvalid_data = Cassava_leaf_disease_dataset(train_pth, valid_df, valid_augs)\nvalidloader = torch.utils.data.DataLoader(valid_data, batch_size=batch_size, shuffle=True, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torchvision\nimport os\n\ndef imshow(inp, title=None):\n    inp = inp.numpy().transpose((1, 2, 0))\n    plt.figure(figsize=(15,15))\n    plt.axis('off')\n    plt.imshow(inp)\n    if title is not None:\n        plt.title(title)\n    plt.pause(0.001)\n\ndef show_databatch(inputs):\n    out = torchvision.utils.make_grid(inputs[:16])\n    imshow(out)\n\n# Get a batch of training data\ninputs, labels = next(iter(trainloader))\nshow_databatch(inputs['image'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision import models\nimport torch.nn as nn \n\ndef get_model(name, pretrain=False, grad=False):\n\n    if name == 'resnet18':\n        model = models.resnet18(pretrained=pretrain)\n        \n        if grad:\n            for param in model.parameters():\n                param.requires_grad = grad \n        \n        n_inputs = 512\n        model.fc = nn.Sequential(\n            nn.Linear(n_inputs, 256), nn.ReLU(), nn.Dropout(0.80),\n            nn.Linear(256, 5))\n        \n    elif name == 'densenet161':\n        model = models.densenet161(pretrained=pretrain)\n        \n        if grad:\n            for param in model.features.parameters():\n                param.requires_grad = grad \n                \n        n_inputs = 2208\n        model.classifier = nn.Sequential(\n            nn.Linear(n_inputs, 256), nn.ReLU(), nn.Dropout(0.80),\n            nn.Linear(256, 5))\n    \n    elif name == 'mobilenet':\n        model = models.mobilenet_v2(pretrained=pretrain)\n        \n        if grad:\n            for param in model.features.parameters():\n                param.requires_grad = grad \n        \n        n_inputs = 1280\n        model.classifier = nn.Sequential(\n            nn.Linear(n_inputs, 256), nn.ReLU(), nn.Dropout(0.850),\n            nn.Linear(256, 5))\n        \n    elif name == 'efficientnet':\n        from efficientnet_pytorch import EfficientNet\n        model = EfficientNet.from_pretrained('efficientnet-b7')\n        n_inputs = 2560\n        model._fc = nn.Sequential(\n            nn.Linear(n_inputs, 512), nn.ReLU(), nn.Dropout(0.850),\n            nn.Linear(512, 5))\n        model._quires_grad = True\n    \n    elif name == 'transformer':\n        !pip install vision_transformer_pytorch\n        from vision_transformer_pytorch import VisionTransformer\n        model = VisionTransformer.from_name('ViT-B_16', num_classes=5) \n        for param in model.transformer.parameters():\n            param.requires_grad = grad\n        \n        \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch import cuda\n# Check if gpu is available\ngpu = cuda.is_available()\nprint('Gpu is available:',gpu)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.optim as optim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score\n\n# Training function\ndef run(model, epochs, criterion, optimizer, scheduler, train_loader, val_loader, cuda=False, path=None, load_model=False):\n    valid_loss_min = np.Inf\n    stats = []\n    if load_model: model.load_state_dict(torch.load('./state_dict.pt'))       \n    print('\\nTraining Started\\n')\n    for epoch in range(0, epochs):  \n        train_losses = []\n        train_pred, train_orig, valid_pred, valid_orig = [], [], [], []\n        model.train()\n        print('Epoch: {}/{}...'.format(epoch+1, epochs))\n        for inputs, labels in train_loader:\n            #labels = one_hot_label(labels, 5)\n            inputs = inputs['image']\n            if cuda: inputs, labels = inputs.to('cuda'), labels.to('cuda')\n            else: inputs, labels = inputs.to('cpu'), labels.to('cpu')    \n            model.zero_grad()\n            output = model.forward(inputs)\n            loss = criterion(output, labels.long())\n            train_losses.append(loss.item())\n            loss.backward()\n            optimizer.step()\n    \n            # Calculate train accuracy\n            output = F.softmax(output, dim=1)\n            _, pred = torch.max(output, dim=1)\n            train_pred.extend(pred.cpu().numpy())\n            train_orig.extend(labels.cpu().numpy())\n        scheduler.step() \n        tacc = accuracy_score(np.array(train_orig), np.array(train_pred))\n        print(\"\\tTrain Loss: {:.6f}\".format(np.mean(train_losses))+\", Train accuracy: {:.3f}%\".format(tacc))\n        train_f1 = f1_score(np.array(train_orig), np.array(train_pred), average='micro')\n        print(f'\\tTrain F1 Score: {train_f1:.6f}')\n            \n        # Running model on validation data\n        with torch.no_grad():\n            val_losses, val_acc, val_f1, vo, vp , valid_loss_min = Validation_eval(model, val_loader, criterion, valid_orig, valid_pred, valid_loss_min, cuda)\n            stats.append([np.mean(train_losses), np.mean(val_losses), tacc, val_acc, train_f1, val_f1])\n        \n        \n        # Plotting confusion matrix after some epochs\n        if epoch % 3 == 0: \n            Confusion_matrix(np.array(vo), np.array(vp), title ='Validation Confusion matrix', labels = ['CBB','CBSD','CGM','CMD','Healthy'], path=None)    \n\n        stat = pd.DataFrame(stats, columns=['train_loss', 'valid_loss', 'train_acc','valid_acc', 'train_f1', 'val_f1'])\n        if epoch % 10 == 5: plotCurves(stat, path, f1=True)\n    print('Finished Training')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score, confusion_matrix, accuracy_score\n# METHOD FOR EVALUTAING VALIDATION DATASET PERFORMANCE\ndef Validation_eval(model, val_loader, criterion, vp, vo, valid_loss_min, cuda=False):\n    model.eval()\n    val_losses = []\n    for inp, target in val_loader:\n        #target = one_hot_label(target, 5)\n        inp = inp['image']\n        if cuda: inp, target = inp.to('cuda'), target.to('cuda')\n        else: inp, target = inp.to('cpu'), labels.to('cpu')\n        out = model.forward(inp)\n        val_loss = criterion(out, target.long())\n        val_losses.append(val_loss.item())\n\n        # Calculate validation accuracy\n        output = F.softmax(out, dim=1)\n        _, val_pred = torch.max(output, dim=1)\n        vp.extend(val_pred.cpu().numpy())\n        vo.extend(target.cpu().numpy())\n    vacc = accuracy_score(np.array(vo), np.array(vp))\n    print(\"\\tValidation Loss: {:.6f}\".format(np.mean(val_losses))+\", Validation accuracy: {:.3f}%\".format(vacc))\n    val_f1 = f1_score(np.array(vo), np.array(vp), average='micro')\n    print(f'\\tValid F1 Score: {val_f1:.6f}')\n        \n    # Saving model if mean of val loss is smaller than min val loss\n    if np.mean(val_losses) <= valid_loss_min:\n        torch.save(model.state_dict(), './state_dict.pt')\n        print('Validation loss decreased ({:.6f} --> {:.6f}). Saving model ...'.format(valid_loss_min,np.mean(val_losses))+'\\n')\n        valid_loss_min = np.mean(val_losses)\n            \n    return val_losses, vacc, val_f1, vo, vp, valid_loss_min ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.gridspec as gridspec\n\n# METHOD TO PLOT ACCURACY AND LOSS CURVES\ndef plotCurves(stats, path, f1=False):\n    fig = plt.figure(constrained_layout=True)\n    spec2 = gridspec.GridSpec(ncols=2, nrows=2, figure=fig)\n    ax1 = fig.add_subplot(spec2[0, 0])\n    ax2 = fig.add_subplot(spec2[0, 1])\n    ax3 = fig.add_subplot(spec2[1, 0])\n    for i in ['train_loss', 'valid_loss']: \n        ax1.plot(stats[i], label=i)\n    ax1.legend()\n    ax1.set_xlabel('Epochs')\n    ax1.set_ylabel('Loss')\n\n    for j in ['train_acc', 'valid_acc']: \n        ax2.plot(100 * stats[j], label=j)\n    ax2.legend()\n    ax2.set_xlabel('Epochs')\n    ax2.set_ylabel('Accuracy')\n    \n    if f1:\n        for c in ['train_f1', 'val_f1']: \n            ax3.plot(stats[c], label=c)\n        ax3.legend()\n        ax3.set_xlabel('Epochs')\n        ax3.set_ylabel('F1 Score')\n    plt.show()\n\n\n# METHOD TO PLOT CONFUSION MATRIX\ndef Confusion_matrix(original, prediction, title='Confusion matrix', labels=['0','1'], path=None):\n    fig = plt.figure(constrained_layout=True)\n    ax = fig.add_subplot()\n    ax.set_title(title)\n    cm = confusion_matrix(original, prediction)\n    sns.heatmap(cm, annot=True, cmap='Blues', fmt=\"d\", xticklabels=labels, yticklabels=labels)\n    if path != None: plt.savefig(path)\n    else: plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchsummary import summary\nif gpu:  \n    model = get_model('efficientnet', True, True).to('cuda')\nelse:\n    model = get_model('efficientnet', True, True).to('cpu')\ntry:summary(model, (3, 256, 256), batch_size, device='cuda')\nexcept: pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cls0, cls1, cls2, cls3, cls4 = 0,0,0,0,0\nfor i, each in enumerate(train_df['label']):\n    if each == 0:cls0 +=1\n    elif each == 1:cls1 +=1\n    elif each == 2:cls2 +=1\n    elif each == 3:cls3 +=1\n    elif each == 4:cls4 +=1\nsamples = [cls0, cls1, cls2, cls3, cls4]\nsamples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport torch\nimport torch.nn.functional as F\n\nif __name__ == \"__main__\":\n\n    EPOCHS = 30\n    # LOSS FUNCTION AND OPTIMIZATION FUNCTION\n    #torch.tensor(class_weights).float().to('cuda')\n    #criterion = nn.CrossEntropyLoss()\n    criterion = CB_Loss(samples_per_cls=samples, no_of_classes=5, loss_type='softmax')\n    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.1)\n    \n    \n    # START TRAINING\n    run(model, EPOCHS, criterion, optimizer, scheduler, trainloader, validloader, gpu, None, load_model=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Test(model, test_loader, cuda=False):\n    model = model.load_state_dict(torch.load('/kaggle/output/state_dict.pt'))  \n    \n    model.eval()\n    for inp, target in test_loader:\n        inp = inp['image']\n        if cuda: inp, target = inp.to('cuda'), target.to('cuda')\n        else: inp, target = inp.to('cpu'), labels.to('cpu')\n        out = model.forward(inp)\n\n        # Calculate validation accuracy\n        output = F.softmax(out, dim=1)\n        _, test_pred = torch.max(output, dim=1)\n        vp.extend(val_pred.cpu().numpy())\n        vo.extend(target.cpu().numpy())\n    vacc = accuracy_score(np.array(vo), np.array(vp))\n    print(\"Test accuracy: {:.3f}%\".format(vacc))\n    print(f'Test prediction is ', test_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport torch\nimport torch.nn.functional as F\n\n\nclass CB_Loss(nn.Module):\n    def __init__(self, samples_per_cls, no_of_classes=2, loss_type='sigmoid', beta=0.999, gamma=2.0):\n        super(CB_Loss, self).__init__()\n        self.samples_per_cls = samples_per_cls\n        self.no_of_classes = no_of_classes\n        self.loss_type = loss_type\n        self.beta = beta\n        self.gamma = gamma\n\n    def focal_loss(sel, labels, logits, alpha, gamma):\n        \"\"\"Compute the focal loss between `logits` and the ground truth `labels`.\n        Focal loss = -alpha_t * (1-pt)^gamma * log(pt)\n        where pt is the probability of being classified to the true class.\n        pt = p (if true class), otherwise pt = 1 - p. p = sigmoid(logit).\n        Args:\n          labels: A float tensor of size [batch, num_classes].\n          logits: A float tensor of size [batch, num_classes].\n          alpha: A float tensor of size [batch_size]\n            specifying per-example weight for balanced cross entropy.\n          gamma: A float scalar modulating loss from hard and easy examples.\n        Returns:\n          focal_loss: A float32 scalar representing normalized total loss.\n        \"\"\"    \n        BCLoss = F.binary_cross_entropy_with_logits(input = logits, target = labels,reduction = \"none\")\n\n        if gamma == 0.0:\n            modulator = 1.0\n        else:\n            modulator = torch.exp(-gamma * labels * logits - gamma * torch.log(1 + \n                torch.exp(-1.0 * logits)))\n\n        loss = modulator * BCLoss\n\n        weighted_loss = alpha * loss\n        focal_loss = torch.sum(weighted_loss)\n\n        focal_loss /= torch.sum(labels)\n        return focal_loss\n\n\n\n    def forward(self, logits, labels):\n        \"\"\"Compute the Class Balanced Loss between `logits` and the ground truth `labels`.\n        Class Balanced Loss: ((1-beta)/(1-beta^n))*Loss(labels, logits)\n        where Loss is one of the standard losses used for Neural Networks.\n        Args:\n          labels: A int tensor of size [batch].\n          logits: A float tensor of size [batch, no_of_classes].\n          samples_per_cls: A python list of size [no_of_classes].\n          no_of_classes: total number of classes. int\n          loss_type: string. One of \"sigmoid\", \"focal\", \"softmax\".\n          beta: float. Hyperparameter for Class balanced loss.\n          gamma: float. Hyperparameter for Focal loss.\n        Returns:\n          cb_loss: A float tensor representing class balanced loss\n        \"\"\"\n        effective_num = 1.0 - np.power(self.beta, self.samples_per_cls)\n        weights = (1.0 - self.beta) / np.array(effective_num)\n        weights = weights / np.sum(weights) * self.no_of_classes\n        labels_one_hot = F.one_hot(labels, self.no_of_classes).float().to('cuda')\n\n        weights = torch.tensor(weights).float()\n        weights = weights.unsqueeze(0).to('cuda')\n        weights = weights.repeat(labels_one_hot.shape[0],1) * labels_one_hot\n        weights = weights.sum(1)\n        weights = weights.unsqueeze(1)\n        weights = weights.repeat(1,self.no_of_classes)\n\n        if self.loss_type == \"focal\":\n            cb_loss = self.focal_loss(labels_one_hot, logits, weights, self.gamma)\n        elif self.loss_type == \"sigmoid\":\n            cb_loss = F.binary_cross_entropy_with_logits(input = logits,target = labels_one_hot, weights = weights)\n        elif self.loss_type == \"softmax\":\n            pred = logits.softmax(dim = 1)\n            cb_loss = F.binary_cross_entropy(input = pred, target = labels_one_hot, weight = weights)\n        return cb_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}