{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Cassava Leaf Disease Modelling\nThere is already a great kernel [Vision Transformer (ViT): Tutorial + Baseline](https://www.kaggle.com/abhinand05/vision-transformer-vit-tutorial-baseline) that shows us how to use visiontransformer on TPU and why. Here, we make further improvements on the basis of the official implementation, in order to provide better pre-training parameters and user-friendly API as `Effecientnet-PyTorch`. You can find all the details on https://github.com/tczhangzhi/VisionTransformer-Pytorch.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/vision-transformer-pytorch/VisionTransformer-Pytorch\n!pip install ../input/pytorch-image-models/pytorch-image-models-master\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\n\npackage_path = '../input/vision-transformer-pytorch/VisionTransformer-Pytorch'\nsys.path.append(package_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import libraries\nimport os\nimport pandas as pd\nimport albumentations as albu\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold, train_test_split\nimport json\nimport seaborn as sns\nimport cv2\nimport albumentations as albu\nimport numpy as np\nfrom albumentations import Compose, Normalize, HorizontalFlip, VerticalFlip\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\nfrom albumentations.pytorch import ToTensorV2\n# ====================================================\n# Library\n# ====================================================\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nsys.path.append('../input/vision-transformer-pytorch/VisionTransformer-Pytorch')\nfrom vision_transformer_pytorch import VisionTransformer\n\nimport os\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\n#from efficientnet_pytorch import EfficientNet\nfrom vision_transformer_pytorch import VisionTransformer\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nimport timm\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"BASE_DIR=\"../input/cassava-leaf-disease-classification/\"\nTRAIN_IMAGES_DIR=os.path.join(BASE_DIR,'train_images')\n\ntrain_df=pd.read_csv(os.path.join(BASE_DIR,'train.csv'))\nclass CFG:\n    epoch = 10\n    debug=False\n    num_workers=8\n    model_name='resnext50_32x4d'\n    size=512\n    size_vit=384\n    batch_size=32\n    seed=2020\n    target_size=5\n    target_col='label'\n    n_fold=5\n    trn_fold=[0, 1, 2, 3, 4]\n    inference=True\n    weights = [1,1,1,1,1]\n    used_epochs = [0,1,2,3,4]\n    tta = 8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Count of training images {0}\".format(len(os.listdir(TRAIN_IMAGES_DIR))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(f'{BASE_DIR}/label_num_to_disease_map.json', 'r') as f:\n    name_mapping = json.load(f)\n    \nname_mapping = {int(k): v for k, v in name_mapping.items()}\ntrain_df[\"class_id\"]=train_df[\"label\"].map(name_mapping)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"name_mapping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_df)\nfolds = 5\n\ndata_dir = '../input/cassava-leaf-disease-classification'\ndf_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))\nimage_folder = os.path.join(data_dir, 'train_images')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#splitting data\nskf = StratifiedKFold(folds,shuffle=True,random_state=42)\ntrain_df['fold'] = -1\n\nfor i, (train_idx, valid_idx) in enumerate(skf.split(train_df, train_df['label'])):\n    train_df.loc[valid_idx, 'fold'] = i\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''def visualize_images(image_ids,labels):\n    plt.figure(figsize=(16,12))\n    \n    for ind,(image_id,label) in enumerate(zip(image_ids,labels)):\n        plt.subplot(3,3,ind+1)\n        \n        image=cv2.imread(os.path.join(TRAIN_IMAGES_DIR,image_id))\n        image=cv2.cvtColor(image,cv2.COLOR_BGR2RGB)\n        \n        plt.imshow(image)\n        plt.title(f\"Class: {label}\",fontsize=12)\n        \n        plt.axis(\"off\")\n    plt.show()\n    \n\ndef plot_augmentation(image_id,transform):\n    plt.figure(figsize=(16,4))\n    \n    img=cv2.imread(os.path.join(TRAIN_IMAGES_DIR,image_id))\n    img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n    \n    plt.subplot(1,3,1)\n    plt.imshow(img)\n    plt.axis(\"off\")\n    \n    plt.subplot(1,3,2)\n    x=transform(image=img)[\"image\"]\n    plt.imshow(x)\n    plt.axis(\"off\")\n    \n    plt.subplot(1,3,3)\n    x=transform(image=img)[\"image\"]\n    plt.imshow(x)\n    plt.axis(\"off\")\n    \n    plt.show()\n    \n    \ndef visualize(images, transform):\n    \"\"\"\n    Plot images and their transformations\n    \"\"\"\n    fig = plt.figure(figsize=(32, 16))\n    \n    for i, im in enumerate(images):\n        ax = fig.add_subplot(2, 5, i + 1, xticks=[], yticks=[])\n        plt.imshow(im)\n        \n    for i, im in enumerate(images):\n        ax = fig.add_subplot(2, 5, i + 6, xticks=[], yticks=[])\n        plt.imshow(transform(image=im)['image'])'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_size = 384","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modelling"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score\n\nfrom albumentations.pytorch import ToTensorV2\n# from efficientnet_pytorch import EfficientNet\nimport time\nimport datetime\nimport copy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CASSAVA(Dataset):\n    def __init__(self,df,transforms=None):\n        self.df = df\n        #print(self.df.shape[0])\n        self.transforms = transforms\n    def __len__(self):\n        return self.df.shape[0]\n    def __getitem__(self,idx):\n        row = self.df.iloc[idx]\n        img_dir = os.path.join(image_folder,row['image_id'])\n        img = cv2.imread(img_dir)\n        if self.transforms is not None:\n            img = self.transforms(image=img)['image']\n        label = row.label\n        return img,torch.tensor(label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TaylorSoftmax(nn.Module):\n\n    def __init__(self, dim=1, n=2):\n        super(TaylorSoftmax, self).__init__()\n        assert n % 2 == 0\n        self.dim = dim\n        self.n = n\n\n    def forward(self, x):\n        \n        fn = torch.ones_like(x)\n       \n        denor = 1.\n        for i in range(1, self.n+1):\n            denor *= i\n            fn = fn + x.pow(i) /(denor+1e-6)\n\n        out = fn / fn.sum(dim=self.dim, keepdims=True)\n        return out\n    \nclass LabelSmoothingLoss(nn.Module):\n\n    def __init__(self, classes, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        \"\"\"Taylor Softmax and log are already applied on the logits\"\"\"\n        with torch.no_grad(): \n            true_dist = torch.zeros_like(pred) \n            if self.cls-1==0:\n                raise Exception('self.cls = 1')\n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))\n    \n\nclass TaylorCrossEntropyLoss(nn.Module):\n\n    def __init__(self, n=2, ignore_index=-1, reduction='mean', smoothing=0.2):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=-1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n        self.lab_smooth = LabelSmoothingLoss(out_dim, smoothing=smoothing)\n\n    def forward(self, logits, labels):\n \n        log_probs = self.taylor_softmax(logits).log()\n\n        loss = self.lab_smooth(log_probs, labels)\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transforms():\n    return Compose([\n            RandomResizedCrop(image_size, image_size),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \ntrain_augs = get_train_transforms()\n\nvalid_augs = albu.Compose([\n    albu.Resize(height=384, width=384, p=1.0),\n    albu.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],),\n    ToTensorV2(),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def log_t(u, t):\n    \"\"\"Compute log_t for `u'.\"\"\"\n    if t==1.0:\n        return u.log()\n    else:\n        return (u.pow(1.0 - t) - 1.0) / (1.0 - t)\n\ndef exp_t(u, t):\n    \"\"\"Compute exp_t for `u'.\"\"\"\n    if t==1:\n        return u.exp()\n    else:\n        return (1.0 + (1.0-t)*u).relu().pow(1.0 / (1.0 - t))\n\ndef compute_normalization_fixed_point(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t > 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same shape as activation with the last dimension being 1.\n    \"\"\"\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations_step_0 = activations - mu\n\n    normalized_activations = normalized_activations_step_0\n\n    for _ in range(num_iters):\n        logt_partition = torch.sum(\n                exp_t(normalized_activations, t), -1, keepdim=True)\n        normalized_activations = normalized_activations_step_0 * \\\n                logt_partition.pow(1.0-t)\n\n    logt_partition = torch.sum(\n            exp_t(normalized_activations, t), -1, keepdim=True)\n    normalization_constants = - log_t(1.0 / logt_partition, t) + mu\n\n    return normalization_constants\n\ndef compute_normalization_binary_search(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t < 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (< 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations = activations - mu\n\n    effective_dim = \\\n        torch.sum(\n                (normalized_activations > -1.0 / (1.0-t)).to(torch.int32),\n            dim=-1, keepdim=True).to(activations.dtype)\n\n    shape_partition = activations.shape[:-1] + (1,)\n    lower = torch.zeros(shape_partition, dtype=activations.dtype, device=activations.device)\n    upper = -log_t(1.0/effective_dim, t) * torch.ones_like(lower)\n\n    for _ in range(num_iters):\n        logt_partition = (upper + lower)/2.0\n        sum_probs = torch.sum(\n                exp_t(normalized_activations - logt_partition, t),\n                dim=-1, keepdim=True)\n        update = (sum_probs < 1.0).to(activations.dtype)\n        lower = torch.reshape(\n                lower * update + (1.0-update) * logt_partition,\n                shape_partition)\n        upper = torch.reshape(\n                upper * (1.0 - update) + update * logt_partition,\n                shape_partition)\n\n    logt_partition = (upper + lower)/2.0\n    return logt_partition + mu\n\nclass ComputeNormalization(torch.autograd.Function):\n    \"\"\"\n    Class implementing custom backward pass for compute_normalization. See compute_normalization.\n    \"\"\"\n    @staticmethod\n    def forward(ctx, activations, t, num_iters):\n        if t < 1.0:\n            normalization_constants = compute_normalization_binary_search(activations, t, num_iters)\n        else:\n            normalization_constants = compute_normalization_fixed_point(activations, t, num_iters)\n\n        ctx.save_for_backward(activations, normalization_constants)\n        ctx.t=t\n        return normalization_constants\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        activations, normalization_constants = ctx.saved_tensors\n        t = ctx.t\n        normalized_activations = activations - normalization_constants \n        probabilities = exp_t(normalized_activations, t)\n        escorts = probabilities.pow(t)\n        escorts = escorts / escorts.sum(dim=-1, keepdim=True)\n        grad_input = escorts * grad_output\n        \n        return grad_input, None, None\n\ndef compute_normalization(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example. \n    Backward pass is implemented.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    return ComputeNormalization.apply(activations, t, num_iters)\n\ndef tempered_sigmoid(activations, t, num_iters = 5):\n    \"\"\"Tempered sigmoid function.\n    Args:\n      activations: Activations for the positive class for binary classification.\n      t: Temperature tensor > 0.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_probabilities = tempered_softmax(internal_activations, t, num_iters)\n    return internal_probabilities[..., 0]\n\n\ndef tempered_softmax(activations, t, num_iters=5):\n    \"\"\"Tempered softmax function.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature > 1.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    if t == 1.0:\n        return activations.softmax(dim=-1)\n\n    normalization_constants = compute_normalization(activations, t, num_iters)\n    return exp_t(activations - normalization_constants, t)\n\ndef bi_tempered_binary_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing = 0.0,\n        num_iters=5,\n        reduction='mean'):\n\n    \"\"\"Bi-Tempered binary logistic loss.\n    Args:\n      activations: A tensor containing activations for class 1.\n      labels: A tensor with shape as activations, containing probabilities for class 1\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A loss tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_labels = torch.stack([labels.to(activations.dtype),\n        1.0 - labels.to(activations.dtype)],\n        dim=-1)\n    return bi_tempered_logistic_loss(internal_activations, \n            internal_labels,\n            t1,\n            t2,\n            label_smoothing = label_smoothing,\n            num_iters = num_iters,\n            reduction = reduction)\n\ndef bi_tempered_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing=0.0,\n        num_iters=5,\n        reduction = 'mean'):\n\n    \"\"\"Bi-Tempered Logistic Loss.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      labels: A tensor with shape and dtype as activations (onehot), \n        or a long tensor of one dimension less than activations (pytorch standard)\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing parameter between [0, 1). Default 0.0.\n      num_iters: Number of iterations to run the method. Default 5.\n      reduction: ``'none'`` | ``'mean'`` | ``'sum'``. Default ``'mean'``.\n        ``'none'``: No reduction is applied, return shape is shape of\n        activations without the last dimension.\n        ``'mean'``: Loss is averaged over minibatch. Return shape (1,)\n        ``'sum'``: Loss is summed over minibatch. Return shape (1,)\n    Returns:\n      A loss tensor.\n    \"\"\"\n\n    if len(labels.shape)<len(activations.shape): #not one-hot\n        labels_onehot = torch.zeros_like(activations)\n        labels_onehot.scatter_(1, labels[..., None], 1)\n    else:\n        labels_onehot = labels\n\n    if label_smoothing > 0:\n        num_classes = labels_onehot.shape[-1]\n        labels_onehot = ( 1 - label_smoothing * num_classes / (num_classes - 1) ) \\\n                * labels_onehot + \\\n                label_smoothing / (num_classes - 1)\n\n    probabilities = tempered_softmax(activations, t2, num_iters)\n\n    loss_values = labels_onehot * log_t(labels_onehot + 1e-10, t1) \\\n            - labels_onehot * log_t(probabilities, t1) \\\n            - labels_onehot.pow(2.0 - t1) / (2.0 - t1) \\\n            + probabilities.pow(2.0 - t1) / (2.0 - t1)\n    loss_values = loss_values.sum(dim = -1) #sum over classes\n\n    if reduction == 'none':\n        return loss_values\n    if reduction == 'sum':\n        return loss_values.sum()\n    if reduction == 'mean':\n        return loss_values.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''def plot_image(img_dict):\n    image_tensor = img_dict[0]\n#     print(type(image_tensor))\n    target = img_dict[1]\n    print(target)\n    plt.figure(figsize=(10, 10))\n    image = image_tensor.permute(1, 2, 0) \n    plt.imshow(image)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomResNext(nn.Module):\n    def __init__(self, model_name='resnext50_32x4d', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm.auto import tqdm\n\ndef load_state(model_path):\n    model = CustomResNext('resnext50_32x4d', pretrained=False)\n    try:  # single GPU model_file\n        model.load_state_dict(torch.load(model_path)['model'], strict=True)\n        state_dict = torch.load(model_path)['model']\n    except:  # multi GPU model_file\n        state_dict = torch.load(model_path)['model']\n        state_dict = {k[7:] if k.startswith('module.') else k: state_dict[k] for k in state_dict.keys()}\n\n    return state_dict\n\ndef train_model(datasets, dataloaders, model, criterion, optimizer, scheduler, num_epochs, device):\n    MODEL_DIR = '../input/cassava-resnext50-32x4d-weights/'\n    states_res = [load_state(MODEL_DIR+'resnext50_32x4d_fold{}.pth'.format(e)) for e in range(5)]\n    \n    since = time.time()\n    model_res = CustomResNext('resnext50_32x4d', pretrained=False)\n    #model_res.to(device)\n\n    model_vit = VisionTransformer.from_name('ViT-B_16', num_classes=5)\n    model_vit.to(device)\n    model_vit.load_state_dict(torch.load('../input/vit-model-1/ViT-B_16.pt'))\n    \n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n\n    for epoch in range(num_epochs):\n        print('Epoch {}/{}'.format(epoch, num_epochs-1))\n        print('-' * 10)\n\n        for phase in ['train', 'valid']:\n            if phase == 'train':\n                model.train()\n            else:\n                model.eval()\n\n            running_loss = 0.0\n            running_corrects = 0.0\n\n            for inputs, labels in tqdm(dataloaders[phase]):\n                inputs = inputs.to(device)\n                #labels=labels.to(device)\n                labels = torch.zeros((inputs.size()[0],5)).to(device)\n                for state in states_res:\n                    model_res.load_state_dict(state)\n                    model_res.to(device)\n                    model_res.eval()\n                    with torch.no_grad():\n                        labels+=torch.softmax(model_res(inputs),1).detach()\n                with torch.no_grad():\n                    model_vit.eval()\n                    l=model_vit(inputs)\n                    labels+=tempered_softmax(l, 1.4,5).detach()    \n                labels.to(device)\n                \n                # Zero out the grads\n                optimizer.zero_grad()\n\n                # Forward\n                # Track history in train mode\n                with torch.set_grad_enabled(phase == 'train'):\n                    model=model.to(device)\n                    outputs = model(inputs)\n                    preds = torch.argmax(outputs, 1)\n                    loss = bi_tempered_logistic_loss(outputs, labels,\n                                                     t1=0.8, t2=1.4,\n                                                     label_smoothing=0.05)\n\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n                # Statistics\n                labels = torch.argmax(labels,-1)\n                running_loss += loss.item()*inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n            if phase == 'train':\n                scheduler.step()\n\n            epoch_loss = running_loss/len(datasets[phase])\n            epoch_acc = running_corrects.double()/len(datasets[phase])\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n                phase, epoch_loss, epoch_acc))\n\n            if phase == 'valid' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n\n        print()\n\n    time_elapsed = time.time()-since\n    print('Training complete in {:.0f}m {:.0f}s'.format(\n        time_elapsed // 60, time_elapsed % 60))\n    print('Best val Acc: {:4f}'.format(best_acc))\n\n    model.load_state_dict(best_model_wts)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fold=3\n\n\ntrain_idx = np.where((train_df['fold'] != fold))[0]\nvalid_idx = np.where((train_df['fold'] == fold))[0]\n        \ndf_curr = train_df.loc[train_idx]\ndf_val = train_df.loc[valid_idx]\n\ntrain_dataset = CASSAVA(df_curr , transforms=train_augs)\nvalid_dataset = CASSAVA(df_val, transforms=valid_augs)\n        \ntrain_loader = torch.utils.data.DataLoader(\n            dataset=train_dataset,\n            batch_size=8,\n            drop_last=True,\n            num_workers=2\n        )\n\n\nvalid_loader = torch.utils.data.DataLoader(\n            dataset=valid_dataset, \n            batch_size=8,\n\n            drop_last=True,\n            num_workers=2,\n        )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from vision_transformer_pytorch import VisionTransformer\n\n\ndatasets={'train':train_dataset,'valid':valid_dataset}\ndataloaders={'train':train_loader,'valid':valid_loader}\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = VisionTransformer.from_pretrained('ViT-B_16', num_classes=5) \n\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.001)\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1)\ncriterion=nn.CrossEntropyLoss()\nnum_epochs=6","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train the model after uncommenting below"},{"metadata":{"trusted":true},"cell_type":"code","source":"trained_model=train_model(datasets,dataloaders,model,None,optimizer,scheduler,4,device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Epoch 0/5\n# ----------\n# train Loss: 0.5318 Acc: 0.8119\n# valid Loss: 0.4009 Acc: 0.8650\n\n# Epoch 1/5\n# ----------\n# train Loss: 0.4384 Acc: 0.8467\n# valid Loss: 0.3999 Acc: 0.8612\n\n# Epoch 2/5\n# ----------\n# train Loss: 0.3511 Acc: 0.8778\n# valid Loss: 0.3558 Acc: 0.8771\n\n# Epoch 3/5\n# ----------\n# train Loss: 0.3266 Acc: 0.8879\n# valid Loss: 0.3468 Acc: 0.8836\n\n# Epoch 4/5\n# ----------\n# train Loss: 0.3066 Acc: 0.8924\n# valid Loss: 0.3384 Acc: 0.8911\n\n# Epoch 5/5\n# ----------\n# train Loss: 0.3060 Acc: 0.8926\n# valid Loss: 0.3421 Acc: 0.8869\n\n# Training complete in 121m 52s\n# Best val Acc: 0.891121","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Save the model after training"},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save(model.state_dict(), 'ViT-B_16.pt')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Load the model when model is trained and saved and notebook has to be run without internet"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''model.load_state_dict(torch.load('../input/vit-model-1/ViT-B_16.pt'))'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''test_df = pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")\nimage_path = \"../input/cassava-leaf-disease-classification/test_images/\"\n# fake targets\ntest_targets = test_df.label.values\n\n\ntest_aug = albu.Compose([\n            albu.CenterCrop(512, 512, p=1.),\n            albu.Resize(384, 384),\n            albu.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0),\n            ToTensorV2()], p=1.)\n\ntest_dataset=CassavaDataset(\n    df=test_df,\n    imfolder=image_path,\n    train=False,\n    transforms=test_aug\n)\n\ntest_loader =  DataLoader(\n        test_dataset,\n        batch_size=4,\n        num_workers=4,\n        shuffle=False,\n)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\npredictions=[]\n\nfor imgs in test_loader:\n\n    imgs = imgs.to(device)\n    with torch.no_grad():\n        model=model.to(device)\n        outputs = model(imgs)\n        _, predicted = torch.max(outputs, dim=1)\n        predicted=predicted.to('cpu')\n        predictions.append(predicted)\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''test_df['label'] = np.concatenate(predictions)'''\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''test_df.to_csv('submission.csv', index=False)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}