{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Cassava Leaf Disease Modelling\nThere is already a great kernel [Vision Transformer (ViT): Tutorial + Baseline](https://www.kaggle.com/abhinand05/vision-transformer-vit-tutorial-baseline) that shows us how to use visiontransformer on TPU and why. Here, we make further improvements on the basis of the official implementation, in order to provide better pre-training parameters and user-friendly API as `Effecientnet-PyTorch`. You can find all the details on https://github.com/tczhangzhi/VisionTransformer-Pytorch.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install vision_transformer_pytorch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\n\npackage_path = '../input/vision-transformer-pytorch/VisionTransformer-Pytorch'\nsys.path.append(package_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import libraries\nimport os\nimport pandas as pd\nimport albumentations as albu\nimport matplotlib.pyplot as plt\nimport json\nimport seaborn as sns\nimport cv2\nimport albumentations as albu\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"BASE_DIR=\"../input/cassava-leaf-disease-classification/\"\nTRAIN_IMAGES_DIR=os.path.join(BASE_DIR,'train_images')\n\ntrain_df=pd.read_csv(os.path.join(BASE_DIR,'train.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Count of training images {0}\".format(len(os.listdir(TRAIN_IMAGES_DIR))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(f'{BASE_DIR}/label_num_to_disease_map.json', 'r') as f:\n    name_mapping = json.load(f)\n    \nname_mapping = {int(k): v for k, v in name_mapping.items()}\ntrain_df[\"class_id\"]=train_df[\"label\"].map(name_mapping)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"name_mapping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize_images(image_ids,labels):\n    plt.figure(figsize=(16,12))\n    \n    for ind,(image_id,label) in enumerate(zip(image_ids,labels)):\n        plt.subplot(3,3,ind+1)\n        \n        image=cv2.imread(os.path.join(TRAIN_IMAGES_DIR,image_id))\n        image=cv2.cvtColor(image,cv2.COLOR_BGR2RGB)\n        \n        plt.imshow(image)\n        plt.title(f\"Class: {label}\",fontsize=12)\n        \n        plt.axis(\"off\")\n    plt.show()\n    \n\ndef plot_augmentation(image_id,transform):\n    plt.figure(figsize=(16,4))\n    \n    img=cv2.imread(os.path.join(TRAIN_IMAGES_DIR,image_id))\n    img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n    \n    plt.subplot(1,3,1)\n    plt.imshow(img)\n    plt.axis(\"off\")\n    \n    plt.subplot(1,3,2)\n    x=transform(image=img)[\"image\"]\n    plt.imshow(x)\n    plt.axis(\"off\")\n    \n    plt.subplot(1,3,3)\n    x=transform(image=img)[\"image\"]\n    plt.imshow(x)\n    plt.axis(\"off\")\n    \n    plt.show()\n    \n    \ndef visualize(images, transform):\n    \"\"\"\n    Plot images and their transformations\n    \"\"\"\n    fig = plt.figure(figsize=(32, 16))\n    \n    for i, im in enumerate(images):\n        ax = fig.add_subplot(2, 5, i + 1, xticks=[], yticks=[])\n        plt.imshow(im)\n        \n    for i, im in enumerate(images):\n        ax = fig.add_subplot(2, 5, i + 6, xticks=[], yticks=[])\n        plt.imshow(transform(image=im)['image'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modelling"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold, train_test_split\nfrom albumentations.pytorch import ToTensorV2\n# from efficientnet_pytorch import EfficientNet\nimport time\nimport datetime\nimport copy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DataSet class\n\nclass CassavaDataset(Dataset):\n    def __init__(self,df:pd.DataFrame,imfolder:str,train:bool = True, transforms=None):\n        self.df=df\n        self.imfolder=imfolder\n        self.train=train\n        self.transforms=transforms\n        \n    def __getitem__(self,index):\n        im_path=os.path.join(self.imfolder,self.df.iloc[index]['image_id'])\n        x=cv2.imread(im_path,cv2.IMREAD_COLOR)\n        x=cv2.cvtColor(x,cv2.COLOR_BGR2RGB)\n        \n        if(self.transforms):\n            x=self.transforms(image=x)['image']\n        \n        if(self.train):\n            y=self.df.iloc[index]['label']\n            return x,y\n        else:\n            return x\n        \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_augs = albu.Compose([\n    albu.RandomResizedCrop(height=384, width=384, p=1.0),\n    albu.HorizontalFlip(p=0.5),\n    albu.VerticalFlip(p=0.5),\n    albu.RandomBrightnessContrast(p=0.5),\n    albu.ShiftScaleRotate(p=0.5),\n    albu.Normalize(    \n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],),\n    ToTensorV2(),\n])\n\nvalid_augs = albu.Compose([\n    albu.Resize(height=384, width=384, p=1.0),\n    albu.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],),\n    ToTensorV2(),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train, valid = train_test_split(\n    train_df, \n    test_size=0.1, \n    random_state=42,\n    stratify=train_df.label.values\n)\n\n\n# reset index on both dataframes\ntrain = train.reset_index(drop=True)\nvalid = valid.reset_index(drop=True)\n\ntrain_targets = train.label.values\n\n# targets for validation\nvalid_targets = valid.label.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset=CassavaDataset(\n    df=train,\n    imfolder=TRAIN_IMAGES_DIR,\n    train=True,\n    transforms=train_augs\n)\n\nvalid_dataset=CassavaDataset(\n    df=valid,\n    imfolder=TRAIN_IMAGES_DIR,\n    train=True,\n    transforms=valid_augs\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_image(img_dict):\n    image_tensor = img_dict[0]\n#     print(type(image_tensor))\n    target = img_dict[1]\n    print(target)\n    plt.figure(figsize=(10, 10))\n    image = image_tensor.permute(1, 2, 0) \n    plt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_image(train_dataset[5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = DataLoader(\n    train_dataset,\n    batch_size=16,\n    num_workers=4,\n    shuffle=True,\n)\n\nvalid_loader = DataLoader(\n    valid_dataset,\n    batch_size=16,\n    num_workers=4,\n    shuffle=False,\n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(datasets, dataloaders, model, criterion, optimizer, scheduler, num_epochs, device):\n    since = time.time()\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n\n    for epoch in range(num_epochs):\n        print('Epoch {}/{}'.format(epoch, num_epochs-1))\n        print('-' * 10)\n\n        for phase in ['train', 'valid']:\n            if phase == 'train':\n                model.train()\n            else:\n                model.eval()\n\n            running_loss = 0.0\n            running_corrects = 0.0\n\n            for inputs, labels in dataloaders[phase]:\n                inputs = inputs.to(device)\n                labels=labels.to(device)\n\n                # Zero out the grads\n                optimizer.zero_grad()\n\n                # Forward\n                # Track history in train mode\n                with torch.set_grad_enabled(phase == 'train'):\n                    model=model.to(device)\n                    outputs = model(inputs)\n                    _, preds = torch.max(outputs, 1)\n                    loss = criterion(outputs, labels)\n\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n                # Statistics\n                running_loss += loss.item()*inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n            if phase == 'train':\n                scheduler.step()\n\n            epoch_loss = running_loss/len(datasets[phase])\n            epoch_acc = running_corrects.double()/len(datasets[phase])\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n                phase, epoch_loss, epoch_acc))\n\n            if phase == 'valid' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n\n        print()\n\n    time_elapsed = time.time()-since\n    print('Training complete in {:.0f}m {:.0f}s'.format(\n        time_elapsed // 60, time_elapsed % 60))\n    print('Best val Acc: {:4f}'.format(best_acc))\n\n    model.load_state_dict(best_model_wts)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from vision_transformer_pytorch import VisionTransformer\n\n# model_name = 'efficientnet-b7'\ndatasets={'train':train_dataset,'valid':valid_dataset}\ndataloaders={'train':train_loader,'valid':valid_loader}\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# model=models.(pretrained=True)\n# model.fc=nn.Linear(512,5)\n# model = EfficientNet.from_pretrained(model_name, num_classes=5) \n# model=models.resnext50_32x4d()#Add Pretrained=True to use pretrained with internet enabled\n\n\n# model.fc=nn.Linear(model.fc.in_features,5)\n# model = VisionTransformer.from_name('ViT-B_16', num_classes=5) \n\n# optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.001)\n# scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1)\n# criterion=nn.CrossEntropyLoss()\n# num_epochs=6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install --no-deps '../input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install timm\nimport timm\n\n\n\nprint(\"Available Vision Transformer Models: \")\ntimm.list_models(\"vit*\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ViTBase16(nn.Module):\n    def __init__(self, n_classes, pretrained=False):\n\n        super(ViTBase16, self).__init__()\n\n        # self.model = timm.create_model(\"vit_base_patch16_224\", pretrained=False)\n\n        self.model = timm.create_model(\"vit_base_patch16_384\", pretrained=True)\n        #         if pretrained:\n        #             self.model.load_state_dict(torch.load(MODEL_PATH))\n\n        self.model.head = nn.Linear(self.model.head.in_features, n_classes)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n\n    def train_one_epoch(self, train_loader, criterion, optimizer, device):\n        # keep track of training loss\n        epoch_loss = 0.0\n        epoch_accuracy = 0.0\n\n        ###################\n        # train the model #\n        ###################\n        self.model.train()\n        for i, (data, target) in enumerate(train_loader):\n            # move tensors to GPU if CUDA is available\n            if device.type == \"cuda\":\n                data, target = data.cuda(), target.cuda()\n            elif device.type == \"xla\":\n                data = data.to(device, dtype=torch.float32)\n                target = target.to(device, dtype=torch.int64)\n\n            # clear the gradients of all optimized variables\n            optimizer.zero_grad()\n            # forward pass: compute predicted outputs by passing inputs to the model\n            output = self.forward(data)\n            # calculate the batch loss\n            loss = criterion(output, target)\n            # backward pass: compute gradient of the loss with respect to model parameters\n            loss.backward()\n            # Calculate Accuracy\n            accuracy = (output.argmax(dim=1) == target).float().mean()\n            # update training loss and accuracy\n            epoch_loss += loss\n            epoch_accuracy += accuracy\n\n            # perform a single optimization step (parameter update)\n            if device.type == \"xla\":\n                xm.optimizer_step(optimizer)\n\n                if i % 20 == 0:\n                    xm.master_print(f\"\\tBATCH {i+1}/{len(train_loader)} - LOSS: {loss}\")\n\n            else:\n                optimizer.step()\n\n        return epoch_loss / len(train_loader), epoch_accuracy / len(train_loader)\n\n    def validate_one_epoch(self, valid_loader, criterion, device):\n        # keep track of validation loss\n        valid_loss = 0.0\n        valid_accuracy = 0.0\n\n        ######################\n        # validate the model #\n        ######################\n        self.model.eval()\n        for data, target in valid_loader:\n            # move tensors to GPU if CUDA is available\n            if device.type == \"cuda\":\n                data, target = data.cuda(), target.cuda()\n            elif device.type == \"xla\":\n                data = data.to(device, dtype=torch.float32)\n                target = target.to(device, dtype=torch.int64)\n\n            with torch.no_grad():\n                # forward pass: compute predicted outputs by passing inputs to the model\n                output = self.model(data)\n                # calculate the batch loss\n                loss = criterion(output, target)\n                # Calculate Accuracy\n                accuracy = (output.argmax(dim=1) == target).float().mean()\n                # update average validation loss and accuracy\n                valid_loss += loss\n                valid_accuracy += accuracy\n\n        return valid_loss / len(valid_loader), valid_accuracy / len(valid_loader)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Save the model after training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# torch.save(model.state_dict(), 'ViT-B_16.pt')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Load the model when model is trained and saved and notebook has to be run without internet"},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.load_state_dict(torch.load('../input/vit-model-1/ViT-B_16.pt'))\n\n# model = VisionTransformer.from_name('ViT-B_16',\n#                                     image_size=384, patch_size=16, num_classes=5) \n\n\nmodel = ViTBase16(n_classes=5, pretrained=False)\n\n\nmodel.load_state_dict(torch.load('../input/cassava-vit-train-384/model_0.bin'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")\nimage_path = \"../input/cassava-leaf-disease-classification/test_images/\"\n# fake targets\ntest_targets = test_df.label.values\n\n\ntest_aug = albu.Compose([\n            albu.CenterCrop(512, 512, p=1.),\n            albu.Resize(384, 384),\n            albu.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0),\n            ToTensorV2()], p=1.)\n\ntest_dataset=CassavaDataset(\n    df=test_df,\n    imfolder=image_path,\n    train=False,\n    transforms=test_aug\n)\n\ntest_loader =  DataLoader(\n        test_dataset,\n        batch_size=4,\n        num_workers=4,\n        shuffle=False,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions=[]\n\nfor imgs in test_loader:\n\n    imgs = imgs.to(device)\n    with torch.no_grad():\n        model=model.to(device)\n        outputs = model(imgs)\n        _, predicted = torch.max(outputs, dim=1)\n        predicted=predicted.to('cpu')\n        predictions.append(predicted)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['label'] = np.concatenate(predictions)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}