{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Cassava Leaf Disease Modelling\nThere is already a great kernel [Vision Transformer (ViT): Tutorial + Baseline](https://www.kaggle.com/abhinand05/vision-transformer-vit-tutorial-baseline) that shows us how to use visiontransformer on TPU and why. Here, we make further improvements on the basis of the official implementation, in order to provide better pre-training parameters and user-friendly API as `Effecientnet-PyTorch`. You can find all the details on https://github.com/tczhangzhi/VisionTransformer-Pytorch.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install vision_transformer_pytorch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nimport timm\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import libraries\nimport os\nimport pandas as pd\nimport albumentations as albu\nimport matplotlib.pyplot as plt\nimport json\nimport seaborn as sns\nimport cv2\nimport albumentations as albu\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualization"},{"metadata":{},"cell_type":"markdown","source":"# Modelling"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold, train_test_split\nfrom albumentations.pytorch import ToTensorV2\n# from efficientnet_pytorch import EfficientNet\nimport time\nimport datetime\nimport copy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DataSet class\n\nclass CassavaDataset(Dataset):\n    def __init__(self,df:pd.DataFrame,imfolder:str,train:bool = True, transforms=None):\n        self.df=df\n        self.imfolder=imfolder\n        self.train=train\n        self.transforms=transforms\n        \n    def __getitem__(self,index):\n        im_path=os.path.join(self.imfolder,self.df.iloc[index]['image_id'])\n        x=cv2.imread(im_path,cv2.IMREAD_COLOR)\n        x=cv2.cvtColor(x,cv2.COLOR_BGR2RGB)\n        \n        if(self.transforms):\n            x=self.transforms(image=x)['image']\n        \n        if(self.train):\n            y=self.df.iloc[index]['label']\n            return x,y\n        else:\n            return x\n        \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        print(self.model)\n        n_features = self.model.head.in_features\n        self.model.head = nn.Linear(n_features, n_class)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# model_name = 'efficientnet-b7'\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# model=models.(pretrained=True)\n# model.fc=nn.Linear(512,5)\n# model = EfficientNet.from_pretrained(model_name, num_classes=5) \n# model=models.resnext50_32x4d()#Add Pretrained=True to use pretrained with internet enabled\n# model.fc=nn.Linear(model.fc.in_features,5)\n#model = VisionTransformer.from_name('ViT-B_16', num_classes=5) \n#model = CassvaImgClassifier(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = CassvaImgClassifier('vit_large_patch16_384', n_class=5)\nmodel.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/vitweights","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train the model after uncommenting below"},{"metadata":{"trusted":true},"cell_type":"code","source":"# trained_model=train_model(datasets,dataloaders,model,criterion,optimizer,scheduler,num_epochs,device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Epoch 0/5\n# ----------\n# train Loss: 0.5318 Acc: 0.8119\n# valid Loss: 0.4009 Acc: 0.8650\n\n# Epoch 1/5\n# ----------\n# train Loss: 0.4384 Acc: 0.8467\n# valid Loss: 0.3999 Acc: 0.8612\n\n# Epoch 2/5\n# ----------\n# train Loss: 0.3511 Acc: 0.8778\n# valid Loss: 0.3558 Acc: 0.8771\n\n# Epoch 3/5\n# ----------\n# train Loss: 0.3266 Acc: 0.8879\n# valid Loss: 0.3468 Acc: 0.8836\n\n# Epoch 4/5\n# ----------\n# train Loss: 0.3066 Acc: 0.8924\n# valid Loss: 0.3384 Acc: 0.8911\n\n# Epoch 5/5\n# ----------\n# train Loss: 0.3060 Acc: 0.8926\n# valid Loss: 0.3421 Acc: 0.8869\n\n# Training complete in 121m 52s\n# Best val Acc: 0.891121","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Save the model after training"},{"metadata":{},"cell_type":"markdown","source":"Load the model when model is trained and saved and notebook has to be run without internet"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_state_dict(torch.load('../input/zzkvit/vit_large_patch16_384_best_fold.pth'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")\nimage_path = \"../input/cassava-leaf-disease-classification/test_images/\"\n# fake targets\ntest_targets = test_df.label.values\n\n\ntest_aug = albu.Compose([\n            albu.CenterCrop(512, 512, p=1.),\n            albu.Resize(384, 384),\n            albu.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0),\n            ToTensorV2()], p=1.)\n\ntest_dataset=CassavaDataset(\n    df=test_df,\n    imfolder=image_path,\n    train=False,\n    transforms=test_aug\n)\n\ntest_loader =  DataLoader(\n        test_dataset,\n        batch_size=4,\n        num_workers=4,\n        shuffle=False,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions=[]\n\nfor imgs in test_loader:\n\n    imgs = imgs.to(device)\n    with torch.no_grad():\n        model=model.to(device)\n        outputs = model(imgs)\n        _, predicted = torch.max(outputs, dim=1)\n        predicted=predicted.to('cpu')\n        predictions.append(predicted)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['label'] = np.concatenate(predictions)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/vitbaseline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}