{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Cassava Leaf Disease Modelling\nThere is already a great kernel [Vision Transformer (ViT): Tutorial + Baseline](https://www.kaggle.com/abhinand05/vision-transformer-vit-tutorial-baseline) that shows us how to use visiontransformer on TPU and why. Here, we make further improvements on the basis of the official implementation, in order to provide better pre-training parameters and user-friendly API as `Effecientnet-PyTorch`. You can find all the details on https://github.com/tczhangzhi/VisionTransformer-Pytorch.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\n\npackage_paths = [\n    '../input/pytorch-image-models/pytorch-image-models-master', #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\n    '../input/image-fmix/FMix-master'\n]\nfor pth in package_paths:\n    sys.path.append(pth)\nimport timm\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport glob\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom  torch.cuda.amp import autocast, GradScaler\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\nimport timm #from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom\nfrom sklearn.metrics import log_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import libraries\nimport os\nimport pandas as pd\nimport albumentations as albu\nimport matplotlib.pyplot as plt\nimport json\nimport seaborn as sns\nimport cv2\nimport albumentations as albu\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda:0')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualization"},{"metadata":{},"cell_type":"markdown","source":"# Modelling"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold, train_test_split\nfrom albumentations.pytorch import ToTensorV2\n# from efficientnet_pytorch import EfficientNet\nimport time\nimport datetime\nimport copy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 10,\n    'seed': 719,\n    'model_arch': 'tf_efficientnet_b3_ns',\n    'img_size': 384,\n    'epochs': 32,\n    'train_bs': 32,\n    'valid_bs': 32,\n    'lr': 1e-4,\n    'num_workers': 4,\n    'accum_iter': 1, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'tta': 4,\n    'used_epochs': [8],#[6,7,8,9],\n    'weights': [1,1,1,1,1]\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DataSet class\n\nclass CassavaDataset(Dataset):\n    def __init__(self,df:pd.DataFrame,imfolder:str,train:bool = True, transforms=None):\n        self.df=df\n        self.imfolder=imfolder\n        self.train=train\n        self.transforms=transforms\n        \n    def __getitem__(self,index):\n        im_path=os.path.join(self.imfolder,self.df.iloc[index]['image_id'])\n        x=cv2.imread(im_path,cv2.IMREAD_COLOR)\n        x=cv2.cvtColor(x,cv2.COLOR_BGR2RGB)\n        \n        if(self.transforms):\n            x=self.transforms(image=x)['image']\n        \n        if(self.train):\n            y=self.df.iloc[index]['label']\n            return x,y\n        else:\n            return x\n        \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from vision_transformer_pytorch import VisionTransformer\n\n# model_name = 'efficientnet-b7'\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nclass CustomDeiT(nn.Module):\n    def __init__(self, model_name='model_name', pretrained=False):\n        super().__init__()\n        self.model = torch.hub.load('facebookresearch/deit:main', model_name, pretrained=0)\n        n_features = self.model.head.in_features\n        self.model.head = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\nclass CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        print(self.model)\n        n_features = self.model.head.in_features\n        self.model.head = nn.Linear(n_features, n_class)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \nclass CustomViT(nn.Module):\n    def __init__(self, model_name='', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.head.in_features\n        self.model.head = nn.Linear(n_features,5)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n# model = create_model(\n#         'deit_base_patch16_224',\n#         pretrained=False,\n#         num_classes=5,\n        \n#         drop_block_rate=None,\n#     ).to(device)\n# model=models.(pretrained=True)\n# model.fc=nn.Linear(512,5)\n# model = EfficientNet.from_pretrained(model_name, num_classes=5) \n# model=models.resnext50_32x4d()#Add Pretrained=True to use pretrained with internet enabled\n# model.fc=nn.Linear(model.fc.in_features,5)\nmodel = CustomViT(model_name='vit_base_patch16_384', pretrained=False).to(device)\ndevice = torch.device(CFG['device'])\nmodel.to(device)\nmodel.eval()\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train the model after uncommenting below"},{"metadata":{"trusted":true},"cell_type":"code","source":"# trained_model=train_model(datasets,dataloaders,model,criterion,optimizer,scheduler,num_epochs,device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Epoch 0/5\n# ----------\n# train Loss: 0.5318 Acc: 0.8119\n# valid Loss: 0.4009 Acc: 0.8650\n\n# Epoch 1/5\n# ----------\n# train Loss: 0.4384 Acc: 0.8467\n# valid Loss: 0.3999 Acc: 0.8612\n\n# Epoch 2/5\n# ----------\n# train Loss: 0.3511 Acc: 0.8778\n# valid Loss: 0.3558 Acc: 0.8771\n\n# Epoch 3/5\n# ----------\n# train Loss: 0.3266 Acc: 0.8879\n# valid Loss: 0.3468 Acc: 0.8836\n\n# Epoch 4/5\n# ----------\n# train Loss: 0.3066 Acc: 0.8924\n# valid Loss: 0.3384 Acc: 0.8911\n\n# Epoch 5/5\n# ----------\n# train Loss: 0.3060 Acc: 0.8926\n# valid Loss: 0.3421 Acc: 0.8869\n\n# Training complete in 121m 52s\n# Best val Acc: 0.891121","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Save the model after training"},{"metadata":{},"cell_type":"markdown","source":"Load the model when model is trained and saved and notebook has to be run without internet"},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass CassavaDataset(Dataset):\n    def __init__(\n        self, df, data_root, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        \n        img  = get_img(path)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations.pytorch import ToTensorV2\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout,RandomCrop, ShiftScaleRotate, CenterCrop, Resize\n)\n\ndef get_inference_transforms(CFG):\n    return Compose([\n            CenterCrop(512,512),\n            Resize(CFG['img_size'], CFG['img_size']),\n           Transpose(p=0.5),\n           HorizontalFlip(p=0.5),\n           VerticalFlip(p=0.5),\n#             HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n#             RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nfrom tqdm import tqdm\ndef inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)   #output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all\nif __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])\n    \n#     folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    tst_preds = []\n        \n    if 1:\n        # we'll train fold 0 first\n\n\n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms(CFG), output_label=False)\n\n        tst_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n\n        device = torch.device(CFG['device'])\n#         if CFG['model_arch'] == 'resnext50_32x4d':\n#             model = CustomResNext().to(device)\n#         if 'efficientnet' in CFG['model_arch']:\n#             model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique()).to(device)\n\n        #for epoch in range(CFG['epochs']-3):\n        for (i,path) in enumerate(glob.glob('../input/vit5fold/*.pth')):    \n            model.load_state_dict(torch.load(path)['model'])\n            \n            with torch.no_grad():\n                for _ in range(CFG['tta']):\n                    #val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]\n                    tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]\n\n        #val_preds = np.mean(val_preds, axis=0)         \n        #print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))\n       #print('fold {} validation accuracy = {:.5f}'.format(fold, (valid_.label.values==np.argmax(val_preds, axis=1)).mean()))\n        del model\n        torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tst_preds = np.mean(tst_preds, axis=0) \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['label'] = np.argmax(tst_preds, axis=1)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}