{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install vision_transformer_pytorch","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n\npackage_path = '../input/vision-transformer-pytorch/VisionTransformer-Pytorch'\nsys.path.append(package_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import libraries\nimport os\nimport pandas as pd\nimport albumentations as albu\nimport matplotlib.pyplot as plt\nimport json\nimport seaborn as sns\nimport cv2\nimport albumentations as albu\nimport numpy as np","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR=\"../input/cassava-leaf-disease-classification/\"\nTRAIN_IMAGES_DIR=os.path.join(BASE_DIR,'train_images')\n\ntrain_df=pd.read_csv(os.path.join(BASE_DIR,'train.csv'))","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Count of training images {0}\".format(len(os.listdir(TRAIN_IMAGES_DIR))))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(f'{BASE_DIR}/label_num_to_disease_map.json', 'r') as f:\n    name_mapping = json.load(f)\n    \nname_mapping = {int(k): v for k, v in name_mapping.items()}\ntrain_df[\"class_id\"]=train_df[\"label\"].map(name_mapping)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name_mapping","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualization","metadata":{}},{"cell_type":"code","source":"def visualize_images(image_ids,labels):\n    plt.figure(figsize=(16,12))\n    \n    for ind,(image_id,label) in enumerate(zip(image_ids,labels)):\n        plt.subplot(3,3,ind+1)\n        \n        image=cv2.imread(os.path.join(TRAIN_IMAGES_DIR,image_id))\n        image=cv2.cvtColor(image,cv2.COLOR_BGR2RGB)\n        \n        plt.imshow(image)\n        plt.title(f\"Class: {label}\",fontsize=12)\n        \n        plt.axis(\"off\")\n    plt.show()\n    \n\ndef plot_augmentation(image_id,transform):\n    plt.figure(figsize=(16,4))\n    \n    img=cv2.imread(os.path.join(TRAIN_IMAGES_DIR,image_id))\n    img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n    \n    plt.subplot(1,3,1)\n    plt.imshow(img)\n    plt.axis(\"off\")\n    \n    plt.subplot(1,3,2)\n    x=transform(image=img)[\"image\"]\n    plt.imshow(x)\n    plt.axis(\"off\")\n    \n    plt.subplot(1,3,3)\n    x=transform(image=img)[\"image\"]\n    plt.imshow(x)\n    plt.axis(\"off\")\n    \n    plt.show()\n    \n    \ndef visualize(images, transform):\n    \"\"\"\n    Plot images and their transformations\n    \"\"\"\n    fig = plt.figure(figsize=(32, 16))\n    \n    for i, im in enumerate(images):\n        ax = fig.add_subplot(2, 5, i + 1, xticks=[], yticks=[])\n        plt.imshow(im)\n        \n    for i, im in enumerate(images):\n        ax = fig.add_subplot(2, 5, i + 6, xticks=[], yticks=[])\n        plt.imshow(transform(image=im)['image'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modelling","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold, train_test_split\nfrom albumentations.pytorch import ToTensorV2\n# from efficientnet_pytorch import EfficientNet\nimport time\nimport datetime\nimport copy","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DataSet class\n\nclass CassavaDataset(Dataset):\n    def __init__(self,df:pd.DataFrame,imfolder:str,train:bool = True, transforms=None):\n        self.df=df\n        self.imfolder=imfolder\n        self.train=train\n        self.transforms=transforms\n        \n    def __getitem__(self,index):\n        im_path=os.path.join(self.imfolder,self.df.iloc[index]['image_id'])\n        x=cv2.imread(im_path,cv2.IMREAD_COLOR)\n        x=cv2.cvtColor(x,cv2.COLOR_BGR2RGB)\n        \n        if(self.transforms):\n            x=self.transforms(image=x)['image']\n        \n        if(self.train):\n            y=self.df.iloc[index]['label']\n            return x,y\n        else:\n            return x\n        \n    def __len__(self):\n        return len(self.df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_augs = albu.Compose([\n    albu.RandomResizedCrop(height=384, width=384, p=1.0),\n    albu.HorizontalFlip(p=0.5),\n    albu.VerticalFlip(p=0.5),\n    albu.RandomBrightnessContrast(p=0.5),\n    albu.ShiftScaleRotate(p=0.5),\n    albu.Normalize(    \n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],),\n    ToTensorV2(),\n])\n\nvalid_augs = albu.Compose([\n    albu.Resize(height=384, width=384, p=1.0),\n    albu.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],),\n    ToTensorV2(),\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, valid = train_test_split(\n    train_df, \n    test_size=0.1, \n    random_state=42,\n    stratify=train_df.label.values\n)\n\n\n# reset index on both dataframes\ntrain = train.reset_index(drop=True)\nvalid = valid.reset_index(drop=True)\n\ntrain_targets = train.label.values\n\n# targets for validation\nvalid_targets = valid.label.values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset=CassavaDataset(\n    df=train,\n    imfolder=TRAIN_IMAGES_DIR,\n    train=True,\n    transforms=train_augs\n)\n\nvalid_dataset=CassavaDataset(\n    df=valid,\n    imfolder=TRAIN_IMAGES_DIR,\n    train=True,\n    transforms=valid_augs\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_image(img_dict):\n    image_tensor = img_dict[0]\n#     print(type(image_tensor))\n    target = img_dict[1]\n    print(target)\n    plt.figure(figsize=(10, 10))\n    image = image_tensor.permute(1, 2, 0) \n    plt.imshow(image)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_image(train_dataset[5])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(\n    train_dataset,\n    batch_size=16,\n    num_workers=4,\n    shuffle=True,\n)\n\nvalid_loader = DataLoader(\n    valid_dataset,\n    batch_size=16,\n    num_workers=4,\n    shuffle=False,\n)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(datasets, dataloaders, model, criterion, optimizer, scheduler, num_epochs, device):\n    since = time.time()\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n\n    for epoch in range(num_epochs):\n        print('Epoch {}/{}'.format(epoch, num_epochs-1))\n        print('-' * 10)\n\n        for phase in ['train', 'valid']:\n            if phase == 'train':\n                model.train()\n            else:\n                model.eval()\n\n            running_loss = 0.0\n            running_corrects = 0.0\n\n            for inputs, labels in dataloaders[phase]:\n                inputs = inputs.to(device)\n                labels=labels.to(device)\n\n                # Zero out the grads\n                optimizer.zero_grad()\n\n                # Forward\n                # Track history in train mode\n                with torch.set_grad_enabled(phase == 'train'):\n                    model=model.to(device)\n                    outputs = model(inputs)\n                    _, preds = torch.max(outputs, 1)\n                    loss = criterion(outputs, labels)\n\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n                # Statistics\n                running_loss += loss.item()*inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n            if phase == 'train':\n                scheduler.step()\n\n            epoch_loss = running_loss/len(datasets[phase])\n            epoch_acc = running_corrects.double()/len(datasets[phase])\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n                phase, epoch_loss, epoch_acc))\n\n            if phase == 'valid' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n\n        print()\n\n    time_elapsed = time.time()-since\n    print('Training complete in {:.0f}m {:.0f}s'.format(\n        time_elapsed // 60, time_elapsed % 60))\n    print('Best val Acc: {:4f}'.format(best_acc))\n\n    model.load_state_dict(best_model_wts)\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from vision_transformer_pytorch import VisionTransformer\n\n# model_name = 'efficientnet-b7'\ndatasets={'train':train_dataset,'valid':valid_dataset}\ndataloaders={'train':train_loader,'valid':valid_loader}\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# model=models.(pretrained=True)\n# model.fc=nn.Linear(512,5)\n# model = EfficientNet.from_pretrained(model_name, num_classes=5) \n# model=models.resnext50_32x4d()#Add Pretrained=True to use pretrained with internet enabled\n# model.fc=nn.Linear(model.fc.in_features,5)\nmodel = VisionTransformer.from_name('ViT-B_16', num_classes=5) \n\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.001)\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1)\ncriterion=nn.CrossEntropyLoss()\nnum_epochs=6","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train the model after uncommenting below","metadata":{}},{"cell_type":"code","source":"trained_model=train_model(datasets,dataloaders,model,criterion,optimizer,scheduler,num_epochs,device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Epoch 0/5\n# ----------\n# train Loss: 0.5318 Acc: 0.8119\n# valid Loss: 0.4009 Acc: 0.8650\n\n# Epoch 1/5\n# ----------\n# train Loss: 0.4384 Acc: 0.8467\n# valid Loss: 0.3999 Acc: 0.8612\n\n# Epoch 2/5\n# ----------\n# train Loss: 0.3511 Acc: 0.8778\n# valid Loss: 0.3558 Acc: 0.8771\n\n# Epoch 3/5\n# ----------\n# train Loss: 0.3266 Acc: 0.8879\n# valid Loss: 0.3468 Acc: 0.8836\n\n# Epoch 4/5\n# ----------\n# train Loss: 0.3066 Acc: 0.8924\n# valid Loss: 0.3384 Acc: 0.8911\n\n# Epoch 5/5\n# ----------\n# train Loss: 0.3060 Acc: 0.8926\n# valid Loss: 0.3421 Acc: 0.8869\n\n# Training complete in 121m 52s\n# Best val Acc: 0.891121","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Save the model after training","metadata":{}},{"cell_type":"code","source":"torch.save(model.state_dict(), 'ViT-B_16.pt')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load the model when model is trained and saved and notebook has to be run without internet","metadata":{}},{"cell_type":"code","source":"model.load_state_dict(torch.load('../input/vit-model-1/ViT-B_16.pt'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")\nimage_path = \"../input/cassava-leaf-disease-classification/test_images/\"\n# fake targets\ntest_targets = test_df.label.values\n\ntrain_augs = albu.Compose([\n    albu.RandomResizedCrop(height=384, width=384, p=1.0),\n    albu.HorizontalFlip(p=0.5),\n    albu.VerticalFlip(p=0.5),\n    albu.RandomBrightnessContrast(p=0.5),\n    albu.ShiftScaleRotate(p=0.5),\n    albu.Normalize(    \n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],),\n    ToTensorV2(),\n])\n\nvalid_augs = albu.Compose([\n    albu.Resize(height=384, width=384, p=1.0),\n    albu.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],),\n    ToTensorV2(),\n])\n\ntest_aug = albu.Compose([\n            albu.CenterCrop(512, 512, p=1.),\n            albu.Resize(384, 384),\n            albu.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0),\n            ToTensorV2()], p=1.)\n\ntest_dataset=CassavaDataset(\n    df=test_df,\n    imfolder=image_path,\n    train=False,\n    transforms=test_aug\n)\n\ntest_loader =  DataLoader(\n        test_dataset,\n        batch_size=4,\n        num_workers=4,\n        shuffle=False,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions=[]\n\nfor imgs in test_loader:\n\n    imgs = imgs.to(device)\n    with torch.no_grad():\n        model=model.to(device)\n        outputs = model(imgs)\n        _, predicted = torch.max(outputs, dim=1)\n        predicted=predicted.to('cpu')\n        predictions.append(predicted)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['label'] = np.concatenate(predictions)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}