{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = '../input/cassava-leaf-disease-classification/train_images/'\ntest_path = '../input/cassava-leaf-disease-classification/test_images/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")\ndf_test = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn \nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, CosineAnnealingWarmRestarts\nimport torchvision\nimport cv2\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nfrom albumentations import *\nfrom albumentations.pytorch import ToTensorV2\n#libary for efficientnet and VIT\nimport timm\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\nwarnings.simplefilter('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nimport os\ntrain_list = glob.glob(os.path.join(train_path, '*')) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nfor i in range(3*3):\n    plt.subplot(3,3,i+1)\n    img = cv2.imread(train_list[i])\n    img = img[:,:,::-1]\n    plt.imshow(img)\n    plt.title(df_train[df_train['image_id'] == train_list[i].split('/')[-1] ]['label'].values[0])\n    plt.xlabel(img.shape)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import model_selection\n#create kfold\ndf_train['kfold'] = -1\n#random row from data\ndf_train = df_train.sample(frac=1).reset_index(drop=True)\n#initiate kfold \nkf = model_selection.StratifiedKFold(n_splits=5)\n#fill kfold to data\nfor f, (t_, v_) in enumerate(kf.split(X=df_train, y=df_train.label.values)):\n  df_train.loc[v_, 'kfold'] = f\nprint(df_train['kfold'].value_counts())\n#save to csv\nfor fold in range(5):\n  train_fold = df_train[df_train['kfold'] != fold]\n  valid_fold = df_train[df_train['kfold'] == fold]\n  train_fold.to_csv(f'fold_{fold}_train.csv',index=None)\n  valid_fold.to_csv(f'fold_{fold}_valid.csv',index=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_size = 384\nepochs = 10\nbatch_size = 16\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Augments:\n  \"\"\"Contains Train, Validation and Testing Augments\"\"\"\n  train_augments = Compose([\n                            Resize(image_size,image_size),\n                            RandomResizedCrop(image_size, image_size),\n                            Transpose(p=0.5),\n                            HorizontalFlip(p=0.5),\n                            VerticalFlip(p=0.5),\n                            ShiftScaleRotate(p=0.5),\n                            Rotate(limit=45, p=0.5),\n                            HueSaturationValue(hue_shift_limit=0.2,sat_shift_limit=0.2,val_shift_limit=0.5),\n                            RandomBrightnessContrast(brightness_limit=(-0.1,0.1),contrast_limit=(-0.1,0.1),p=0.5),\n                            Normalize(mean=[0.485,0.456,0.406],std=[0.229,0.224,0.225],max_pixel_value=225.0,p=1.0),\n                            CoarseDropout(p=0.5),\n                            Cutout(p=0.5),\n                            ToTensorV2(p=1.0)\n  ],p=1.0\n  )\n\n  valid_augments = Compose([\n                            CenterCrop(image_size,image_size,p=1.),\n                            Resize(image_size,image_size),\n                            Normalize(mean=[0.485,0.456,0.406],std=[0.229,0.224, 0.225], max_pixel_value=225.0,p=1.0),\n                            ToTensorV2(p=1.0)\n  ],p=1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#define model\n#efficientnet-b7\nclass EfficientNetModel(nn.Module):\n  def __init__(self, num_classes = 5, model_name = 'efficientnet_b7',pretrained = True):\n    super(EfficientNetModel, self).__init__()\n    self.model = timm.create_model(model_name, pretrained=pretrained)\n    self.model.fc = nn.Linear(self.model.classifier.in_features, num_classes)\n\n  def forward(self, x):\n    x = self.model(x)\n    return x\n\n#VIT model\nclass VITModel(nn.Module):\n  def __init__(self, num_classes = 5, model_name = 'vit_base_patch16_384', pretrained = True):\n    super(VITModel, self).__init__()\n    self.model = timm.create_model(model_name, pretrained= pretrained)\n    self.model.fc = nn.Linear(self.model.head.in_features, num_classes)\n  \n  def forward(self, x):\n    x = self.model(x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Datasetclass\nclass CustomDataset(Dataset):\n  def __init__(self, df, num_classes = 5, is_train = True, augments=None, \n               image_size = image_size,folder_path = train_path):\n    super().__init__()\n    self.df = df.sample(frac = 1).reset_index(drop=True)\n    self.num_classes = num_classes\n    self.is_train = is_train\n    self.augments = augments\n    self.image_size = image_size\n    self.folder_path = folder_path\n\n  def __len__(self):\n    return len(self.df)\n\n  def __getitem__(self, idx):\n    img_path = os.path.join(self.folder_path,self.df['image_id'][idx])\n    img = cv2.imread(img_path)\n    img = img[ :, :, ::-1]\n    #Augmentation\n    if self.augments:\n        img = self.augments(image=img)['image']\n    if self.is_train:\n        label = self.df['label'][idx]\n        return img, label\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train function\ndef train_one_cycle(model, dataloader, loss_fn, optim):\n    model.train() #train mode\n    train_prog = tqdm(dataloader, total = len(dataloader))\n    \n    all_labels = []\n    all_preds = []\n    \n    run_loss = 0.0\n    scaler = GradScaler()\n    \n    for inputs, labels in train_prog:\n        inputs = inputs.to(device).float()\n        labels = labels.to(device).long()\n        with autocast():\n            #get prediction\n            outputs = model(inputs)\n            #loss\n            train_loss = loss_fn(outputs, labels)\n            #backward\n            scaler.scale(train_loss).backward()\n            \n            scaler.step(optim)\n            scaler.update()\n            optim.zero_grad()\n            \n            #loss\n            run_loss += train_loss\n            \n            #accuracy\n            preds = torch.argmax(outputs, 1).detach().cpu().numpy()\n            labels = labels.detach().cpu().numpy()\n            \n            #append to list\n            all_preds += [preds]\n            all_labels += [labels]\n        #show current process\n        train_pbar = f'loss: {train_loss.item():.3f}'\n        train_prog.set_description(desc = train_pbar)\n    #calculate total accuracy\n    all_preds = np.concatenate(all_preds)\n    all_labels = np.concatenate(all_labels)\n    acc = (all_preds == all_labels).mean()\n    print(f'Training Accuracy: {acc:.3f}')\n    #calculate loss\n    floss = run_loss / len(dataloader)\n    #free memory\n    del all_preds, all_labels, run_loss\n    return (acc, floss)\n            \n            \n#train function\ndef valid_one_cycle(model, dataloader, loss_fn):\n    model.eval() #eval model\n    valid_prog = tqdm(dataloader, total = len(dataloader))\n    with torch.no_grad():\n        all_labels = []\n        all_preds = []\n\n        run_loss = 0.0\n        scaler = GradScaler()\n\n        for inputs, labels in valid_prog:\n            inputs = inputs.to(device).float()\n            labels = labels.to(device).long()\n            #get prediction\n            outputs = model(inputs)\n            #loss\n            valid_loss = loss_fn(outputs, labels)\n            #loss\n            run_loss += valid_loss.item()\n\n            #accuracy\n            preds = torch.argmax(outputs, 1).detach().cpu().numpy()\n            labels = labels.detach().cpu().numpy()\n\n            #append to list\n            all_preds += [preds]\n            all_labels += [labels]\n            #show current process\n            valid_pbar = f'loss: {valid_loss.item():.3f}'\n            valid_prog.set_description(desc = valid_pbar)\n        #calculate total accuracy\n        all_preds = np.concatenate(all_preds)\n        all_labels = np.concatenate(all_labels)\n        acc = (all_preds == all_labels).mean()\n        print(f'Valid Accuracy: {acc:.3f}')\n        #calculate loss\n        floss = run_loss / len(dataloader)\n        #free memory\n        del all_preds, all_labels, run_loss\n    return (acc, floss, model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_results(train_acc, valid_acc, train_loss, valid_loss, nb_epochs):\n    epochs = [i for i in range(nb_epochs)]\n    \n    fig, ax = plt.subplots(1, 2)\n    fig.set_size_inches(20, 10)\n    \n    ax[0].plot(epochs, train_acc, 'go-', label='Training Accuracy')\n    ax[0].plot(epochs, valid_acc, 'ro-', label='Validation Accuracy')\n    ax[0].set_title('Training & Validation Accuracy')\n    ax[0].legend()\n    ax[0].set_xlabel('Epochs')\n    ax[0].set_ylabel('Accuracy')\n    \n    ax[1].plot(epochs, train_loss, 'go-', label='Training Loss')\n    ax[1].plot(epochs, valid_loss, 'ro-', label='Validation Loss')\n    ax[1].set_title('Training & Validation Loss')\n    ax[1].legend()\n    ax[1].set_xlabel('Epochs')\n    ax[1].set_ylabel('Loss')\n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run(fold):\n  train_fold = pd.read_csv(f\"./fold_{fold}_train.csv\")\n  valid_fold = pd.read_csv(f\"./fold_{fold}_valid.csv\")\n\n  train_set = CustomDataset(df=train_fold, augments=Augments.train_augments)\n  valid_set = CustomDataset(df=valid_fold, augments=Augments.valid_augments)\n\n  train = DataLoader(train_set,batch_size=batch_size,shuffle=True,pin_memory=False,drop_last=False,num_workers=8)\n\n  valid = DataLoader(valid_set,batch_size=batch_size,shuffle=False,pin_memory=False,num_workers=8)\n\n  model = VITModel(num_classes=5, model_name='vit_base_patch16_384').to(device)\n  optim = torch.optim.AdamW(model.parameters(), lr=1e-5, weight_decay=1e-6)\n  loss_fn = nn.CrossEntropyLoss().to(device)\n\n  train_accs = []\n  valid_accs = []\n  train_losses = []\n  valid_losses = []\n  best_acc = 0.0\n    \n  #scaler = GradScaler()\n\n  for epoch in range(epochs):\n      print(f\"{'-'*20} EPOCH: {epoch}/{epochs} {'-'*20}\")\n\n      # Run one training epoch\n      current_train_acc, current_train_loss = train_one_cycle(model = model, dataloader = train, loss_fn=loss_fn, optim =optim)\n      train_accs.append(current_train_acc)\n      train_losses.append(current_train_loss)\n\n      # Run one validation epoch\n      current_val_acc, current_val_loss, op_model = valid_one_cycle(model = model, dataloader = valid,loss_fn = loss_fn)\n      valid_accs.append(current_val_acc)\n      valid_losses.append(current_val_loss)\n\n      # Empty CUDA cache\n      torch.cuda.empty_cache()\n      \n      # Save the model every epoch\n      if best_acc < current_val_acc:\n            print(f\"Saving Model for this epoch...\")\n            torch.save(op_model.state_dict(), f\"vit_base_p16_384_fold_{fold}_model.pth\")\n      \n  del train_set, valid_set, train, valid #, scaler\n  torch.cuda.empty_cache()\n  print(f'Best Accuracy of {fold} fold: {best_acc:.3f}')  \n  #plot_results(train_accs, valid_accs, train_losses, valid_losses, nb_epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold in range(1):\n  run(fold)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = VITModel(num_classes=5, model_name='vit_base_patch16_384').to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_state_dict(torch.load('../input/model-kfold0/vit_base_p16_384_fold_0_model.pth'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#prediction\ndef show_predictions(model, data_loader):\n    preds = []\n    model = model.eval()\n    with torch.no_grad():\n        for i, (inputs, labels) in enumerate(data_loader):\n          inputs = inputs.to(device)\n          labels = labels.to(device)\n\n          outputs = model(inputs)\n          _, pred = torch.max(outputs, 1)\n          pred = pred.cpu().numpy()\n          preds.append(pred[0])\n    return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_set = CustomDataset(df=df_test, augments=Augments.valid_augments,folder_path = test_path)\ntest = DataLoader(test_set,batch_size=batch_size,shuffle=False,pin_memory=False,num_workers=8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = show_predictions(model, test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test['label'] = predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.to_csv(\"submission.csv\",index=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}