{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport timm\nimport cv2\nimport torch.optim as optim\nimport torchvision.models as models\nimport torchvision\n\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nfrom sklearn import model_selection, metrics\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\nfrom transformers import ViTModel\nfrom torch.optim.lr_scheduler import StepLR\nfrom transformers import ViTForImageClassification, ViTConfig\nfrom statistics import mean\n\nplt.style.use('fivethirtyeight')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"use_cuda = torch.cuda.is_available()\ndevice = torch.device('cuda' if use_cuda else 'cpu')\nuse_cuda, device","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_PATH = '../input/cassava-leaf-disease-classification'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image = cv2.imread('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\nplt.imshow(image)\nplt.show()\nimage.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE = 224\nBATCH_SIZE = 16\nLR = 1e-4\nEPOCHS = 10\nnum_classes = 5\n\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/cassava-leaf-disease-classification/train.csv')\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = train[:1000]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(6, 3))\n\ntrain['label'].value_counts().plot(\n    kind='bar',\n    color='#558364',\n    width=0.7\n)\n\nplt.xlabel('Label', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.title('Distribution of Labels', fontsize=15)\nplt.xticks(rotation=360)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, val_df = model_selection.train_test_split(\n    train, test_size=0.12, random_state=42, stratify=train['label'].values\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LeafDataset(torch.utils.data.Dataset):\n    \n    def __init__(self, df, data_path=BASE_PATH, mode='train', transforms=None):\n        super().__init__()\n        self.df_data = df.values\n        self.data_path = data_path\n        self.transforms = transforms \n        self.mode = mode\n        self.data_dir = 'train_images' if mode == 'train' else 'test_images'\n    \n    def __len__(self):\n        return len(self.df_data)\n    \n    def __getitem__(self, index):\n        img_name, label = self.df_data[index]\n        img_path = os.path.join(self.data_path, self.data_dir, img_name)\n        img = Image.open(img_path).convert('RGB')\n        \n        if self.transforms is not None:\n            img = self.transforms(img)\n            \n        return img, label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transforms_train = transforms.Compose([\n    transforms.Resize((IMG_SIZE, IMG_SIZE)),\n    transforms.RandomHorizontalFlip(p=0.2),\n    transforms.RandomVerticalFlip(p=0.2),\n    transforms.RandomResizedCrop(IMG_SIZE),\n    transforms.ToTensor(),\n    transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))\n])\n\ntransforms_val = transforms.Compose([\n    transforms.Resize((IMG_SIZE, IMG_SIZE)),\n    transforms.ToTensor(),\n    transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = LeafDataset(df=train_df, data_path=BASE_PATH, mode='train', transforms=transforms_train)\nval_dataset = LeafDataset(df=val_df, data_path=BASE_PATH, mode='train', transforms=transforms_val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_dataset), len(val_dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Vit Pretrained Model","metadata":{}},{"cell_type":"code","source":"model = ViTForImageClassification.from_pretrained(\"google/vit-base-patch16-224\")\nmodel.classifier = nn.Linear(in_features=768, out_features=num_classes, bias=True)\nmodel.to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_losses = []\ntrain_accs = []\nval_losses = []\nval_accs = []\n\ndef train_model(model, train_dataset, val_dataset, learning_rate, epochs):\n\n    train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n    val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=BATCH_SIZE)\n    \n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n    scheduler = StepLR(optimizer, step_size=1, gamma=0.9)\n    \n    if use_cuda:\n        model = model.cuda()\n        criterion = criterion.cuda()\n\n    for epoch_num in range(epochs):\n\n            total_acc_train = 0\n            total_loss_train = 0\n\n            for train_images, train_labels in tqdm(train_dataloader):\n                \n                train_images = train_images.to(device)\n                train_labels = train_labels.to(device)\n                \n                optimizer.zero_grad()\n\n                output = model(train_images)\n                \n                batch_loss = criterion(output.logits, train_labels.long())\n                total_loss_train += batch_loss.item()\n            \n                _, predicted = torch.max(output.logits.data, 1)\n                acc = (predicted == train_labels).sum().item()\n                total_acc_train += acc\n\n                batch_loss.backward()\n                optimizer.step()\n                \n            scheduler.step()\n            \n            total_acc_val = 0\n            total_loss_val = 0\n\n            with torch.no_grad():\n\n                for val_images, val_labels in val_dataloader:\n                    \n                    val_images = val_images.to(device)\n                    val_labels = val_labels.to(device)\n                    \n                    output = model(val_images)\n\n                    batch_loss = criterion(output.logits, val_labels.long())\n                    total_loss_val += batch_loss.item()\n                    \n                    _, predicted = torch.max(output.logits.data, 1)\n                    acc = (predicted == val_labels).sum().item()\n                    total_acc_val += acc\n            \n            print(f'Epochs: {epoch_num + 1} | Train Loss: {total_loss_train / len(train_dataset): .3f} \\\n            | Train Accuracy: {total_acc_train / len(train_dataset): .3f} \\\n            | Val Loss: {total_loss_val / len(val_dataset): .3f} \\\n            | Val Accuracy: {total_acc_val / len(val_dataset): .3f}')\n            \n            train_losses.append(total_loss_train / len(train_dataset))\n            train_accs.append(total_acc_train / len(train_dataset))\n            val_losses.append(total_loss_val / len(val_dataset))\n            val_accs.append(total_acc_val / len(val_dataset))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_model(model, train_dataset, val_dataset, LR, 10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(18, 6))\n\nplt.plot(\n    train_losses, \n    label='Train_Losses', \n    color='red', \n    linewidth=1.5\n)\nplt.plot(\n    val_losses, \n    label='Val_Losses', \n    color='blue', \n    linewidth=1.5\n)\n\nplt.plot(\n    train_accs, \n    label='Train_Accuracy', \n    color='green', \n    linewidth=1.5\n)\nplt.plot(\n    val_accs, \n    label='Val_Accuracy', \n    color='pink', \n    linewidth=1.5\n)\n\nplt.xlabel('Epoch')\nplt.ylabel('Loss / Accuracy')\nplt.title('Loss / Accuracy on train / validation')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv('/kaggle/input/cassava-leaf-disease-classification/sample_submission.csv')\nsub_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\n\ntest_dataset = LeafDataset(df=sub_df, data_path=BASE_PATH, mode='test', transforms=transforms_val)\n\ndef predict(model, test_dataset):\n    \n    test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=BATCH_SIZE)\n    \n    for test_images, test_labels in tqdm(test_dataloader):\n        test_images = test_images.to(device)\n        test_labels = test_labels.to(device)\n\n        output = model(test_images)\n\n        _, predicted = torch.max(output.logits.data, 1)\n        preds.extend(predicted.cpu().data.numpy())\n        \n    print(preds)\n        \npredict(model, test_dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df['label'] = preds\nsub_df.to_csv('submission.csv', index=False)\nsub_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. ResNet Pretrained Model","metadata":{}},{"cell_type":"code","source":"# model = models.resnet34(weights='DEFAULT')\n# in_features = int(model.fc.in_features)\n# model.fc = nn.Linear(in_features, 5, device)\n# model = model.to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# criterion = nn.CrossEntropyLoss()\n# optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)\n# lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n# val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=BATCH_SIZE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# images, targets = next(iter(train_dataloader))\n# grid_images = torchvision.utils.make_grid(images, nrow=8, padding=10)\n\n# def imshow(images, mean, std):\n#     np_image = np.array(images).transpose((1, 2, 0))\n#     unnorm_image = np_image * std + mean\n#     plt.figure(figsize=(20, 10))\n#     plt.imshow(unnorm_image)\n    \n# imshow(grid_images, mean, std)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# num_epochs = 2 \n# train_losses = [] \n# val_losses = []\n# train_accs = []\n# val_accs = []\n\n# for epoch in range(num_epochs):\n    \n#     total_acc_train = 0\n#     total_loss_train = 0\n#     model.train()\n    \n#     for i, (inputs, targets) in enumerate(train_dataloader):\n        \n#         inputs = inputs.to(device)\n#         targets = targets.to(device)\n        \n#         outputs = model(inputs)\n        \n#         batch_loss = criterion(outputs, targets)\n#         total_loss_train += batch_loss\n        \n#         _, preds = torch.max(outputs, 1)\n#         total_acc_train += (preds == targets.data).sum()\n        \n#         optimizer.zero_grad()\n#         batch_loss.backward()\n        \n#         optimizer.step()\n        \n#         if (i + 1) % 20 == 0:\n#             print('Epoch [%2d/%2d], Step [%3d/%3d], Batch_Loss: %.4f' % (epoch + 1, num_epochs, i + 1, len(train_dataset) // BATCH_SIZE, batch_loss.item()))\n    \n#     lr_scheduler.step()\n#     model.eval()\n#     total_acc_val = 0\n#     total_loss_val = 0\n    \n#     with torch.no_grad():\n#         for i, (inputs, targets) in enumerate(val_dataloader):\n#             val_loss = []\n                    \n#             inputs = inputs.to(device)\n#             targets = targets.to(device)\n            \n#             outputs = model(inputs)\n            \n#             batch_loss = criterion(outputs, targets)\n#             total_loss_val += batch_loss\n            \n#             _, preds = torch.max(outputs, 1)\n#             total_acc_val += (preds == targets.data).sum()\n        \n#     print(f'Epoch {epoch + 1}  :  Epoch_train_accuracy: {total_acc_train / len(train_dataset)}%', f' Epoch_val_accuracy: {total_acc_val / len(val_dataset)}')\n    \n#     train_losses.append(total_loss_train / len(train_dataset))\n#     train_accs.append(total_acc_train / len(train_dataset))\n#     val_losses.append(total_loss_val / len(val_dataset))\n#     val_accs.append(total_acc_val / len(val_dataset))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_losses_list = []\n# for value in train_losses:\n#     num = value.detach().cpu().numpy()\n#     train_losses_list.append(num)\n\n# val_losses_list = []\n# for value in val_losses:\n#     num = value.detach().cpu().numpy()\n#     val_losses_list.append(num)\n    \n# train_accs_list = []\n# for value in train_accs:\n#     num = value.detach().cpu().numpy()\n#     train_accs_list.append(num)\n\n# val_accs_list = []\n# for value in val_accs:\n#     num = value.detach().cpu().numpy()\n#     val_accs_list.append(num)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(18, 6))\n\n# plt.plot(\n#     train_losses_list, \n#     label='Train_Losses', \n#     color='red', \n#     linewidth=1.5\n# )\n# plt.plot(\n#     val_losses_list, \n#     label='Val_Losses', \n#     color='blue', \n#     linewidth=1.5\n# )\n\n# plt.plot(\n#     train_accs_list, \n#     label='Train_Accuracy', \n#     color='green', \n#     linewidth=1.5\n# )\n# plt.plot(\n#     val_accs_list, \n#     label='Val_Accuracy', \n#     color='pink', \n#     linewidth=1.5\n# )\n\n# plt.xlabel('Epoch')\n# plt.ylabel('Loss / Accuracy')\n# plt.title('Loss / Accuracy on train / validation')\n# plt.legend()\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}