{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Cassava Leaf Disease Classification Baseline -- Modified From Previous Competition\n\nHi everyone,\n\nI'm here to show you how to **REUSE** previous competition code to build a baseline method for new a competition.\n\nThe previous competition I mean PANDA https://www.kaggle.com/c/prostate-cancer-grade-assessment here.\n\nWhich is also an image classification competition like this one. So the code structure of the two competitions will have a lot in common.\n\nYou will find that how little code need to be changed from an image classification competition to another image classification competition in this notebook!\n\n(check out the diff between this 2 notebooks from `versions` button on the top-right of this page!)\n\nIf you find find any of the following idea helps, please upvote me, THANKS!\n\n\n# Summary of This Baseline\n\n* Based on baseline code of previous competition PANDA: https://www.kaggle.com/haqishen/train-efficientnet-b0-w-36-tiles-256-lb0-87\n* Just modified a little bit, such as Dataset part and Loss part\n\n\n# To Be Updated"},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install git+https://github.com/ildoonet/pytorch-gradual-warmup-lr.git","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path = [\n    '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',\n] + sys.path","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import time\nimport skimage.io\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport PIL.Image\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SubsetRandomSampler, RandomSampler, SequentialSampler\nfrom warmup_scheduler import GradualWarmupScheduler\nfrom efficientnet_pytorch import model as enet\nimport albumentations\nfrom sklearn.model_selection import StratifiedKFold\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import cohen_kappa_score\nfrom tqdm import tqdm_notebook as tqdm\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = '../input/cassava-leaf-disease-classification'\ndf_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))\nimage_folder = os.path.join(data_dir, 'train_images')\n\nkernel_type = 'baseline'\n\nenet_type = 'efficientnet-b0'\nfold = 0\nimage_size = 256\nbatch_size = 64\nnum_workers = 2\nout_dim = 5\ninit_lr = 1e-4\nwarmup_factor = 10\nwarmup_epo = 1\nn_epochs = 1 if DEBUG else 10\ndf_train = df_train.sample(100).reset_index(drop=True) if DEBUG else df_train\n\ndevice = torch.device('cuda')\n\nprint(image_folder)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Folds"},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(5, shuffle=True, random_state=42)\ndf_train['fold'] = -1\nfor i, (train_idx, valid_idx) in enumerate(skf.split(df_train, df_train['label'])):\n    df_train.loc[valid_idx, 'fold'] = i\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"pretrained_model = {\n    'efficientnet-b0': '../input/efficientnet-pytorch/efficientnet-b0-08094119.pth'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class enetv2(nn.Module):\n    def __init__(self, backbone, out_dim):\n        super(enetv2, self).__init__()\n        self.enet = enet.EfficientNet.from_name(backbone)\n        self.enet.load_state_dict(torch.load(pretrained_model[backbone]))\n\n        self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)\n        self.enet._fc = nn.Identity()\n\n    def extract(self, x):\n        return self.enet(x)\n\n    def forward(self, x):\n        x = self.extract(x)\n        x = self.myfc(x)\n        return x\n    \nm = enetv2(enet_type, out_dim=out_dim)\nm(torch.rand(2,3,256,256)).shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass LEAFDataset(Dataset):\n    def __init__(self, df, transforms=None):\n\n        self.df = df.reset_index(drop=True)\n        self.transforms = transforms\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        img_id = row.image_id\n        \n        image_file = os.path.join(image_folder, img_id)\n        image = cv2.imread(image_file)\n        image = image[:, :, ::-1]\n\n        if self.transforms is not None:\n            image = self.transforms(image=image)['image']\n        image = image.astype(np.float32)\n        image /= 255\n        image = image.transpose(2, 0, 1)\n\n        return torch.tensor(image), torch.tensor(row.label)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_train = albumentations.Compose([\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.Resize(image_size, image_size),\n])\ntransforms_val = albumentations.Compose([\n    albumentations.Resize(image_size, image_size),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_show = LEAFDataset(df_train, transforms=transforms_train)\nfrom pylab import rcParams\nrcParams['figure.figsize'] = 20,10\nfor i in range(2):\n    f, axarr = plt.subplots(1,5)\n    for p in range(5):\n        idx = np.random.randint(0, len(dataset_show))\n        img, label = dataset_show[idx]\n        axarr[p].imshow(img.transpose(0, 1).transpose(1,2).squeeze())\n        axarr[p].set_title(str(label))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Scheduler"},{"metadata":{"trusted":true},"cell_type":"code","source":"class GradualWarmupSchedulerV2(GradualWarmupScheduler):\n    def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None):\n        super(GradualWarmupSchedulerV2, self).__init__(optimizer, multiplier, total_epoch, after_scheduler)\n    def get_lr(self):\n        if self.last_epoch > self.total_epoch:\n            if self.after_scheduler:\n                if not self.finished:\n                    self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs]\n                    self.finished = True\n                return self.after_scheduler.get_lr()\n            return [base_lr * self.multiplier for base_lr in self.base_lrs]\n        if self.multiplier == 1.0:\n            return [base_lr * (float(self.last_epoch) / self.total_epoch) for base_lr in self.base_lrs]\n        else:\n            return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in self.base_lrs]\n\noptimizer = optim.Adam(m.parameters(), lr=init_lr)\nscheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, n_epochs-warmup_epo)\nscheduler_warmup = GradualWarmupSchedulerV2(optimizer, multiplier=10, total_epoch=warmup_epo, after_scheduler=scheduler_cosine)\nlrs = []\nfor epoch in range(1, n_epochs+1):\n    scheduler_warmup.step(epoch-1)\n    lrs.append(optimizer.param_groups[0][\"lr\"])\nrcParams['figure.figsize'] = 20,3\nplt.plot(lrs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train & Val"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_epoch(loader, optimizer):\n\n    model.train()\n    train_loss = []\n    bar = tqdm(loader)\n    for (data, target) in bar:\n        \n        data, target = data.to(device), target.to(device)\n        optimizer.zero_grad()\n        logits = model(data)\n        loss = criterion(logits, target)\n        loss.backward()\n        optimizer.step()\n\n        loss_np = loss.detach().cpu().numpy()\n        train_loss.append(loss_np)\n        smooth_loss = sum(train_loss[-100:]) / min(len(train_loss), 100)\n        bar.set_description('loss: %.5f, smth: %.5f' % (loss_np, smooth_loss))\n    return train_loss\n\n\ndef val_epoch(loader, get_output=False):\n\n    model.eval()\n    val_loss = []\n    LOGITS = []\n    PREDS = []\n    TARGETS = []\n\n    with torch.no_grad():\n        for (data, target) in tqdm(loader):\n            data, target = data.to(device), target.to(device)\n            logits = model(data)\n\n            loss = criterion(logits, target)\n\n            pred = logits.softmax(1).argmax(1).detach()\n            LOGITS.append(logits)\n            PREDS.append(pred)\n            TARGETS.append(target)\n\n            val_loss.append(loss.detach().cpu().numpy())\n        val_loss = np.mean(val_loss)\n\n    LOGITS = torch.cat(LOGITS).cpu().numpy()\n    PREDS = torch.cat(PREDS).cpu().numpy()\n    TARGETS = torch.cat(TARGETS).cpu().numpy()\n    acc = (PREDS == TARGETS).mean() * 100.\n\n    if get_output:\n        return LOGITS\n    else:\n        return val_loss, acc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Dataloader & Model & Optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_idx = np.where((df_train['fold'] != fold))[0]\nvalid_idx = np.where((df_train['fold'] == fold))[0]\n\ndf_this  = df_train.loc[train_idx]\ndf_valid = df_train.loc[valid_idx]\n\ndataset_train = LEAFDataset(df_this , transforms=transforms_train)\ndataset_valid = LEAFDataset(df_valid, transforms=transforms_val)\n\ntrain_loader = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size, sampler=RandomSampler(dataset_train), num_workers=num_workers)\nvalid_loader = torch.utils.data.DataLoader(dataset_valid, batch_size=batch_size, sampler=SequentialSampler(dataset_valid), num_workers=num_workers)\n\nmodel = enetv2(enet_type, out_dim=out_dim)\nmodel = model.to(device)\n\noptimizer = optim.Adam(model.parameters(), lr=init_lr/warmup_factor)\nscheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, n_epochs-warmup_epo)\nscheduler = GradualWarmupScheduler(optimizer, multiplier=warmup_factor, total_epoch=warmup_epo, after_scheduler=scheduler_cosine)\n\nprint(len(dataset_train), len(dataset_valid))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Run Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"acc_max = 0.\nbest_file = f'{kernel_type}_best_fold{fold}.pth'\nfor epoch in range(1, n_epochs+1):\n    print(time.ctime(), 'Epoch:', epoch)\n    scheduler.step(epoch-1)\n\n    train_loss = train_epoch(train_loader, optimizer)\n    val_loss, acc = val_epoch(valid_loader)\n\n    content = time.ctime() + ' ' + f'Epoch {epoch}, lr: {optimizer.param_groups[0][\"lr\"]:.7f}, train loss: {np.mean(train_loss):.5f}, val loss: {np.mean(val_loss):.5f}, acc: {(acc):.5f}'\n    print(content)\n    with open(f'log_{kernel_type}.txt', 'a') as appender:\n        appender.write(content + '\\n')\n\n    if acc > acc_max:\n        print('acc ({:.6f} --> {:.6f}).  Saving model ...'.format(acc_max, acc))\n        torch.save(model.state_dict(), best_file)\n        acc_max = acc\n\ntorch.save(model.state_dict(), os.path.join(f'{kernel_type}_final_fold{fold}.pth'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Thank you for reading!"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}