{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Hello Everyone !!\nIn this Notebook we will be using EfficientNet B3 for Leaf Disease Classification. Most parts of this notebook is adapted version of [Alex Shonenkov](https://www.kaggle.com/shonenkov)'s work.","metadata":{}},{"cell_type":"markdown","source":"# Import Dependecies","metadata":{}},{"cell_type":"code","source":"!pip install -q efficientnet_pytorch > /dev/null ","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:03:38.428748Z","iopub.execute_input":"2023-04-04T19:03:38.429092Z","iopub.status.idle":"2023-04-04T19:03:48.647801Z","shell.execute_reply.started":"2023-04-04T19:03:38.429057Z","shell.execute_reply":"2023-04-04T19:03:48.646798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport sys\nimport matplotlib.pyplot as plt\nimport random\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.optimizer import Optimizer\nimport cv2\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport time\nfrom datetime import datetime\nfrom tqdm.autonotebook import tqdm\nfrom efficientnet_pytorch import EfficientNet\n\nimport warnings\nwarnings.filterwarnings(\"ignore\") \nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning) ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-04T19:05:59.054708Z","iopub.execute_input":"2023-04-04T19:05:59.055042Z","iopub.status.idle":"2023-04-04T19:06:02.040354Z","shell.execute_reply.started":"2023-04-04T19:05:59.055012Z","shell.execute_reply":"2023-04-04T19:06:02.039296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(42)","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2023-04-04T19:06:02.042288Z","iopub.execute_input":"2023-04-04T19:06:02.042694Z","iopub.status.idle":"2023-04-04T19:06:02.052911Z","shell.execute_reply.started":"2023-04-04T19:06:02.042654Z","shell.execute_reply":"2023-04-04T19:06:02.052090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fold Splitting","metadata":{}},{"cell_type":"markdown","source":"I have made splitted the data in 5 folds. Each fold has same number of 5 different classes. The csv can be found [here](https://www.kaggle.com/zaber666/cld-dataset).","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/cld-dataset/train_5fold.csv')","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:04.126602Z","iopub.execute_input":"2023-04-04T19:06:04.126946Z","iopub.status.idle":"2023-04-04T19:06:04.158542Z","shell.execute_reply.started":"2023-04-04T19:06:04.126915Z","shell.execute_reply":"2023-04-04T19:06:04.157595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#samples per fold\ndf.fold.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:04.462046Z","iopub.execute_input":"2023-04-04T19:06:04.462406Z","iopub.status.idle":"2023-04-04T19:06:04.476408Z","shell.execute_reply.started":"2023-04-04T19:06:04.462374Z","shell.execute_reply":"2023-04-04T19:06:04.475389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#different labels per fold\ndf_fold1 = df[df.fold == 1]\ndf_fold1.label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:04.928142Z","iopub.execute_input":"2023-04-04T19:06:04.928580Z","iopub.status.idle":"2023-04-04T19:06:04.953079Z","shell.execute_reply.started":"2023-04-04T19:06:04.928543Z","shell.execute_reply":"2023-04-04T19:06:04.952443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentation","metadata":{}},{"cell_type":"code","source":"def train_transform():\n    return A.Compose(\n        [\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.Rotate(),\n            A.Resize(height=256, width=256, p=1),\n            A.Cutout(num_holes=8, max_h_size=32, max_w_size=32, fill_value=0, p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.0\n    )\n\ndef valid_transform():\n    return A.Compose(\n        [\n            A.Resize(height=256, width=256, p=1),\n            ToTensorV2(p=1.0),\n        ], p=1.0\n    )","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:09.227811Z","iopub.execute_input":"2023-04-04T19:06:09.228149Z","iopub.status.idle":"2023-04-04T19:06:09.235446Z","shell.execute_reply.started":"2023-04-04T19:06:09.228116Z","shell.execute_reply":"2023-04-04T19:06:09.234544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset & Dataloader","metadata":{}},{"cell_type":"markdown","source":"I have used resized 256x256 images from [here](https://www.kaggle.com/konradb/resized-data-256).","metadata":{}},{"cell_type":"code","source":"DATA_DIR = '/kaggle/input/cassava-leaf-disease-classification/train_images'","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:10.994878Z","iopub.execute_input":"2023-04-04T19:06:10.995235Z","iopub.status.idle":"2023-04-04T19:06:10.999560Z","shell.execute_reply.started":"2023-04-04T19:06:10.995179Z","shell.execute_reply":"2023-04-04T19:06:10.998477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CLD_Dataset(Dataset):\n    def __init__(self, image_ids, labels, transform=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.labels = labels\n        self.transform = transform\n\n    def __getitem__(self, index):\n        image_id = self.image_ids[index]\n        label = self.labels[index]\n\n        image = cv2.imread(f'{DATA_DIR}/{image_id}',  cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image = image / 255.0\n\n        if self.transform is not None:\n            image = self.transform(**{'image':image})['image']\n\n        ohe_label = torch.zeros(5, dtype=torch.float32)\n        ohe_label[label] = 1\n\n        return image, ohe_label\n    \n    def __len__(self):\n        return len(self.image_ids)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:11.538009Z","iopub.execute_input":"2023-04-04T19:06:11.538395Z","iopub.status.idle":"2023-04-04T19:06:11.547677Z","shell.execute_reply.started":"2023-04-04T19:06:11.538360Z","shell.execute_reply":"2023-04-04T19:06:11.546696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLD = 0\nDEBUG = False\n\nif not DEBUG:\n    dataset_train = CLD_Dataset(\n        image_ids = df[df.fold != FOLD].image_id.values,\n        labels = df[df.fold != FOLD].label.values,\n        transform = train_transform()\n    )\n    dataset_val = CLD_Dataset(\n        image_ids = df[df.fold == FOLD].image_id.values,\n        labels = df[df.fold == FOLD].label.values,\n        transform = valid_transform()\n    )\nelse:\n    dataset_train = CLD_Dataset(\n        image_ids = df[df.fold != FOLD].image_id.values[:128],\n        labels = df[df.fold != FOLD].label.values[:128],\n        transform = train_transform()\n    )\n    dataset_val = CLD_Dataset(\n        image_ids = df[df.fold == FOLD].image_id.values[:128],\n        labels = df[df.fold == FOLD].label.values[:128],\n        transform = valid_transform()\n    ) ","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:12.255890Z","iopub.execute_input":"2023-04-04T19:06:12.256246Z","iopub.status.idle":"2023-04-04T19:06:12.271695Z","shell.execute_reply.started":"2023-04-04T19:06:12.256187Z","shell.execute_reply":"2023-04-04T19:06:12.270701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure, axes = plt.subplots(nrows=3, ncols=2, figsize=(10,10))\nfor i in range(6):\n    image, _ = dataset_train[i]\n    image = image.permute(1,2,0).cpu().numpy()\n    axes[i//2, i%2].imshow(image)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:13.973351Z","iopub.execute_input":"2023-04-04T19:06:13.973694Z","iopub.status.idle":"2023-04-04T19:06:15.020631Z","shell.execute_reply.started":"2023-04-04T19:06:13.973656Z","shell.execute_reply":"2023-04-04T19:06:15.017013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 32","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:15.029169Z","iopub.execute_input":"2023-04-04T19:06:15.029978Z","iopub.status.idle":"2023-04-04T19:06:15.042864Z","shell.execute_reply.started":"2023-04-04T19:06:15.029891Z","shell.execute_reply":"2023-04-04T19:06:15.042092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(\n    dataset_train,\n    batch_size=BATCH_SIZE,\n    drop_last=True,\n    num_workers=2,\n    shuffle=True\n)\nval_loader = DataLoader(\n    dataset_val,\n    batch_size=BATCH_SIZE,\n    drop_last=False,\n    num_workers=2\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:15.085135Z","iopub.execute_input":"2023-04-04T19:06:15.085580Z","iopub.status.idle":"2023-04-04T19:06:15.098881Z","shell.execute_reply.started":"2023-04-04T19:06:15.085540Z","shell.execute_reply":"2023-04-04T19:06:15.096906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<b>We will mixup in this notebook. Let's see some exmaples of mixup</b>","metadata":{}},{"cell_type":"code","source":"def mixup(data, targets, alpha):\n    \n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_targets = targets[indices]\n\n    lm = np.random.beta(alpha, alpha)\n    data = lm*data + (1-lm)*shuffled_data\n    \n    targets = (targets, shuffled_targets, lm)\n\n    return data, targets","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:17.777402Z","iopub.execute_input":"2023-04-04T19:06:17.777749Z","iopub.status.idle":"2023-04-04T19:06:17.784291Z","shell.execute_reply.started":"2023-04-04T19:06:17.777716Z","shell.execute_reply":"2023-04-04T19:06:17.783161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, targets = next(iter(train_loader))\nimages, _ = mixup(images, targets, 0.5) ","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:20.683272Z","iopub.execute_input":"2023-04-04T19:06:20.683659Z","iopub.status.idle":"2023-04-04T19:06:22.965247Z","shell.execute_reply.started":"2023-04-04T19:06:20.683623Z","shell.execute_reply":"2023-04-04T19:06:22.963996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure, axes = plt.subplots(nrows=3, ncols=2, figsize=(10,10))\nfor i in range(6):\n    image = images[i]\n    image = image.permute(1,2,0).cpu().numpy()\n    axes[i//2, i%2].imshow(image)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:22.967602Z","iopub.execute_input":"2023-04-04T19:06:22.968065Z","iopub.status.idle":"2023-04-04T19:06:23.865951Z","shell.execute_reply.started":"2023-04-04T19:06:22.968021Z","shell.execute_reply":"2023-04-04T19:06:23.864416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def make_model():\n    model = EfficientNet.from_pretrained('efficientnet-b4') \n    model._fc = nn.Linear(in_features=1792, out_features=5, bias=True)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:24.592234Z","iopub.execute_input":"2023-04-04T19:06:24.592588Z","iopub.status.idle":"2023-04-04T19:06:24.597877Z","shell.execute_reply.started":"2023-04-04T19:06:24.592557Z","shell.execute_reply":"2023-04-04T19:06:24.596821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = make_model()","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:25.020318Z","iopub.execute_input":"2023-04-04T19:06:25.020690Z","iopub.status.idle":"2023-04-04T19:06:26.001529Z","shell.execute_reply.started":"2023-04-04T19:06:25.020656Z","shell.execute_reply":"2023-04-04T19:06:26.000476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"class LabelSmoothing(nn.Module):\n    def __init__(self, smoothing = 0.01):\n        super(LabelSmoothing, self).__init__()\n        self.confidence = 1.0 - smoothing\n        self.smoothing = smoothing\n\n    def forward(self, x, target):\n        if self.training:\n            x = x.float()\n            target = target.float()\n            logprobs = torch.nn.functional.log_softmax(x, dim = -1)\n\n            nll_loss = -logprobs * target\n            nll_loss = nll_loss.sum(-1)\n    \n            smooth_loss = -logprobs.mean(dim=-1)\n\n            loss = self.confidence * nll_loss + self.smoothing * smooth_loss\n\n            return loss.mean()\n        else:\n            return torch.nn.functional.cross_entropy(x, target)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:28.963138Z","iopub.execute_input":"2023-04-04T19:06:28.963531Z","iopub.status.idle":"2023-04-04T19:06:28.971696Z","shell.execute_reply.started":"2023-04-04T19:06:28.963496Z","shell.execute_reply":"2023-04-04T19:06:28.970555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AverageMeter(object):\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:29.595530Z","iopub.execute_input":"2023-04-04T19:06:29.595913Z","iopub.status.idle":"2023-04-04T19:06:29.603409Z","shell.execute_reply.started":"2023-04-04T19:06:29.595876Z","shell.execute_reply":"2023-04-04T19:06:29.602458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Fitter:\n    def __init__(self, model, device, work_dir, epochs):\n        self.epoch = 0\n        self.model = model\n        self.work_dir = work_dir\n        if not os.path.exists(self.work_dir):\n            os.makedirs(self.work_dir)\n        self.log_path = f'{self.work_dir}/log.txt'\n        self.best_score = 0\n        self.device = device\n        self.model.to(device)\n\n        param_optimizer = list(self.model.named_parameters())\n        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n        optimizer_grouped_parameters = [\n            {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n            {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n        ]\n\n        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0005)\n        self.scheduler = torch.optim.lr_scheduler.OneCycleLR(\n                                    self.optimizer, \n                                    max_lr=0.001, \n                                    epochs=30,  \n                                    steps_per_epoch=int(len(dataset_train) / BATCH_SIZE), \n                                    pct_start=0.1, \n                                    anneal_strategy='cos', \n                                    final_div_factor=10**5)\n        \n        self.criterion = LabelSmoothing().to(self.device)\n\n        self.log(f'Fitter prepared. Device is {self.device}')\n\n    def log(self, message):\n        with open(self.log_path, 'a+') as f:\n            f.write(f'{message}\\n')\n        print(message)\n\n    def fit(self, train_loader, val_loader):\n        for epc in range(20):\n            lr = self.optimizer.param_groups[0]['lr']\n            timestamp = datetime.utcnow().isoformat()\n            self.log(f'\\n{timestamp}\\nLR: {lr}')\n\n            t = time.time()\n            loss, accuracy = self.train_one_epoch(train_loader)\n            self.log(f'[RESULT]: Train. Epoch: {self.epoch}, loss: {loss.avg:.5f}, accuracy: {accuracy.avg:.5f}, time: {(time.time() - t):.5f}')\n\n            t = time.time()\n            loss, accuracy = self.validation(val_loader)\n            self.log(f'[RESULT]: Val. Epoch: {self.epoch}, loss: {loss.avg:.5f}, accuracy: {accuracy.avg:.5f}, time: {(time.time() - t):.5f}')\n\n            if accuracy.avg > self.best_score:\n                self.best_score = accuracy.avg\n                self.save_model(f'{self.work_dir}/best-checkpoint.bin')\n            \n            self.save_model(f'{self.work_dir}/last-checkpoint.bin')\n            self.epoch += 1\n\n    def train_one_epoch(self, train_loader):\n        self.model.train()\n        summary_loss = AverageMeter()\n        accuracy = AverageMeter()\n\n        t = time.time()\n        tk = tqdm(train_loader, total=len(train_loader), desc='Training')\n        for step, (images, targets) in enumerate(tk):\n\n            targets = targets.to(self.device).float()\n            images = images.to(self.device).float()\n            batch_size = images.shape[0]\n\n            if np.random.random() < 0.5 :\n                # mixup\n                \n                mixup_images, mixup_targets = mixup(images, targets, 0.4)\n                targets, shuffled_targets, lm = mixup_targets\n                self.optimizer.zero_grad()\n                outputs = self.model(mixup_images)\n                \n                loss = lm*self.criterion(outputs, targets) + (1-lm)*self.criterion(outputs, shuffled_targets)\n            else:\n                self.optimizer.zero_grad()\n                outputs = self.model(images)\n                loss = self.criterion(outputs, targets)\n\n            loss.backward()\n            outputs = nn.functional.softmax(outputs, dim=1)\n            acc = (outputs.argmax(1)==targets.argmax(1)).sum().item() / batch_size\n\n            summary_loss.update(loss.detach().item(), batch_size)\n            accuracy.update(acc, batch_size)\n            self.optimizer.step()\n\n\n        self.scheduler.step()\n        return summary_loss, accuracy\n\n    def validation(self, val_loader):\n        self.model.eval()\n        summary_loss = AverageMeter()\n        accuracy = AverageMeter()\n\n        t = time.time()\n        tk = tqdm(val_loader, total=len(val_loader), desc='Validating')\n        for step, (images, targets) in enumerate(tk):\n                \n            with torch.no_grad():\n                targets = targets.to(self.device).float()\n                images = images.to(self.device).float()\n                batch_size = images.shape[0]\n                outputs = self.model(images)\n                loss = self.criterion(outputs, targets)\n                outputs = nn.functional.softmax(outputs, dim=1)\n                acc = (outputs.argmax(dim=1)==targets.argmax(dim=1)).sum().item() / batch_size\n                summary_loss.update(loss.detach().item(), batch_size)\n                accuracy.update(acc, batch_size)\n\n        return summary_loss, accuracy\n    \n    def save_model(self, path):\n        self.model.eval()\n        torch.save({\n            'model_state_dict':self.model.state_dict(),\n            'optimizer_state_dict':self.optimizer.state_dict(),\n            'scheduler_state_dict':self.scheduler.state_dict(),\n            'best_score':self.best_score,\n            'epoch':self.epoch\n        }, path)\n\n    def load_model(self, path):\n        ckpt = torch.load(path, map_location=self.device)\n        self.model.load_state_dict(ckpt['model_state_dict'])\n        self.optimizer.load_state_dict(ckpt['optimizer_state_dict'])\n        self.scheduler.load_state_dict(ckpt['scheduler_state_duct'])\n        self.best_score = ckpt['best_score']\n        self.epoch = ckpt['epoch']","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:30.075648Z","iopub.execute_input":"2023-04-04T19:06:30.075997Z","iopub.status.idle":"2023-04-04T19:06:30.112398Z","shell.execute_reply.started":"2023-04-04T19:06:30.075964Z","shell.execute_reply":"2023-04-04T19:06:30.110802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fitter = Fitter(model=model, device='cuda', work_dir='/kaggle/working/output', epochs=10)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:30.786296Z","iopub.execute_input":"2023-04-04T19:06:30.786670Z","iopub.status.idle":"2023-04-04T19:06:34.987790Z","shell.execute_reply.started":"2023-04-04T19:06:30.786632Z","shell.execute_reply":"2023-04-04T19:06:34.986945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fitter.fit(train_loader, val_loader) ","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:06:37.398172Z","iopub.execute_input":"2023-04-04T19:06:37.398544Z","iopub.status.idle":"2023-04-04T19:38:01.249562Z","shell.execute_reply.started":"2023-04-04T19:06:37.398511Z","shell.execute_reply":"2023-04-04T19:38:01.247072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Have fun changing different parts and experimenting.\n\nHappy Coding.","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}