{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master')\nsys.path.append('../input/timm-h/pytorch-image-models-master_h')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport os\nimport cv2\nimport glob\nimport random\nimport itertools\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, StepLR\nimport albumentations as albu\nfrom albumentations.pytorch import ToTensorV2\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.callbacks import EarlyStopping\nfrom pytorch_lightning import metrics\n\nimport timm\nfrom timm import create_model\n\npd.set_option('display.max_rows', None)\npd.set_option('display.max_columns', None)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"os.listdir('/kaggle/input/cassava-leaf-disease-classification')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = '/kaggle/input/cassava-leaf-disease-classification'\ntrain = pd.read_csv(os.path.join(data_dir, 'train.csv'))\nsub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Seed Setting"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Label Distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(x='label', data=train)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['label'].value_counts().sort_index()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, data_dir, transform=None, phase='train', df=None):\n        self.df = df\n        self.data_dir = data_dir\n        self.transform = transform\n        self.phase = phase\n        if self.phase == 'test':\n            img_dir = 'test_images'\n        else:\n            img_dir = 'train_images'\n        self.img_path = glob.glob(os.path.join(self.data_dir, img_dir, '*.jpg'))\n\n    def __len__(self):\n        if self.df is None:\n            return len(self.img_path)\n        else:\n            return len(self.df)\n\n    def __getitem__(self, idx):\n        \n        if self.phase == 'test':\n            target_img_path = self.img_path[idx]\n        else:\n            row = self.df.iloc[idx]\n            target_img_id = row['image_id']\n            target_img_path = os.path.join(self.data_dir, 'train_images', f'{target_img_id}')\n            \n        img = cv2.imread(target_img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img_id = os.path.basename(target_img_path)\n\n        if self.transform is not None:\n            img = self.transform(img, self.phase)\n        else:\n            img = torch.from_numpy(img.transpose((2, 0, 1)))\n            img = img / 255.\n\n        if self.phase == 'test':\n            return img, img_id\n        \n        else:\n            label = self.df[self.df['image_id'] == img_id]['label'].values\n            label = torch.tensor(label, dtype=torch.long)\n\n            return img, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## ImageTransform"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageTransform:\n    def __init__(self, img_size=224, mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)):\n        self.transform = {\n            'train': albu.Compose([\n                albu.RandomShadow(p=0.5),\n                albu.RandomResizedCrop(img_size, img_size, interpolation=cv2.INTER_AREA),\n                albu.ColorJitter(p=0.5),\n                albu.CLAHE(p=0.5),\n                albu.HorizontalFlip(p=0.5),\n                albu.VerticalFlip(p=0.5),\n                albu.Transpose(p=0.5),\n                albu.ShiftScaleRotate(p=0.5),\n                albu.OneOf([\n                    albu.Blur(p=1.0),\n                albu.GaussianBlur(p=1.0)\n                ], p=0.5),\n                albu.CoarseDropout(max_height=15, max_width=15, min_holes=3, p=0.5),\n                albu.Normalize(mean, std),\n                ToTensorV2(),\n            ], p=1.0),\n\n            'val': albu.Compose([\n                albu.Resize(img_size, img_size),\n                albu.Normalize(mean, std),\n                ToTensorV2(),\n            ], p=1.0),\n\n            'test': albu.Compose([\n                albu.Resize(img_size, img_size),\n                albu.HorizontalFlip(p=0.5),\n                albu.VerticalFlip(p=0.5),\n                albu.Normalize(mean, std),\n                ToTensorV2(),\n            ], p=1.0)\n        }\n\n    def __call__(self, img, phase='train'):\n        augmented = self.transform[phase](image=img)\n        augmented = augmented['image']\n\n        return augmented","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sanity Check\ntransform = ImageTransform()\ndataset = CassavaDataset(data_dir, transform, phase='train', df=train)\nimg, label = dataset.__getitem__(0)\nprint(img.size(), label)\nprint(img.max())\nprint(img.min())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataloader = DataLoader(dataset, batch_size=8)\n\nimgs, labels = next(iter(dataloader))\nprint(imgs.size())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sanity Check\ntransform = ImageTransform()\ndataset = CassavaDataset(data_dir, transform, phase='test', df=None)\nimg, label = dataset.__getitem__(0)\nprint(img.size(), label)\nprint(img.max())\nprint(img.min())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataloader = DataLoader(dataset, batch_size=1)\n\nimgs, labels = next(iter(dataloader))\nprint(imgs.size())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lightning DataModule"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataModule(pl.LightningDataModule):\n    def __init__(self, data_dir, cfg, transform, cv, fold):\n        super(CassavaDataModule, self).__init__()\n        self.data_dir = data_dir\n        self.cfg = cfg\n        self.transform = transform\n        self.cv = cv\n        self.fold = fold\n\n\n    def prepare_data(self):\n        # Prepare Data\n        self.df = pd.read_csv(os.path.join(self.data_dir, 'train.csv'))\n\n\n    def setup(self, stage=None):\n        # Validation\n        self.df['fold'] = -1\n        for i, (trn_idx, val_idx) in enumerate(self.cv.split(self.df, self.df['label'])):\n            self.df.loc[val_idx, 'fold'] = i\n        train = self.df[self.df['fold'] != self.fold].reset_index(drop=True)\n        val = self.df[self.df['fold'] == self.fold].reset_index(drop=True)\n        \n        # Dataset\n        self.train_dataset = CassavaDataset(self.data_dir, self.transform, phase='train', df=train)\n        self.val_dataset = CassavaDataset(self.data_dir, self.transform, phase='val', df=val)\n        self.test_dataset = CassavaDataset(self.data_dir, self.transform, phase='test', df=None)\n\n    def train_dataloader(self):\n        return DataLoader(self.train_dataset,\n                          batch_size=self.cfg.train['batch_size'],\n                          pin_memory=True,\n                          num_workers=4,\n                          shuffle=True)\n\n    def val_dataloader(self):\n        return DataLoader(self.val_dataset,\n                          batch_size=self.cfg.train['batch_size'],\n                          pin_memory=True,\n                          num_workers=4,\n                          shuffle=False)\n\n    def test_dataloader(self):\n        batch_size = min(len(self.test_dataset), self.cfg.train['batch_size'])\n        return DataLoader(self.test_dataset,\n                          batch_size=batch_size,\n                          pin_memory=True,\n                          num_workers=4,\n                          shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Timm_model(nn.Module):\n    def __init__(self, model_name='efficientnet_b0', pretrained=True, out_dim=5):\n        super(Timm_model, self).__init__()\n        self.base = create_model(model_name, pretrained=pretrained)\n\n        if 'efficientnet' in model_name:\n            self.base.classifier = nn.Linear(in_features=self.base.classifier.in_features, out_features=out_dim)\n        elif 'vit' in model_name:\n            self.base.head = nn.Linear(in_features=self.base.head.in_features, out_features=out_dim)\n        else:\n            self.base.fc = nn.Linear(in_features=self.base.fc.in_features, out_features=out_dim)\n\n    def forward(self, x):\n        return self.base(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sanity Check\nz = torch.randn(4, 3, 224, 224)\nmodel = Timm_model(pretrained=False)\nout = model(z)\nprint(out.size())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lightning Module"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaLightningSystem(pl.LightningModule):\n    def __init__(self, net, cfg, experiment=None):\n        super(CassavaLightningSystem, self).__init__()\n        self.net = net\n        self.cfg = cfg\n        self.experiment = experiment\n        self.criterion = nn.CrossEntropyLoss()\n        self.best_loss = 1e+9\n        self.best_acc = None\n        self.epoch_num = 0\n        self.acc_fn = metrics.Accuracy()\n        self.loss_list = []\n        self.acc_list = []\n\n    def configure_optimizers(self):\n        self.optimizer = optim.AdamW(self.parameters(), lr=self.cfg.train['lr'], weight_decay=1e-5)\n        self.scheduler = StepLR(self.optimizer, step_size=2, gamma=0.5)\n#         self.scheduler = CosineAnnealingLR(self.optimizer, T_max=self.cfg.train['epoch'], eta_min=0)\n\n        return [self.optimizer], [self.scheduler]\n\n    def forward(self, x):\n        return self.net(x)\n\n    def step(self, batch):\n        inp, label = batch\n        out = self.forward(inp)\n        loss = self.criterion(out, label.squeeze())\n\n        return loss, label, torch.sigmoid(out)\n\n    def training_step(self, batch, batch_idx):\n        loss, label, logits = self.step(batch)\n\n        if self.experiment is not None:\n            logs = {'train/loss': loss.item()}\n            self.experiment.log_metrics(logs, step=batch_idx)\n\n        return {'loss': loss, 'logits': logits, 'labels': label}\n\n    def validation_step(self, batch, batch_idx):\n        loss, label, logits = self.step(batch)\n\n        if self.experiment is not None:\n            val_logs = {'val/loss': loss.item()}\n            self.experiment.log_metrics(val_logs, step=batch_idx)\n\n        return {'val_loss': loss, 'logits': logits, 'labels': label}\n\n    def validation_epoch_end(self, outputs):\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        logits = torch.cat([x['logits'] for x in outputs])\n        labels = torch.cat([x['labels'] for x in outputs])\n        \n        # Accuracy\n        acc = self.acc_fn(logits, labels.squeeze())\n                \n        print(f'Epoch: {self.epoch_num}  Loss: {avg_loss.item():.4f}  Acc {acc.item():.4f}')\n        self.loss_list.append(avg_loss.item())\n        self.acc_list.append(acc.item())\n        \n        if self.experiment is not None:\n            logs = {'val/epoch_loss': avg_loss, 'val/epoch_acc': acc}\n            # Logging\n            self.experiment.log_metrics(logs, step=self.epoch_num)\n\n        # Save Weights\n        if self.best_loss > avg_loss:\n            self.best_loss = avg_loss.item()\n            self.best_acc = acc.item()\n            expname = self.cfg.exp['exp_name']\n            filename = f'{expname}_epoch_{self.epoch_num}_loss_{self.best_loss:.3f}_acc_{self.best_acc:.3f}.pth'\n            torch.save(self.net.state_dict(), filename)\n            if self.experiment is not None:\n                self.experiment.log_model(name=filename, file_or_folder='./'+filename)\n                os.remove(filename)\n            \n        # Update Epoch Num\n        self.epoch_num += 1\n\n        return {'avg_val_loss': avg_loss}\n\n    def test_step(self, batch, batch_idx):\n        inp, img_id = batch\n        out = self.forward(inp)\n        logits = torch.sigmoid(out)\n\n        return {'preds': logits, 'image_id': img_id}\n\n    def test_epoch_end(self, outputs):\n        preds = torch.cat([x['preds'] for x in outputs])\n        preds = preds.detach().cpu().numpy()\n        preds = pd.DataFrame(preds, columns=[f'label_{c}' for c in range(5)])\n        # [tuple, tuple]\n        img_ids = [x['image_id'] for x in outputs]\n        # [list, list]\n        img_ids = [list(x) for x in img_ids]\n        img_ids = list(itertools.chain.from_iterable(img_ids))\n        self.sub = preds\n        self.sub.insert(0, 'image_id', img_ids)\n\n        return None\n    \n    \n    # learning rate warm-up\n    def optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx,\n                       optimizer_closure, on_tpu, using_native_amp, using_lbfgs):\n        # warm up lr\n        if self.trainer.global_step < 500:\n            lr_scale = min(1., float(self.trainer.global_step + 1) / float(500))\n            for pg in optimizer.param_groups:\n                pg['lr'] = lr_scale * self.cfg.train['lr']\n\n        # update params\n        optimizer.step(closure=optimizer_closure)\n        optimizer.zero_grad()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"class cfg:\n    exp = {\n        'exp_name': 'test'\n    }\n    \n    data = {\n        'img_size': 256,\n        'n_splits': 5\n    }\n    \n    train = {\n        'batch_size': 64,\n        'epoch': 10,\n        'seed': 42,\n        'lr': 0.005,\n        'model_name': 'efficientnet_b0'\n    }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Trainer"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data Dir  #################################################################\ndata_dir = '/kaggle/input/cassava-leaf-disease-classification'\nseed_everything(cfg.train['seed'])\n\n# Validation  ###############################################################\ncv = StratifiedKFold(n_splits=cfg.data['n_splits'], shuffle=True, random_state=cfg.train['seed'])\n\n# Transform  ################################################################\ntransform = ImageTransform(img_size=cfg.data['img_size'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def main(data_dir, transform, cfg, cv, fold, TTA=5):\n    # Model  ####################################################################\n    net = Timm_model(model_name=cfg.train['model_name'], pretrained=False)\n    \n    # Lightning Module  #########################################################\n    dm = CassavaDataModule(data_dir, cfg, transform, cv, fold=fold)\n    model = CassavaLightningSystem(net, cfg, experiment=None)\n\n    trainer = Trainer(\n        logger=False,\n        max_epochs=cfg.train['epoch'],\n        gpus=1,\n        amp_backend='apex',\n        amp_level='O2',\n            )\n\n    # Train & Test  ############################################################\n    # Train\n    trainer.fit(model, datamodule=dm)\n\n    # Test\n    if TTA > 0:\n        for i in range(TTA):\n            trainer.test(model, datamodule=dm)\n            if i == 0:\n                res = model.sub\n            else:\n                for j in range(5):\n                    res[f'label_{j}'] += model.sub[f'label_{j}']\n\n    else:\n        trainer.test(model, datamodule=dm)\n        res = model.sub\n\n    res.to_csv(f'submission_fold{fold}.csv', index=False)\n    \n    del net, dm\n    torch.cuda.empty_cache()\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"TTA = 3\nmodels = []\n\nfor fold in range(cfg.data['n_splits']):\n    m = main(data_dir, transform, cfg, cv, fold, TTA)\n    models.append(m)\n    del m","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Summarize Predictions\nsub_paths = glob.glob('submission_fold*')\nfor i, path in enumerate(sub_paths):\n    tmp = pd.read_csv(path)\n    \n    if i == 0:\n        res = tmp\n    else:\n        for j in range(5):\n            res[f'label_{j}'] += tmp[f'label_{j}']\n            \nlabel_cols = [c for c in res.columns if c != 'image_id']\nres['label'] = np.argmax(res[label_cols].values, axis=1)\nres = res[['image_id', 'label']]\nres.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Learning Plot"},{"metadata":{"trusted":true},"cell_type":"code","source":"history = pd.DataFrame({\n    'loss': models[0].loss_list,\n    'acc': models[0].acc_list\n})\n\nfig, axes = plt.subplots(ncols=2, nrows=1, figsize=(16, 6))\n\nfor ax, label in zip(axes.ravel(), ['loss', 'acc']):\n    ax.plot(history[label])\n    ax.set_title(label)\n    ax.set_xlabel('Epoch')\n\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}