{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Description\nThis notebook is the simple baseline using EfficientNet with\n* Early Stopping\n* Test Time Augmentations (TTA)\n* AMP\n\nI tried to write this code that is simple (so, cannot achieve high score...) but easy to read.\n\nAnd this notebook has room for some experiments (batchsize, learning rate, etc.).\nIf you want to do some small experiments, change the configurations below.\n\nI hope this notebook helps someone struggling with Pytorch. Thanks!"},{"metadata":{},"cell_type":"markdown","source":"# Setup Libraries"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nimport os, time, random\nfrom tqdm import tqdm\n\nimport cv2\n\nfrom sklearn.model_selection import train_test_split\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, CosineAnnealingWarmRestarts\nimport torch.utils.data as data\nfrom torch.utils.data import DataLoader\n\nimport torchvision\nfrom torchvision import transforms\n\nfrom torch.cuda.amp import autocast, GradScaler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install '../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\nfrom efficientnet_pytorch import EfficientNet\n# B0: 224\n# B1: 240\n# B2: 260\n# B3: 300\n# B4: 380\n# B5: 456\n# B6: 528\n# B7: 600","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Debug Signal\nDEBUG = False\n\n# Training or Inference\nINFERENCE = True\n\n# Helper Functions\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Randomness\nSEED = 87\nseed_everything(SEED)\n\n# Data Paths\nTRAIN = '../input/cassava-leaf-disease-classification/train.csv'\nTRAIN_IMGS = '../input/cassava-leaf-disease-classification/train_images'\nTEST = '../input/cassava-leaf-disease-classification/sample_submission.csv'\nTEST_IMGS = '../input/cassava-leaf-disease-classification/test_images'\n\n# Cofiguration\ncfg = {'model':{'arch': 'efficientnet-b0'},\n       'data':{'img_size': 224},\n       'optim':{'lr': 1e-3, \n                'T_0': 5,\n                'T_mult': 1, \n                'eta_min': 1e-6},\n       'train':{'batch_size': 64, \n                'shuffle': True,\n                'num_epochs': 5,\n                'patience': 2,\n                'use_checkpoint': False, \n                'checkpoint_path': '../input/cassava-checkpoints-effnet'},\n       'val':{'batch_size': 64,\n              'shuffle': False},\n       'test':{'batch_size': 64,\n               'shuffle': False,\n               'TTA': True,\n               'num_TTA': 5, \n               'checkpoint': '../input/cassava-checkpoints-effnet'}}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"if INFERENCE:\n    test_df = pd.DataFrame()\n    test_df['image_id'] = list(os.listdir(TEST_IMGS + '/'))\n    test_df.loc[:, 'img_path'] = TEST_IMGS + '/' + test_df.image_id\n    \n    if DEBUG:\n        test_df = pd.read_csv(TRAIN).head(128)\n        test_df.loc[:, 'img_path'] = TRAIN_IMGS + '/' + test_df.image_id\nelse:\n    train_df = pd.read_csv(TRAIN)\n    train_df.loc[:, 'img_path'] = TRAIN_IMGS + '/' + train_df.image_id\n    \n    if DEBUG:\n        train_df = train_df.head(200)\n        \n    idx_set = [idx for idx in range(len(train_df))]\n    train_idx, val_idx = train_test_split(idx_set, test_size=0.2)\n\n    val_df = train_df.iloc[val_idx].reset_index(drop=True)\n    train_df = train_df.iloc[train_idx].reset_index(drop=True)\n    \n    val_df['pred'] = np.nan","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if (DEBUG) & (not INFERENCE):\n    num_labels = len(train_df.loc[:, 'label'].unique())\n\n    print('train ds', train_df.loc[:, 'label'].value_counts(normalize=True).values)\n    print('val ds', val_df.loc[:, 'label'].value_counts(normalize=True).values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n    \nclass AlbumTransform():\n    \n    def __init__(self, mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), resize=cfg['data']['img_size']):\n        \n        self.data_transform = {\n            'train': Compose([\n                Transpose(p=0.5),\n                HorizontalFlip(p=0.5),\n                VerticalFlip(p=0.5),\n                ShiftScaleRotate(p=0.5),\n                HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n                RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n                Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),\n                CoarseDropout(p=0.5),\n                Cutout(p=0.5),\n                RandomResizedCrop(resize, resize),\n                ToTensorV2(p=1.0),\n                ], p=1.0), \n            'val': Compose([\n                CenterCrop(resize, resize),\n                Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),\n                ToTensorV2(p=1.0)\n                ], p=1.0),\n            'test': Compose([\n                CenterCrop(resize, resize),\n                Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),\n                ToTensorV2(p=1.0)\n                ], p=1.0),\n            'test_tta': Compose([\n                Transpose(p=0.5),\n                HorizontalFlip(p=0.5),\n                VerticalFlip(p=0.5),\n                ShiftScaleRotate(p=0.5),\n                HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n                RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n                Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),\n                CoarseDropout(p=0.5),\n                Cutout(p=0.5),\n                RandomResizedCrop(resize, resize),\n                ToTensorV2(p=1.0),\n                ], p=1.0)\n        }\n        \n    def __call__(self, img, phase='train'):\n        return self.data_transform[phase](image=img)['image']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEBUG:\n    img = cv2.imread(train_df.img_path[24])\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n    transform = AlbumTransform()\n    transform_train = transform(img=img, phase='train')\n    img_transformed = transform_train.numpy().transpose((1, 2, 0))\n\n    fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(15, 8))\n\n    ax[0].imshow(img)\n    ax[0].set_title('before', fontsize=16)\n    ax[1].imshow(img_transformed)\n    ax[1].set_title('after', fontsize=16)\n\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(data.Dataset):\n    \n    def __init__(self, df, transform=None, phase='train'):\n        self.transform = transform\n        self.phase= phase\n        self.df = df\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_path = self.df.iloc[index].loc['img_path']\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        img_transformed = self.transform(img, self.phase)\n        \n        if (self.phase=='train' or self.phase=='val'):\n            label = self.df.iloc[index].loc['label']\n            return img_transformed, label\n        else:\n            return img_transformed","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataloader"},{"metadata":{"trusted":true},"cell_type":"code","source":"transform = AlbumTransform()\n\nif INFERENCE:\n    if cfg['test']['TTA']:\n        test_ds = CassavaDataset(test_df, transform=transform, phase='test_tta')\n    else:\n        test_ds = CassavaDataset(test_df, transform=transform, phase='test')\n    test_dl = DataLoader(test_ds, batch_size=cfg['test']['batch_size'], \n                         shuffle=cfg['test']['shuffle'], num_workers=os.cpu_count(), pin_memory=True)\n    dataloaders = {'test': test_dl}\nelse:\n    train_ds = CassavaDataset(train_df, transform=transform, phase='train')\n    val_ds = CassavaDataset(val_df, transform=transform, phase='val')\n\n    train_dl = DataLoader(train_ds, batch_size=cfg['train']['batch_size'], \n                          shuffle=cfg['train']['shuffle'], num_workers=os.cpu_count(), pin_memory=True)\n    val_dl = DataLoader(val_ds, batch_size=cfg['val']['batch_size'], \n                        shuffle=cfg['val']['shuffle'], num_workers=os.cpu_count(), pin_memory=True)\n\n    dataloaders = {'train': train_dl, 'val': val_dl}\n\n    if (DEBUG) & (not INFERENCE):\n        iterator = iter(train_dl)\n        inputs, labels = next(iterator)\n        print('input size: ', inputs.size())\n        print('label size: ', labels.size())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEBUG & INFERENCE:\n    fig, ax = plt.subplots(nrows=1, ncols=cfg['test']['num_TTA'], figsize=(17, 8))\n    for tta in range(cfg['test']['num_TTA']):        \n        ax[tta].imshow(test_ds[3].numpy().transpose((1, 2, 0)))       \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaNet(nn.Module):\n    \n    def __init__(self):\n        super().__init__()\n        if INFERENCE:\n            self.model = EfficientNet.from_name(cfg['model']['arch'], in_channels=3)\n        else:\n            self.model = EfficientNet.from_pretrained(cfg['model']['arch'], in_channels=3)\n        num_in_features = self.model._fc.in_features\n        self.model._fc = nn.Linear(num_in_features, 5)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = CassavaNet()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Criterion / Optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(params = model.parameters(), lr=cfg['optim']['lr'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train / Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RunOneEpoch():\n    \n    def __init__(self, model, dataloaders, criterion, optimizer, scheduler=None):\n        self.model = model\n        self.dataloaders = dataloaders\n        self.criterion = criterion.to(device)\n        self.optimizer = optimizer\n        self.scheduler = scheduler\n        \n    def train(self, phase='train'):\n        batch_loss = 0.0\n        batch_corrects = 0\n        \n        epoch_loss = 0.0\n        epoch_corrects = 0\n        \n        # model setup\n        if phase=='train':\n            model.train()\n        else:\n            model.eval()\n        \n        model.to(device)\n        scaler = GradScaler()\n        \n        # choose dataloader\n        dl = self.dataloaders[phase]\n        \n        preds_list = np.array([])\n        progress_bar = tqdm(dl)\n        for i, (inputs, labels) in enumerate(progress_bar):\n            inputs = inputs.to(device, non_blocking=True)\n            labels = labels.to(device, non_blocking=True)\n            \n            optimizer.zero_grad()\n            \n            with torch.set_grad_enabled(phase=='train'):\n                with autocast():\n                    outputs = self.model(inputs)\n                    loss = self.criterion.forward(outputs, labels)\n                    _, preds = torch.max(outputs, axis=1)\n                \n                if phase=='train':\n                    scaler.scale(loss).backward()\n                    scaler.step(self.optimizer)\n                    scaler.update()\n                else:\n                    preds_list = np.append(preds_list, preds.detach().cpu().numpy())\n                \n                batch_loss = loss.item() * cfg[phase]['batch_size']\n                batch_corrects = torch.sum(preds==labels.detach())\n                batch_corrects = batch_corrects.detach().cpu().numpy()\n                batch_acc = batch_corrects / cfg[phase]['batch_size']\n                \n                num_batch = (i + 1) * cfg[phase]['batch_size']\n                epoch_loss += batch_loss\n                epoch_corrects += batch_corrects\n                \n                train_loss = epoch_loss / num_batch\n                train_acc = epoch_corrects / num_batch\n                \n                batch_loss = loss.item() \n                \n                progress_bar.set_description(f\"{phase}　| Batch: loss: {round(batch_loss, 4)} acc: {round(batch_acc, 4)}, Epoch: loss: {round(train_loss, 4)} acc: {round(train_acc, 4)}\")\n        \n        if phase=='train':\n            if self.scheduler: self.scheduler.step()\n            return train_loss, train_acc\n        else:\n            return train_loss, train_acc, preds_list\n    \n    def inference(self, phase='test'):  \n        # model setup\n        model.eval()\n        model.to(device)\n        \n        # choose dataloader\n        dl = self.dataloaders[phase]\n        \n        preds_list = np.array([])\n        progress_bar = tqdm(dl)\n        for i, inputs in enumerate(progress_bar):\n            inputs = inputs.to(device, non_blocking=True)\n            \n            optimizer.zero_grad()\n            \n            with torch.set_grad_enabled(phase=='train'):\n                outputs = self.model(inputs)\n                _, preds = torch.max(outputs, axis=1)\n                preds_list = np.append(preds_list, preds.cpu().numpy())\n                \n        return preds_list\n    \n    def inference_TTA(self, phase='test'):  \n        # model setup\n        model.eval()\n        \n        # choose dataloader\n        dl = self.dataloaders[phase]\n        \n        preds_list = np.array([])\n        outputs_list = np.zeros((len(dl.dataset), 5))\n        progress_bar = tqdm(dl)\n        \n        for tta in range(cfg['test']['num_TTA']):\n            outputs_tta = np.array([])\n            for i, inputs in enumerate(progress_bar):\n                inputs = inputs.to(device, non_blocking=True)\n\n                with torch.set_grad_enabled(phase=='train'):\n                    outputs = self.model(inputs)\n                    outputs_tta = np.append(outputs_tta, outputs.detach().cpu().numpy())\n            \n            outputs_tta = outputs_tta.reshape(-1, 5)\n            outputs_list = outputs_list + outputs_tta\n        \n        outputs_list = torch.Tensor(outputs_list)\n        _, preds_list = torch.max(outputs_list, axis=1)\n                \n        return preds_list, outputs_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_checkpoint(folder, checkpoint_name):\n    checkpoint_path = os.path.join(folder, checkpoint_name)\n    checkpoint = torch.load(checkpoint_path, map_location=device)\n    return checkpoint","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nif INFERENCE:\n    if cfg['test']['TTA']:\n        outputs = []\n        model = CassavaNet().to(device)\n            \n        try:\n            checkpoint = get_checkpoint(cfg['test']['checkpoint'], f'checkpoint.pth')\n            model.load_state_dict(checkpoint['model'])\n        except:\n            print('No checkpoint.')\n\n        criterion = nn.CrossEntropyLoss()\n        optimizer = optim.Adam(params = model.parameters(), lr=cfg['optim']['lr'])\n\n        # inference (TTA)\n        run = RunOneEpoch(model=model, dataloaders=dataloaders, \n                          criterion=criterion, optimizer=optimizer)\n        _, outputs = run.inference_TTA()\n        \n        # infernce\n        _, preds = torch.max(outputs, axis=1)\n        test_df.loc[:, 'label'] = preds\n        test_df.loc[:, 'label'] = test_df.loc[:, 'label'].astype('uint8')\n        test_df.drop('img_path', inplace=True, axis=1)\n        test_df.to_csv('submission.csv', index=False)\n    else:\n        # inference\n        preds = run.inference()\n        test_df.loc[:, 'label'] = preds\n        test_df.loc[:, 'label'] = test_df.loc[:, 'label'].astype('uint8')\n        test_df.drop('img_path', inplace=True, axis=1)\n        test_df.to_csv('submission.csv', index=False)\nelse:\n    # use checkpoint\n    if cfg['train']['use_checkpoint']:\n        checkpoint = get_checkpoint(cfg['train']['checkpoint_path'], f'checkpoint.pth')\n        model.load_state_dict(checkpoint['model'])\n        optimizer.load_state_dict(checkpoint['optimizer'])\n        optimizer.param_groups[0]['lr'] = cfg['optim']['lr']\n        print('Loaded checkpoint state.')\n\n    scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=cfg['optim']['T_0'], \n                                            T_mult=cfg['optim']['T_mult'], \n                                            eta_min=cfg['optim']['eta_min'])\n\n    # training\n    run = RunOneEpoch(model=model, dataloaders=dataloaders, \n                      criterion=criterion, optimizer=optimizer, scheduler=scheduler)\n\n    # setup\n    loss_train = []; acc_train = []\n    loss_val = []; acc_val = []\n\n    patience = cfg['train']['patience']; num_patience = cfg['train']['patience']\n    preds_oof = []\n    checkpoints = []\n\n    for i in range(cfg['train']['num_epochs']):\n        print(f'--------- Epoch {i}')\n\n        # train\n        loss, acc = run.train(phase='train')\n        loss_train.append(loss); acc_train.append(acc)\n\n        # val\n        loss, acc, preds = run.train(phase='val')\n        loss_val.append(loss); acc_val.append(acc)\n        preds_oof.append(preds)\n\n        # checkpoint\n        save_list = {'epoch': i,\n                     'model': run.model.state_dict(),\n                     'optimizer': run.optimizer.state_dict()\n                    }\n\n        checkpoints.append(save_list)\n\n        # Early Stoppoing\n        if i == 0:pass\n        else:\n            if loss > loss_val[-2]:\n                patience -= 1\n                if patience == 0:\n                    print(f'Early stopping at epoch {i}. Checkpoint is at epoch {i-num_patience}')\n                    try:\n                        loss_val_series = pd.Series(loss_val)\n                        idxmin_loss = loss_val_series.idxmin()\n                        val_df.loc[:, 'pred'] = preds_oof[idxmin_loss]\n                        torch.save(checkpoints[idxmin_loss], f'checkpoint.pth')\n                    except:\n                        val_df.loc[:, 'pred'] = preds_oof[-1]\n                        torch.save(checkpoints[-1], f'checkpoint.pth')\n                    break\n            else:\n                patience = cfg['train']['patience']\n\n        if i==cfg['train']['num_epochs']-1:\n            print('Reached maximum train epoch.')\n            val_df.loc[:, 'pred'] = preds_oof[-1]\n            torch.save(checkpoints[-1], f'checkpoint.pth')\n    \n    \n    # plot loss/acc for train/val phase\n    fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(16, 6))\n    ax[0].plot(loss_train, label='train', marker='o')\n    ax[1].plot(acc_train, label='train', marker='o')\n    ax[0].plot(loss_val, label='val', marker='o')\n    ax[1].plot(acc_val, label='val', marker='o')\n    \n    ax[0].set_xlabel('Epoch'); ax[1].set_xlabel('Epoch')\n    ax[0].set_ylabel('CE Loss'); ax[1].set_ylabel('Accuracy')\n    ax[0].set_title(f'Cross Entropy Loss'); ax[1].set_title('Accuracy')\n    \n    ax[0].set_xlim(0,); ax[1].set_xlim(0,)\n    ax[0].legend(); ax[1].legend()\n    plt.show()\n        \n    # show oof accuracy\n    acc_idx = val_df.label==val_df.pred\n    acc = len(val_df.loc[acc_idx]) / len(val_df)\n\n    print(f'OOF Accuracy: {round(acc, 5)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}