{"cells":[{"metadata":{"id":"KnRsrvYbU1jy","executionInfo":{"status":"ok","timestamp":1598193780161,"user_tz":-180,"elapsed":19687,"user":{"displayName":"Olena Yukhymenko","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiaVeFIaNiIE57ytUObFjBTriX3tNj7r4-PJr96Nw=s64","userId":"12610305645255153661"}},"outputId":"1636458f-106c-4da9-91bc-578004da1614","trusted":true},"cell_type":"markdown","source":"This kernel is heavily inspired by brilliant notebooks of [@abhishek](https://www.kaggle.com/abhishek) and [@cdeotte](https://www.kaggle.com/cdeotte) in [SIIM-ISIC Melanoma Classification](https://www.kaggle.com/c/siim-isic-melanoma-classification). Here I am re-using many ideas from SIIMISIC competition as well as some new ideas from [\nUsing Tez in Leaf Disease Classification](https://www.kaggle.com/abhishek/using-tez-in-leaf-disease-classification) by @abhishek .\n\nFor the sake of time, you can disable OOF TTA to make overall processing significantly faster. I just monitor OOF TTA to double-check how well applied TTA methods perform.\n\nInference notebook is [Pytorch EfficientNet with TTA [inference]](https://www.kaggle.com/dunklerwald/pytorch-efficientnet-with-tta-inference).\n\n**UPDATE**:\n- switched to noisy-student\n- added simple upsampling option (disabled by default)\n- added confusion matrix\n\n**UPDATE1**:\n- switched to CosineAnnealingWarmRestarts\n- upgraded to image size 512\n- switched back to B4 effnet\n\n**UPDATE(FINAL)**:\n- added weight decay\n- switched to smoothed cross entropy loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"package_path = '../input/pytorch-image-models/pytorch-image-models-master'\nimport sys\nsys.path.append(package_path)    ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","id":"Tqw3CPzjTeL9","trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport albumentations\n\nimport numpy as np\nimport pandas as pd\nimport warnings\n\nimport time\nimport datetime\n\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.figure_factory as ff \n\nimport torch.nn as nn\nfrom sklearn import metrics, model_selection\nfrom torch.nn import functional as F\nfrom torch.nn.modules.loss import _WeightedLoss\n\nfrom PIL import Image\nfrom PIL import ImageFile\n\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\n\nimport timm\n\nwarnings.simplefilter('ignore')\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Parameters"},{"metadata":{"id":"x6blyHjiJkOH","executionInfo":{"status":"ok","timestamp":1598193819013,"user_tz":-180,"elapsed":58466,"user":{"displayName":"Olena Yukhymenko","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiaVeFIaNiIE57ytUObFjBTriX3tNj7r4-PJr96Nw=s64","userId":"12610305645255153661"}},"outputId":"204ac38e-0365-4adf-c9cb-b49b648663f1","trusted":true},"cell_type":"code","source":"n_epochs = 10\nn_patience = 5\nn_folds = 3\ntrain_bsize = 24\nvalid_bsize = 48\ntest_bsize = 48\nseed = 42\n\neffnet_output = {0: 1280, 1: 1280, 2: 1408, 3: 1536, 4: 1792, 5: 2048, 6: 2304, 7: 2560}\n\nIMG_SIZE = 512\nEFFNET_MODEL = 4\n\nAUGMENTATION =[albumentations.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=15, border_mode=0, p=0.6),\n              albumentations.Flip(p=0.5),\n              albumentations.RandomRotate90(p=0.5),\n              albumentations.RandomBrightness(limit=0.2, p=0.6),\n              albumentations.RandomContrast(limit=0.2, p=0.6),\n              albumentations.HueSaturationValue(hue_shift_limit=20, sat_shift_limit=20, val_shift_limit=20, p=0.6),\n              albumentations.CoarseDropout(max_holes=8, max_height=int(IMG_SIZE*0.2), max_width=int(IMG_SIZE*0.2), p=0.6),\n              albumentations.Cutout(num_holes=1, max_h_size=int(IMG_SIZE*0.33), max_w_size=int(IMG_SIZE*0.33), p=0.6)\n              ]   \n\nIS_TTA = True\nTTA = 3 \n\nUPSAMPLE = False\nN_UPSAMPLE = 1\n\nSCHEDULER_NAME = 'CosineAnnealingLR' # ReduceLROnPlateau, CosineAnnealingLR, CustomSchedulerLR\nLOSS_FN_NAME = 'SmoothCrossEntropyLoss' # SmoothCrossEntropyLoss,CrossEntropyLoss \n\nSMOOTHING = 0.05\n\n\nDISPLAY_PLOT= True\n\ndef set_seed(seed):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"id":"ocRSZoaBgcn0","trusted":true},"cell_type":"code","source":"path = '../input/cassava-leaf-disease-classification/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create folds\ndf = pd.read_csv(path + 'train.csv')\ndf[\"kfold\"] = -1    \ndf = df.sample(frac=1).reset_index(drop=True)\n\n\nkf = model_selection.StratifiedKFold(n_splits=n_folds, random_state = seed, shuffle=True)\n\nfor f, (t_, v_) in enumerate(kf.split(np.arange(df.shape[0]), df.label.values)):\n    df.loc[df.index.isin(v_), ['kfold']] = f\n    \ndf.to_csv(\"train_folds.csv\", index=False)\n\nN_CLASSES = df.label.nunique()\nLABELS = ['Cassava Bacterial Blight','Cassava Brown Streak Disease','Cassava Green Mottle','Cassava Mosaic Disease','Healthy']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plotting"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def plot_fold(history, title):\n\n    fig = make_subplots(specs=[[{\"secondary_y\": True}]])\n\n    \n    trace = (        \n          px.line(history, x=history.index+1, y='train_loss') \n         .add_trace(px.line(history, x=history.index+1, y='val_loss').data[0]) \n         .add_trace(px.line(history, x=history.index+1, y='train_accuracy').data[0])\n         .add_trace(px.line(history, x=history.index+1, y='val_accuracy').data[0]) \n         \n        ).data\n\n    \n    fig.add_trace(trace[0], secondary_y=False) \n    fig.add_trace(trace[1], secondary_y=False) \n    fig.add_trace(trace[2], secondary_y=True)\n    fig.add_trace(trace[3], secondary_y=True)\n\n    \n    fig.data[0].line.dash='dash';fig.data[0].mode ='markers+lines';fig.data[0].line.color='#2ca02c';fig.data[0].line.width=3;fig.data[0].hovertemplate=None;fig.data[0].name='train loss' \n    fig.data[1].line.dash='dash';fig.data[1].mode ='markers+lines';fig.data[1].line.color='#d62728';fig.data[1].line.width=3;fig.data[1].hovertemplate=None;fig.data[1].name='val loss'\n    fig.data[2].line.dash='dashdot';fig.data[2].mode ='markers+lines';fig.data[2].line.color='#ff7f0e';fig.data[2].line.width=3;fig.data[2].hovertemplate=None;fig.data[2].name='train accuracy'\n    fig.data[3].line.dash='dashdot';fig.data[3].mode ='markers+lines';fig.data[3].line.color='#1f77b4';fig.data[3].line.width=3;fig.data[3].hovertemplate=None;fig.data[3].name='val accuracy'\n    \n    \n    # Set x-axis title\n    fig.update_xaxes(title_text=\"Epoch\")\n\n    # Set y-axes titles\n    fig.update_yaxes(title_text=\"Loss\", secondary_y=False)    \n    fig.update_yaxes(title_text=\"Accuracy\", secondary_y=True)\n    fig.update_layout(height=450, margin=dict(r=5, t=50, b=50, l=5), title_text='<b>'+title+'</b>', title_font_size=12, legend=dict(orientation='h',yanchor='top',y=1.03,xanchor='left',x=0.15))\n    fig.update_layout(font_size=12)\n    fig.for_each_annotation(lambda a: a.update(font=dict(size=14)))\n    fig.update_layout(hovermode=\"x unified\")\n    fig.update_traces(showlegend=True)\n    \n    fig.show()\n    \n\n    \ndef plot_confusion_matrix(label, pred):\n    c_matrix = metrics.confusion_matrix(label, pred, labels=range(len(LABELS)), normalize='true')\n    df = pd.DataFrame(c_matrix, index=LABELS, columns=LABELS)\n    df_text = np.around(df.values, decimals=2)\n\n    fig = ff.create_annotated_heatmap(df.values, annotation_text=df_text, x=LABELS, y=LABELS, colorscale='PuBu' )\n    fig.update_layout(font_size=9, height=450, margin=dict(r=5, t=50, b=50, l=5)) \n    \n    fig.show()       ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"id":"x45rWUshEWPz","trusted":true},"cell_type":"code","source":"class ClassificationDataset:\n    def __init__(self, image_paths, targets, resize, augmentations=None):\n        self.image_paths = image_paths\n        self.targets = targets\n        self.resize = resize\n        self.augmentations = augmentations\n\n    def __len__(self):\n        return len(self.image_paths)\n\n    def __getitem__(self, item):\n        image = Image.open(self.image_paths[item])\n        targets = self.targets[item]\n        if self.resize is not None:\n            image = image.resize(\n                (self.resize[1], self.resize[0]), resample=Image.BILINEAR\n            )\n        image = np.array(image)\n        if self.augmentations is not None:\n            augmented = self.augmentations(image=image)\n            image = augmented[\"image\"]\n        image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n        return {\n            \"image\": torch.tensor(image),\n            \"targets\": torch.tensor(targets),\n        }\n\n\nclass ClassificationDataLoader:\n    def __init__(self, image_paths, targets, resize, augmentations=None):\n        self.image_paths = image_paths\n        self.targets = targets\n        self.resize = resize\n        self.augmentations = augmentations\n        self.dataset = ClassificationDataset(\n            image_paths=self.image_paths,\n            targets=self.targets,\n            resize=self.resize,\n            augmentations=self.augmentations\n        )\n    \n    def fetch(self, batch_size, num_workers, drop_last=False, shuffle=True, tpu=False):\n        sampler = None\n\n        data_loader = torch.utils.data.DataLoader(\n            self.dataset,\n            batch_size=batch_size,\n            sampler=sampler,\n            drop_last=drop_last,\n            shuffle=shuffle,\n            num_workers=num_workers\n        )\n        return data_loader","execution_count":null,"outputs":[]},{"metadata":{"id":"8DzS-553ClYb","trusted":true},"cell_type":"code","source":"class AverageMeter:\n    \"\"\"\n    Computes and stores the average and current value\n    \"\"\"\n\n    def __init__(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"id":"9NAsqvfwCssp","trusted":true},"cell_type":"code","source":"class EarlyStopping:\n    def __init__(self, patience=7, mode=\"max\", delta=0.0001, tpu=False):\n        self.patience = patience\n        self.counter = 0\n        self.mode = mode\n        self.best_score = None\n        self.early_stop = False\n        self.tpu = tpu\n        self.delta = delta\n        if self.mode == \"min\":\n            self.val_score = np.Inf\n        else:\n            self.val_score = -np.Inf\n\n    def __call__(self, epoch_score, model, model_path):\n        if self.mode == \"min\":\n            score = -1.0 * epoch_score\n        else:\n            score = np.copy(epoch_score)\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(epoch_score, model, model_path)\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            print(\n                \"EarlyStopping counter: {} out of {}\".format(\n                    self.counter, self.patience\n                )\n            )\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(epoch_score, model, model_path)\n            self.counter = 0\n\n    def save_checkpoint(self, epoch_score, model, model_path):\n        if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:\n            if self.tpu:\n                xm.master_print(\n                    \"Validation score improved ({} --> {}). Saving model!\".format(\n                        self.val_score, epoch_score\n                    )\n                )\n            else:\n                print(\n                    \"Validation score improved ({} --> {}). Saving model!\".format(\n                        self.val_score, epoch_score\n                    )\n                )\n            if self.tpu:\n                xm.save(model.state_dict(), model_path)\n            else:\n                torch.save(model.state_dict(), model_path)\n        self.val_score = epoch_score","execution_count":null,"outputs":[]},{"metadata":{"id":"DJdfxT6KBlWM","trusted":true},"cell_type":"code","source":"class Engine:\n    @staticmethod\n    def train(\n        data_loader,\n        model,\n        optimizer,\n        device,\n        scheduler=None,\n        accumulation_steps=1,\n        fp16=True,\n    ):\n\n        losses = AverageMeter()\n        accuracies = AverageMeter()\n        final_predictions = []\n        model.train()\n        if accumulation_steps > 1:\n            optimizer.zero_grad()\n\n        if fp16:\n          scaler = torch.cuda.amp.GradScaler()    \n\n        for b_idx, data in enumerate(data_loader):\n            for key, value in data.items():\n                data[key] = value.to(device)\n            if accumulation_steps == 1 and b_idx == 0:\n                optimizer.zero_grad()\n            if fp16:    \n                with torch.cuda.amp.autocast():    \n                    predictions, loss, accuracy = model(**data)\n            else:\n                predictions, loss, accuracy = model(**data)\n\n            predictions = predictions.detach().cpu().numpy()  \n            final_predictions.append(predictions) \n\n            with torch.set_grad_enabled(True):\n                if fp16:\n                    scaler.scale(loss).backward()                   \n                else:\n                    loss.backward()\n                if (b_idx + 1) % accumulation_steps == 0:\n                    if fp16:\n                        scaler.step(optimizer)\n                        scaler.update()\n                    else:     \n                        optimizer.step()\n                    if scheduler is not None:\n                         scheduler.step()\n                    if b_idx > 0:\n                        optimizer.zero_grad()\n\n            losses.update(loss.item(), data_loader.batch_size)\n            accuracies.update(accuracy.item(), data_loader.batch_size)\n\n        return final_predictions, losses.avg, accuracies.avg\n\n    @staticmethod\n    def evaluate(data_loader, model, device):\n        losses = AverageMeter()\n        accuracies = AverageMeter()\n        final_predictions = []\n        model.eval()\n        with torch.no_grad():\n            for b_idx, data in enumerate(data_loader):    \n                for key, value in data.items():\n                    data[key] = value.to(device)\n                predictions, loss, accuracy = model(**data)\n                predictions = predictions.detach().cpu().numpy()  \n                final_predictions.append(predictions) \n                \n                losses.update(loss.item(), data_loader.batch_size)    \n                accuracies.update(accuracy.item(), data_loader.batch_size)\n\n        return final_predictions, losses.avg, accuracies.avg\n\n    @staticmethod\n    def predict(data_loader, model, device):\n        model.eval()\n        final_predictions = []\n\n        with torch.no_grad():\n\n            for b_idx, data in enumerate(data_loader):    \n                for key, value in data.items():\n                    data[key] = value.to(device)\n                predictions, _, _ = model(**data)\n                predictions = predictions.detach().cpu().numpy()  \n                final_predictions.append(predictions) \n                   \n        return final_predictions","execution_count":null,"outputs":[]},{"metadata":{"id":"42EPPtZiPJw9","trusted":true},"cell_type":"code","source":"class CustomSchedulerLR:\n    def lrfn(self, epoch):      \n        if epoch < self.lr_ramp_ep:\n            lr = (self.lr_max - self.lr_start) / self.lr_ramp_ep * epoch + self.lr_start           \n        elif epoch < self.lr_ramp_ep + self.lr_sus_ep:\n            lr = self.lr_max\n        else:\n            lr = (self.lr_max - self.lr_min) * self.lr_decay**(epoch - self.lr_ramp_ep - self.lr_sus_ep) + self.lr_min\n\n        return lr   \n\n    def __init__(self, optimizer, epoch, batch_size):\n        self.lr_start = 0.00005\n        self.lr_min = 0.00005\n        self.lr_ramp_ep = 5\n        self.lr_sus_ep = 0\n        self.lr_decay = 0.8  \n        self.lr_max = 0.00001 * batch_size\n        self.optimizer = optimizer\n\n        lr = self.lrfn(epoch)    \n        for param_group in self.optimizer.param_groups:\n            param_group['lr'] = lr\n\n    def step(self, epoch):\n        lr = self.lrfn(epoch)  \n        for param_group in self.optimizer.param_groups:\n            param_group['lr'] = lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#source: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/173733\nclass SmoothCrossEntropyLoss(_WeightedLoss):\n    def __init__(self, weight=None, reduction='mean', smoothing=SMOOTHING):\n        super().__init__(weight=weight, reduction=reduction)\n        self.smoothing = smoothing\n        self.weight = weight\n        self.reduction = reduction\n\n    @staticmethod\n    def _smooth_one_hot(targets:torch.Tensor, n_classes:int, smoothing=SMOOTHING):\n        assert 0 <= smoothing < 1\n        with torch.no_grad():\n            targets = torch.empty(size=(targets.size(0), n_classes),\n                    device=targets.device) \\\n                .fill_(smoothing /(n_classes-1)) \\\n                .scatter_(1, targets.data.unsqueeze(1), 1.-smoothing)\n        return targets\n\n    def forward(self, inputs, targets):\n        targets = SmoothCrossEntropyLoss._smooth_one_hot(targets, inputs.size(-1),\n            self.smoothing)\n        lsm = F.log_softmax(inputs, -1)\n\n        if self.weight is not None:\n            lsm = lsm * self.weight.unsqueeze(0)\n\n        loss = -(targets * lsm).sum(-1)\n\n        if  self.reduction == 'sum':\n            loss = loss.sum()\n        elif  self.reduction == 'mean':\n            loss = loss.mean()\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"id":"2KKNqpqrTeMG","trusted":true},"cell_type":"code","source":"class EfficientNet(nn.Module):\n    def __init__(self, num_classes):\n        super(EfficientNet, self).__init__()\n        self.base_model = timm.create_model(f\"tf_efficientnet_b{str(EFFNET_MODEL)}_ns\", pretrained=True)\n        self.dropout = nn.Dropout(0.2)\n        \n        self.out = nn.Linear(\n            in_features=effnet_output[EFFNET_MODEL], \n            out_features=num_classes, \n            bias=True\n        )\n        \n    def forward(self, image, targets=None):\n        batch_size, _, _, _ = image.shape\n        \n        x = self.base_model.forward_features(image) \n        x = F.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)\n        out = self.out(self.dropout(x))\n        \n        if LOSS_FN_NAME == 'CrossEntropyLoss':\n            loss = nn.CrossEntropyLoss()(out, targets.long())  \n        elif LOSS_FN_NAME == 'SmoothCrossEntropyLoss':\n            loss = SmoothCrossEntropyLoss()(out, targets.long())\n        else:\n            loss = nn.CrossEntropyLoss()(out, targets.long())\n\n        outputs = torch.argmax(out, dim=1).detach().cpu().numpy()\n        targets = targets.detach().cpu().numpy()\n        accuracy = metrics.accuracy_score(targets, outputs)    \n        \n        return out, loss, accuracy","execution_count":null,"outputs":[]},{"metadata":{"id":"8ogikXZGTeMY","trusted":true},"cell_type":"code","source":"def train(fold = 0, apply_tta = False):\n    print('=' * 20, 'Fold', fold, '=' * 20)\n    model_path=f\"model_fold_{fold}.bin\"\n    start_time = time.time()\n\n    device = \"cuda\"\n    epochs = n_epochs\n    train_bs = train_bsize\n    valid_bs = valid_bsize\n\n    training_data_path = path + \"train_images/\"\n    df = pd.read_csv(\"train_folds.csv\")\n\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n\n\n    model = EfficientNet(num_classes=N_CLASSES)\n    model.to(device)\n\n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225)\n\n    train_aug = albumentations.Compose(AUGMENTATION + [albumentations.RandomResizedCrop(IMG_SIZE, IMG_SIZE, always_apply=True), albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)])\n    valid_aug = albumentations.Compose([albumentations.CenterCrop(IMG_SIZE, IMG_SIZE, always_apply=True),albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)])\n\n    train_images = df_train.image_id.values.tolist()\n    train_images = [os.path.join(training_data_path, i ) for i in train_images]\n    train_targets = df_train.label.values\n\n    valid_images = df_valid.image_id.values.tolist()\n    valid_images = [os.path.join(training_data_path, i ) for i in valid_images]\n    valid_targets = df_valid.label.values\n\n    # UPSAMPLE MINORITY CLASSES\n    if UPSAMPLE:\n        train_images_ = df_train.loc[df_train['label']!=3, ['image_id', 'label']]\n        train_images_m = [os.path.join(training_data_path, i) for i in train_images_.image_id.values.tolist()]\n        train_targets_m = train_images_['label']\n        \n        for i in range(N_UPSAMPLE):\n            train_images = train_images + train_images_m\n            train_targets = np.concatenate([train_targets, train_targets_m])\n\n\n    # preliminary shuffle train_images and train_targets correspondingly if shuffle=False for train_loader        \n    train_loader = ClassificationDataLoader(\n        image_paths=train_images,\n        targets=train_targets,\n        resize=None,\n        augmentations=train_aug,\n    ).fetch(\n        batch_size=train_bs, \n        drop_last=True, \n        num_workers=4, \n        shuffle=True\n    )\n\n    valid_loader = ClassificationDataLoader(\n        image_paths=valid_images,\n        targets=valid_targets,\n        resize=None,\n        augmentations=valid_aug,\n    ).fetch(\n        batch_size=valid_bs, \n        drop_last=False, \n        num_workers=4, \n        shuffle=False\n    )\n\n    # SETUP OPTIMIZER AND SCHEDULE\n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-4, weight_decay = 1e-6)\n\n    if SCHEDULER_NAME == 'CosineAnnealingLR': \n        scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=epochs, T_mult=1, eta_min=1e-6, last_epoch=-1)\n    elif SCHEDULER_NAME == 'ReduceLROnPlateau':\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n          optimizer,\n          patience=1, \n          factor=0.25,\n          min_lr=1e-6,\n          verbose=True,\n          mode=\"max\"\n        )\n\n    else:\n        scheduler = CustomSchedulerLR(optimizer=optimizer, epoch=0, batch_size=train_bs)\n\n    \n    es = EarlyStopping(patience=n_patience, mode=\"max\")\n    \n    \n    history = pd.DataFrame(columns=['train_loss','val_loss','train_accuracy','val_accuracy'])\n    \n    \n    # TRAIN\n    print('Training...')\n    for epoch in range(epochs):\n        for param_group in optimizer.param_groups:\n            lr = param_group['lr']\n\n        predictions, train_loss, train_accuracy = Engine.train(train_loader, model, optimizer, device=device)\n        train_predictions = np.vstack((predictions))\n        predictions, valid_loss, val_accuracy = Engine.evaluate(valid_loader, model, device=device)\n        predictions = np.vstack((predictions))\n        \n        history = history.append({'train_loss':train_loss,'val_loss':valid_loss,'train_accuracy':train_accuracy,'val_accuracy':val_accuracy}, ignore_index=True)\n        print(f\"Epoch {epoch+1:03}: | train_loss: {train_loss:.4f} | val_loss {valid_loss:.4f} | train_accuracy {train_accuracy:.4f} | val_accuracy {val_accuracy:.4f} | lr {lr:.6f}\")\n\n        if SCHEDULER_NAME == 'CosineAnnealingLR':\n            scheduler.step(epoch)\n        elif SCHEDULER_NAME == 'ReduceLROnPlateau':\n            scheduler.step(val_accuracy)\n        else:\n            scheduler.step(epoch+1)  \n\n        es(val_accuracy, model, model_path=f\"model_fold_{fold}.bin\")\n        \n        if es.counter == 0: \n            best_prediction = predictions.argmax(axis=1)\n        if es.early_stop:\n            print(\"Early stopping\")\n            break\n    \n    # PREDICT OOF WITH TTA\n    # you can remove this phase to speed up the process\n    if apply_tta:\n        print('Predicting OOF with TTA...')\n        oof_tta_predictions = np.zeros([len(valid_targets),N_CLASSES])\n\n        model = EfficientNet(num_classes=N_CLASSES)\n        model.load_state_dict(torch.load(model_path))\n        model.to(device)\n\n        valid_tta_aug = albumentations.Compose(AUGMENTATION + [valid_aug])\n\n        valid_tta_loader = ClassificationDataLoader(\n            image_paths=valid_images,\n            targets=valid_targets,\n            resize=None,\n            augmentations=valid_tta_aug,\n        ).fetch(\n            batch_size=valid_bs, \n            drop_last=False, \n            num_workers=4, \n            shuffle=False\n        )\n        \n        for i in range(TTA): \n            tta_predictions = Engine.predict(valid_tta_loader, model, device=device)\n            tta_predictions = np.vstack((tta_predictions))\n            oof_tta_predictions += tta_predictions/TTA \n        \n        oof_tta_predictions = oof_tta_predictions.argmax(axis=1)\n        \n        \n    if apply_tta:\n        title = 'Fold {}: | training time: {} | OOF Accuracy without TTA: {:.5f} | OOF Accuracy with TTA: {:.5f} '.format(fold, str(datetime.timedelta(seconds=time.time() - start_time))[:7], es.best_score, metrics.accuracy_score(valid_targets, oof_tta_predictions))\n    else:\n        title = 'Fold {}: | training time: {} | OOF Accuracy without TTA: {:.5f}'.format(fold, str(datetime.timedelta(seconds=time.time() - start_time))[:7], es.best_score)\n    print(title)  \n    \n\n    # PLOT TRAINING\n    if DISPLAY_PLOT:\n        plot_fold(history, title = title)\n    \n\n    if apply_tta:\n        return best_prediction, oof_tta_predictions\n    else:      \n        return best_prediction, _","execution_count":null,"outputs":[]},{"metadata":{"id":"qe8sNlN3TeMb","trusted":true},"cell_type":"code","source":"def predict(fold = 0, apply_tta = False):\n    print('=' * 20, 'Fold', fold, '=' * 20)\n    test_data_path = path + \"test_images/\"\n    df = test\n    device = \"cuda\"\n    model_path=f\"model_fold_{fold}.bin\"\n\n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225)\n    if apply_tta:\n        aug = albumentations.Compose(AUGMENTATION + [albumentations.RandomResizedCrop(IMG_SIZE, IMG_SIZE, always_apply=True), albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)])\n    else:\n        aug = albumentations.Compose([albumentations.CenterCrop(IMG_SIZE, IMG_SIZE, always_apply=True),albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)])\n    \n    images = [os.path.join(test_data_path, x) for x in df.image_id.values]\n    targets = df.label.values\n\n    test_loader = ClassificationDataLoader(\n        image_paths=images,\n        targets=targets,\n        resize=None,\n        augmentations=aug,\n    ).fetch(\n        batch_size=test_bsize, \n        drop_last=False, \n        num_workers=4, \n        shuffle=False\n    )\n\n    model = EfficientNet(num_classes=N_CLASSES)\n    model.load_state_dict(torch.load(model_path))\n    model.to(device)\n\n    # PREDICT\n    print('Predicting...')\n    if apply_tta:\n        predictions = np.zeros([len(images),N_CLASSES])\n        \n        for i in range(TTA): \n            tta_predictions = Engine.predict(test_loader, model, device=device)\n            tta_predictions = np.vstack(tta_predictions)\n            predictions += tta_predictions/TTA  \n        predictions = predictions.reshape((len(images),1, N_CLASSES))    \n    else:\n        predictions = Engine.predict(test_loader, model, device=device)\n\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"id":"Sm6PKRPHZSrb","executionInfo":{"status":"error","timestamp":1598201208172,"user_tz":-180,"elapsed":7447563,"user":{"displayName":"Olena Yukhymenko","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiaVeFIaNiIE57ytUObFjBTriX3tNj7r4-PJr96Nw=s64","userId":"12610305645255153661"}},"outputId":"78bcd547-ee64-41ab-cb48-bd835b544ab0","trusted":true},"cell_type":"code","source":"oof = df[['image_id','kfold','label']].copy()\noof['pred'] = 0\n\nif IS_TTA:\n    oof_tta = oof.copy()\n    for i in range(n_folds):\n        oof.loc[oof['kfold']==i, 'pred'], oof_tta.loc[oof['kfold']==i, 'pred'] = train(fold = i, apply_tta = IS_TTA)    \nelse:\n    for i in range(n_folds):\n        oof.loc[oof['kfold']==i, 'pred'], _ = train(fold = i, apply_tta = IS_TTA)        ","execution_count":null,"outputs":[]},{"metadata":{"id":"E1A6kru0y_6a","trusted":true},"cell_type":"code","source":"if IS_TTA:\n    print('Overall OOF Accuracy without TTA: {:.5f} | OOF Accuracy with TTA: {:.5f}'.format(metrics.accuracy_score(oof['label'], oof['pred']), metrics.accuracy_score(oof['label'], oof_tta['pred'])))\nelse:\n    print('Overall OOF Accuracy without TTA: {:.5f} '.format(metrics.accuracy_score(oof['label'], oof['pred'])))    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot confusion matrix\nplot_confusion_matrix(oof['label'], oof['pred'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict "},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(path + \"sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"id":"e_6dJG-w9PaU","trusted":true},"cell_type":"code","source":"final_preds = None\n\nfor i in range(n_folds):\n    preds = predict(fold = i, apply_tta=IS_TTA)\n    temp_preds = None\n    for p in preds:\n        if temp_preds is None:\n            temp_preds = p\n        else:\n            temp_preds = np.vstack((temp_preds, p))\n    if final_preds is None:\n        final_preds = temp_preds\n    else:\n        final_preds += temp_preds\n\nfinal_preds /= n_folds\nfinal_preds = final_preds.argmax(axis=1)\n\ntest.label = final_preds\ntest.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}