{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Efficient Net Train\n- Amp\n- Focal Loss\n- Fmix\n- Early Stop\n- Adam WR \n\n- if this helps, please do Upvote this code and the original 👍🏼<br><br>\n< Reference Code > <br>\n    -[Cutmix v.s. Fmix with Visualization](https://www.kaggle.com/khyeh0719/cutmix-v-s-fmix-with-visualization)<br>\n    -[Pytorch Efficientnet Baseline [Train] AMP+Aug](https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-train-amp-aug)<br>"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\npackage_path = ['../input/timmpackagelatestwhl', \n                '../input/image-fmix/FMix-master']\nfor pth in package_path:\n    sys.path.append(pth)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\npd.set_option('display.max_row', None)\npd.set_option('display.max_columns', None)\nimport albumentations as albu\nimport matplotlib.pyplot as plt\nimport json\nimport seaborn as sns\nimport cv2\nimport albumentations as albu\nimport numpy as np\nimport random\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings(action='ignore')\n\nimport torch\nimport torch.nn as nn\nimport torchvision.models as models\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torch.optim import AdamW\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, CosineAnnealingWarmRestarts\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.cuda.amp import autocast, GradScaler\n\n\n!pip install ../input/timmpackagelatestwhl/timm-0.3.4-py3-none-any.whl\nimport timm\nfrom fmix import sample_mask, make_low_freq_image, binarise_mask","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load TrainSet"},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE_DIR=\"../input/cassava-leaf-disease-classification/\"\nTRAIN_IMAGES_DIR=os.path.join(BASE_DIR,'train_images/')\ntrain_df=pd.read_csv(os.path.join(BASE_DIR,'train.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(train_df.head())\nprint(train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    return im_rgb\n\ndef rand_bbox(size, lam):\n    W = size[0]\n    H = size[1]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    \n    return bbx1, bby1, bbx2, bby2\n\ndef save_model(model, optimizer, scheduler, fold, epoch, save_every=False, best=False):\n    state = {\n        'model': model.state_dict(),\n        'optimizer': optimizer.state_dict(),\n        'scheduler': scheduler.state_dict()\n    }\n    if save_every == True:\n        if not (os.path.isdir('./saved_model')): os.mkdir('./saved_model')\n        torch.save(state, './saved_model/model_fold_{}_epoch_{}'.format(fold+1, epoch+1))\n    if best == True:\n        if not (os.path.isdir('./best_model')): os.mkdir('./best_model')\n        torch.save(state, './best_model/model_fold_{}_epoch_{}'.format(fold+1, epoch+1))\n        \nclass EarlyStopping:\n    def __init__(self, patience):\n        self.patience = patience\n        self.counter = 0\n        self.early_stop = False\n        self.val_loss_min = np.Inf\n\n    def __call__(self, val_loss, model, optimizer, scheduler, fold, epoch):\n        if self.val_loss_min == np.Inf:\n            self.val_loss_min = val_loss\n        elif val_loss > self.val_loss_min:\n            self.counter += 1\n            print('EarlyStopping counter: {} out of {}'.format(self.counter, self.patience))\n            if self.counter >= self.patience:\n                print('Early Stopping - Fold {} Training is Stopping'.format(fold))\n                self.early_stop = True\n        else:  # val_loss < val_loss_min\n            save_model(model, optimizer, scheduler, fold, epoch, best=True)\n            print('*** Validation loss decreased ({} --> {}).  Saving model... ***'.\\\n                  format(round(self.val_loss_min, 6), round(val_loss, 6)))\n            self.val_loss_min = val_loss\n            self.counter = 0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            RandomResizedCrop(512, 512),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(512, 512, p=1.),\n            Resize(512, 512),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loader"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, df, data_root, transforms=None, do_fmix=False, do_cutmix=False, output_label=True):\n        self.df=df\n        self.data_root=data_root\n        self.transforms=transforms\n        self.do_fmix = do_fmix\n        self.do_cutmix = do_cutmix\n        self.fmix_params={'alpha': 1., \n                          'decay_power': 3., \n                          'shape': (512, 512),\n                          'max_soft': True, \n                          'reformulate': False}\n        self.cutmix_params={'alpha': 1}\n        self.output_label = output_label\n        self.labels = self.df['label'].values\n        \n    def __getitem__(self,index):            \n        img  = get_img(path=\"{}/{}\".format(self.data_root, self.df.image_id.iloc[index]))\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n        if self.output_label:\n            target = self.df.label.iloc[index]\n        \n        if self.do_fmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            with torch.no_grad():\n                lam = np.clip(np.random.beta(self.fmix_params['alpha'], self.fmix_params['alpha']), 0.6, 0.7)\n                mask = make_low_freq_image(self.fmix_params['decay_power'], self.fmix_params['shape'])\n                mask = binarise_mask(mask, lam, self.fmix_params['shape'], self.fmix_params['max_soft'])\n                fmix_ix = np.random.choice(self.df.shape[0], size=1)[0]\n                fmix_img  = get_img(\"{}/{}\".format(self.data_root, self.df.image_id.iloc[fmix_ix]))\n                if self.transforms:\n                    fmix_img = self.transforms(image=fmix_img)['image']\n                mask_torch = torch.from_numpy(mask)\n                rate = mask.sum()/self.fmix_params['shape'][0]/self.fmix_params['shape'][1]\n                \n                img = mask_torch*img+(1.-mask_torch)*fmix_img\n                target = rate*target + (1.-rate)*self.labels[fmix_ix]\n                \n        if self.do_cutmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            with torch.no_grad():\n                cmix_ix = np.random.choice(self.df.index, size=1)[0]\n                cmix_img  = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[cmix_ix]['image_id']))\n                if self.transforms:\n                    cmix_img = self.transforms(image=cmix_img)['image']\n                lam = np.clip(np.random.beta(self.cutmix_params['alpha'], self.cutmix_params['alpha']),0.3,0.4)\n                bbx1, bby1, bbx2, bby2 = rand_bbox(self.fmix_params['shape'], lam)\n                rate = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (self.fmix_params['shape'][0] * self.fmix_params['shape'][1]))\n                img[:, bbx1:bbx2, bby1:bby2] = cmix_img[:, bbx1:bbx2, bby1:bby2]\n                target = rate*target + (1.-rate)*self.labels[cmix_ix]\n                    \n        if self.output_label == True:\n            return img, target\n        else:\n            return img\n        \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, model_name, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Focal Loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass FocalLoss(nn.modules.loss._WeightedLoss):\n    def __init__(self, weight=None, gamma=2,reduction='mean'):\n        super(FocalLoss, self).__init__(weight,reduction=reduction)\n        self.gamma = gamma\n        self.weight = weight\n\n    def forward(self, input_, target):\n        ce_loss = F.cross_entropy(input_, target,reduction=self.reduction,weight=self.weight) \n        pt = torch.exp(-ce_loss)\n        focal_loss = ((1 - pt) ** self.gamma * ce_loss).mean()\n        return focal_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# train / Validation Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_dataloader(df, train_index, val_index, train_batch, valid_batch, num_workers, data_root=TRAIN_IMAGES_DIR):\n    trainset = df.loc[train_index,:].reset_index(drop=True)\n    validset = df.loc[val_index,:].reset_index(drop=True)\n    train_dataset = CassavaDataset(trainset, data_root=data_root, transforms=get_train_transforms(), output_label=True, do_fmix=True)\n    valid_dataset = CassavaDataset(validset, data_root=data_root, transforms=get_valid_transforms(), output_label=True, do_fmix=False)\n    train_loader = DataLoader(train_dataset, batch_size=train_batch, pin_memory=False,drop_last=False,shuffle=True, num_workers=num_workers)\n    val_loader = DataLoader(valid_dataset, batch_size=train_batch, num_workers=num_workers, shuffle=False,pin_memory=False,)\n    return train_loader, val_loader\n\n\ndef train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler):\n    model.train()\n    lst_out = []\n    lst_label = []\n    avg_loss = 0\n\n    status = tqdm(enumerate(train_loader), total=len(train_loader))\n    for step, (images, labels) in status:\n        images = images.to(device).float()\n        labels = labels.to(device).long()\n        with autocast():\n            preds = model(images)\n            lst_out += preds.argmax(1)\n            lst_label += labels\n\n            loss = loss_fn(preds, labels)\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n\n            avg_loss += loss.item() / len(train_loader)\n    scheduler.step()\n    accuracy = accuracy_score(y_pred=torch.tensor(lst_out), y_true=torch.tensor(lst_label))\n    print('{} epoch - train loss : {}, train accuracy : {}'.\\\n          format(epoch + 1, np.round(avg_loss,6), np.round(accuracy*100,2)))\n\ndef valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler):\n    model.eval()\n    lst_val_out = []\n    lst_val_label = []\n    avg_val_loss = 0\n    status = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (images, labels) in status:\n        val_images = images.to(device).float()\n        val_labels = labels.to(device).long()\n\n        val_preds = model(val_images)\n        lst_val_out += val_preds.argmax(1)\n        lst_val_label += val_labels\n        loss = loss_fn(val_preds, val_labels)\n                       \n        avg_val_loss += loss.item() / len(val_loader)\n    accuracy = accuracy_score(y_pred=torch.tensor(lst_val_out), y_true=torch.tensor(lst_val_label))\n    print('{} epoch - valid loss : {}, valid accuracy : {}'.\\\n          format(epoch + 1, np.round(avg_val_loss, 6), np.round(accuracy*100,2)))\n    return avg_val_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Main - Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n    train_batch = 16\n    valid_batch = 32\n    num_workers = 4\n    seed = 719\n    split = 5\n    epochs = 100\n    patience = 5\n\n    n_class = 5\n    model_arch = 'tf_efficientnet_b4_ns'\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    seed_everything(seed)\n    X_train = train_df.iloc[:, :-1]; Y_train = train_df.iloc[:, -1]\n    cv = StratifiedKFold(n_splits=split, random_state=seed, shuffle=True)\n    for fold, (train_index, val_index) in enumerate(cv.split(X_train, Y_train)):\n        if fold == 0:\n            continue\n        torch.cuda.empty_cache()\n        print('---------- Fold {} is training ----------'.format(fold + 1))\n        print('Train Size : {}, Valid Size : {}'.format(len(train_index), len(val_index)))\n        train_loader, val_loader = prepare_dataloader(train_df, train_index, val_index, train_batch, valid_batch, num_workers, data_root=TRAIN_IMAGES_DIR)\n        model = Model(model_arch, n_class, pretrained=True).to(device)\n        loss_tr = FocalLoss().to(device); loss_fn = FocalLoss().to(device)\n        optimizer = AdamW(model.parameters(), lr=1e-4, weight_decay=1e-6)\n        scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1, eta_min=1e-6, last_epoch=-1)\n        scaler = GradScaler()\n        early_stopping = EarlyStopping(patience=patience)\n        for epoch in range(epochs):\n            train_one_epoch(epoch, model, loss_tr, optimizer, train_loader, device, scheduler=scheduler)\n            save_model(model, optimizer, scheduler, fold, epoch, save_every=True)\n            with torch.no_grad():\n                val_loss = valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None)\n                early_stopping(val_loss, model, optimizer, scheduler, fold, epoch)\n                if early_stopping.early_stop:\n                    break\n\n        del model, optimizer, train_loader, val_loader, scheduler, scaler\n        torch.cuda.empty_cache()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}