{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch import nn\nimport os\nimport time\nfrom tqdm import tqdm\nimport random\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\nimport timm\nimport pickle\n\nfrom glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'current_fold': 8,\n    'fold_num': 10,\n    'seed': 2021,\n    'model_name': 'tf_efficientnet_b6_ns',\n    'img_size': 512,\n    'epochs': 15,\n    'train_bs': 8,\n    'valid_bs': 16,\n    'T_0': 15,\n    'lr': 1e-3,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_classes': 12,\n    'num_workers': 4,\n    'accum_iter': 1, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/plant-pathology-2021-fgvc8/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.labels.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = list(train['labels'].value_counts().keys())\nlabels_dict = dict(zip(labels, range(CFG['num_classes'])))\ntrain = train.replace({\"labels\": labels_dict})\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed = 2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    return im_rgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PlantDataset(Dataset):\n    def __init__(\n        self, df, data_root, transform = None):\n        super().__init__()\n        self.df = df\n        self.image_id = df['image'].values\n        self.labels = df['labels'].values\n        self.transform = transform\n        self.data_root = data_root\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, idx):\n        image_id = self.image_id[idx]\n        label = self.labels[idx]\n        path = os.path.join(self.data_root, image_id)\n        img  = get_img(path)\n        \n        if self.transform:\n            img = self.transform(image=img)['image']\n            \n        return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomEfficientNet(nn.Module):\n    def __init__(self, model_name=CFG['model_name'], pretrained=True):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        in_features = self.model.get_classifier().in_features\n        self.model.classifier = nn.Linear(in_features, CFG['num_classes'])\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_dataloader(df, trn_idx, val_idx, \n                       data_root='../input/resized-plant2021/img_sz_512'):\n    \n    train_ = df.loc[trn_idx,:].reset_index(drop=True)\n    valid_ = df.loc[val_idx,:].reset_index(drop=True)\n        \n    train_ds = PlantDataset(train_, data_root, transform=get_train_transforms())\n    valid_ds = PlantDataset(valid_, data_root, transform=get_valid_transforms())\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n    )\n    \n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n    return train_loader, val_loader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, \n                    scheduler=None, schd_batch_update=False):\n    model.train()\n\n    t = time.time()\n    running_loss = None\n\n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n\n        #print(image_labels.shape, exam_label.shape)\n        with autocast():\n            image_preds = model(imgs)\n\n            loss = loss_fn(image_preds, image_labels)\n            \n            scaler.scale(loss).backward()\n\n            if running_loss is None:\n                running_loss = loss.item()\n            else:\n                running_loss = running_loss * .99 + loss.item() * .01\n\n            if ((step + 1) %  CFG['accum_iter'] == 0) or ((step + 1) == len(train_loader)):\n\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad() \n                \n                if scheduler is not None and schd_batch_update:\n                    scheduler.step()\n\n            if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(train_loader)):\n                description = f'epoch {epoch} loss: {running_loss:.4f}'\n                \n                pbar.set_description(description)\n                \n    if scheduler is not None and not schd_batch_update:\n        scheduler.step()\n        \ndef valid_one_epoch(epoch, model, loss_fn, val_loader, device, \n                    scheduler=None, schd_loss_update=False):\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    sample_num = 0\n    image_preds_all = []\n    image_targets_all = []\n    \n    pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n        \n        image_preds = model(imgs)   #output = model(input)\n        #print(image_preds.shape, exam_pred.shape)\n        image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n        image_targets_all += [image_labels.detach().cpu().numpy()]\n        \n        loss = loss_fn(image_preds, image_labels)\n        \n        loss_sum += loss.item()*image_labels.shape[0]\n        sample_num += image_labels.shape[0]  \n\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(val_loader)):\n            description = f'epoch {epoch} loss: {loss_sum/sample_num:.4f}'\n            pbar.set_description(description)\n    \n    image_preds_all = np.concatenate(image_preds_all)\n    image_targets_all = np.concatenate(image_targets_all)\n    accuracy = (image_preds_all==image_targets_all).mean()\n    print('validation multi-class accuracy = {:.4f}'.format(accuracy))\n    \n    if scheduler is not None:\n        if schd_loss_update:\n            scheduler.step(loss_sum/sample_num)\n        else:\n            scheduler.step()\n    return accuracy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])\n    \n    for fold in range(CFG['fold_num']):\n\n        if fold != CFG['current_fold']:\n            continue\n            \n        pickle_in = open(\"../input/2021fgvc8folds/train_fold_\" + str(fold) + \".pickle\",\"rb\")\n        train_folds = pickle.load(pickle_in)\n        pickle_in.close()\n    \n        pickle_in = open(\"../input/2021fgvc8folds/valid_fold_\" + str(fold) + \".pickle\",\"rb\")\n        valid_folds = pickle.load(pickle_in)\n        pickle_in.close()\n        \n        trn_idx = list(train_folds)\n        val_idx = list(valid_folds)\n        \n        print('Training with {} started'.format(fold))\n\n        print(len(trn_idx), len(val_idx))\n        train_loader, val_loader = prepare_dataloader(train, trn_idx, val_idx)\n\n        device = torch.device(CFG['device'])\n        \n        model = CustomEfficientNet(CFG['model_name'], pretrained=True).to(device)\n        scaler = GradScaler()   \n        optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'])\n        #scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=CFG['epochs']-1)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)\n        #scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=25, \n        #                                                max_lr=CFG['lr'], epochs=CFG['epochs'], steps_per_epoch=len(train_loader))\n        \n        loss_tr = nn.CrossEntropyLoss().to(device) #MyCrossEntropyLoss().to(device)\n        loss_fn = nn.CrossEntropyLoss().to(device)\n        \n        best_accuracy = 0\n        for epoch in range(CFG['epochs']):\n            train_one_epoch(epoch, model, loss_tr, optimizer, train_loader, device, scheduler=scheduler, schd_batch_update=False)\n            \n            valid_acc = 0\n            with torch.no_grad():\n                valid_acc = valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False)\n\n            if valid_acc > best_accuracy:\n                best_accuracy = valid_acc\n                print('Save best model at epoch {} : {}%'.format(epoch, valid_acc))\n                torch.save(model.state_dict(),'{}_fold_{}.pth'.format(CFG['model_name'], fold))\n            \n        #torch.save(model.cnn_model.state_dict(),'{}/cnn_model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag']))\n        del model, optimizer, train_loader, val_loader, scaler, scheduler\n        torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}