{"cells":[{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:15.956716Z","iopub.status.busy":"2021-01-13T08:46:15.955788Z","iopub.status.idle":"2021-01-13T08:46:49.200130Z","shell.execute_reply":"2021-01-13T08:46:49.199440Z"},"papermill":{"duration":33.274494,"end_time":"2021-01-13T08:46:49.200279","exception":false,"start_time":"2021-01-13T08:46:15.925785","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"!pip install --upgrade pip\n!pip install git+https://github.com/ildoonet/pytorch-randaugment > /dev/null\nfrom RandAugment import RandAugment\n!pip install git+https://github.com/ildoonet/pytorch-gradual-warmup-lr.git > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:49.251465Z","iopub.status.busy":"2021-01-13T08:46:49.250186Z","iopub.status.idle":"2021-01-13T08:46:49.926461Z","shell.execute_reply":"2021-01-13T08:46:49.925146Z"},"papermill":{"duration":0.703653,"end_time":"2021-01-13T08:46:49.926597","exception":false,"start_time":"2021-01-13T08:46:49.222944","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"package_paths = [\n    '../input/pytorch-image-models/pytorch-image-models-master',\n    '../input/image-fmix/FMix-master'\n]\nimport sys; \n\nfor pth in package_paths:\n    sys.path.append(pth)\n    \nfrom fmix import sample_mask, make_low_freq_image, binarise_mask","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2021-01-13T08:46:49.981079Z","iopub.status.busy":"2021-01-13T08:46:49.980234Z","iopub.status.idle":"2021-01-13T08:46:51.424695Z","shell.execute_reply":"2021-01-13T08:46:51.423543Z"},"papermill":{"duration":1.476417,"end_time":"2021-01-13T08:46:51.424824","exception":false,"start_time":"2021-01-13T08:46:49.948407","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport gc\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\n\nimport timm\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\nimport pickle\nfrom scipy.ndimage.interpolation import zoom","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:51.483625Z","iopub.status.busy":"2021-01-13T08:46:51.480538Z","iopub.status.idle":"2021-01-13T08:46:51.484869Z","shell.execute_reply":"2021-01-13T08:46:51.484305Z"},"papermill":{"duration":0.038527,"end_time":"2021-01-13T08:46:51.484981","exception":false,"start_time":"2021-01-13T08:46:51.446454","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 719,\n    'model_arch': 'deit_base_patch16_224',\n    'img_size': 224,\n    'target_size':5,\n    'freeze_epo' : 0,\n    'warmup_epo' : 1, # GradualWarmupSchedulerV2\n    'cosine_epo' : 19, # GradualWarmupSchedulerV2\n    'epochs': 30,\n    'train_bs': 16,\n    'valid_bs': 32,\n    'T_0': 10,\n    'lr': 1e-4,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_workers': 4,\n    'accum_iter': 2, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'smoothing' : 0.2,\n    't1' : 0.8,\n    't2' : 1.4,\n    'N':3,\n    'M':11,\n    'Fold':4\n}","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2021-01-13T08:46:51.533591Z","iopub.status.busy":"2021-01-13T08:46:51.532923Z","iopub.status.idle":"2021-01-13T08:46:51.573802Z","shell.execute_reply":"2021-01-13T08:46:51.574311Z"},"papermill":{"duration":0.068633,"end_time":"2021-01-13T08:46:51.574444","exception":false,"start_time":"2021-01-13T08:46:51.505811","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:51.620051Z","iopub.status.busy":"2021-01-13T08:46:51.619378Z","iopub.status.idle":"2021-01-13T08:46:51.628888Z","shell.execute_reply":"2021-01-13T08:46:51.628384Z"},"papermill":{"duration":0.033674,"end_time":"2021-01-13T08:46:51.628987","exception":false,"start_time":"2021-01-13T08:46:51.595313","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:51.678665Z","iopub.status.busy":"2021-01-13T08:46:51.678061Z","iopub.status.idle":"2021-01-13T08:46:51.689015Z","shell.execute_reply":"2021-01-13T08:46:51.689512Z"},"papermill":{"duration":0.038328,"end_time":"2021-01-13T08:46:51.689640","exception":false,"start_time":"2021-01-13T08:46:51.651312","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.023536,"end_time":"2021-01-13T08:46:51.735411","exception":false,"start_time":"2021-01-13T08:46:51.711875","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Helper Functions"},{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:51.788818Z","iopub.status.busy":"2021-01-13T08:46:51.788025Z","iopub.status.idle":"2021-01-13T08:46:52.071740Z","shell.execute_reply":"2021-01-13T08:46:52.072307Z"},"papermill":{"duration":0.314307,"end_time":"2021-01-13T08:46:52.072447","exception":false,"start_time":"2021-01-13T08:46:51.758140","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.025929,"end_time":"2021-01-13T08:46:52.125159","exception":false,"start_time":"2021-01-13T08:46:52.099230","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:52.193558Z","iopub.status.busy":"2021-01-13T08:46:52.191588Z","iopub.status.idle":"2021-01-13T08:46:52.194389Z","shell.execute_reply":"2021-01-13T08:46:52.194992Z"},"papermill":{"duration":0.042732,"end_time":"2021-01-13T08:46:52.195128","exception":false,"start_time":"2021-01-13T08:46:52.152396","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"TRAIN_PATH = '../input/cassava-leaf-disease-classification/train_images'\nclass TrainDataset(Dataset):\n    def __init__(self, df, transform=None , randagument= False):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.labels = df['label'].values\n        self.transform = transform\n        self.randagument = randagument\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TRAIN_PATH}/{file_name}'\n        image = get_img(file_path)\n        if self.randagument:\n            image = transforms.Compose([\n                        transforms.ToPILImage(),\n                        RandAugment(CFG['N'], CFG['M']),\n                    ])(image)\n            image = np.array(image)\n        if self.transform:\n            image = self.transform(image=image)['image']\n        label = torch.tensor(self.labels[idx]).long()\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.027206,"end_time":"2021-01-13T08:46:52.248999","exception":false,"start_time":"2021-01-13T08:46:52.221793","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Define Train\\Validation Image Augmentations"},{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:52.316469Z","iopub.status.busy":"2021-01-13T08:46:52.314787Z","iopub.status.idle":"2021-01-13T08:46:53.002803Z","shell.execute_reply":"2021-01-13T08:46:53.003376Z"},"papermill":{"duration":0.727622,"end_time":"2021-01-13T08:46:53.003529","exception":false,"start_time":"2021-01-13T08:46:52.275907","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:53.094357Z","iopub.status.busy":"2021-01-13T08:46:53.093293Z","iopub.status.idle":"2021-01-13T08:46:53.502376Z","shell.execute_reply":"2021-01-13T08:46:53.503520Z"},"papermill":{"duration":0.458566,"end_time":"2021-01-13T08:46:53.503704","exception":false,"start_time":"2021-01-13T08:46:53.045138","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from matplotlib import pyplot as plt\n\ntrain_dataset = TrainDataset(train, transform=None)\n\nfor i in range(1):\n    image, label = train_dataset[i]\n    plt.imshow(image)\n    plt.title(f'label: {label}')\n    plt.show() ","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:53.622481Z","iopub.status.busy":"2021-01-13T08:46:53.621799Z","iopub.status.idle":"2021-01-13T08:46:54.446565Z","shell.execute_reply":"2021-01-13T08:46:54.447083Z"},"papermill":{"duration":0.889285,"end_time":"2021-01-13T08:46:54.447240","exception":false,"start_time":"2021-01-13T08:46:53.557955","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train_dataset = TrainDataset(train, transform=get_train_transforms())\nfor i in range(3):\n    image, label = train_dataset[i]\n    plt.imshow(image[0])\n    plt.title(f'label: {label}')\n    plt.show() ","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.046362,"end_time":"2021-01-13T08:46:54.535976","exception":false,"start_time":"2021-01-13T08:46:54.489614","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Model"},{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:54.631618Z","iopub.status.busy":"2021-01-13T08:46:54.629669Z","iopub.status.idle":"2021-01-13T08:46:54.632346Z","shell.execute_reply":"2021-01-13T08:46:54.632854Z"},"papermill":{"duration":0.055607,"end_time":"2021-01-13T08:46:54.632979","exception":false,"start_time":"2021-01-13T08:46:54.577372","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_name=CFG['model_arch'], pretrained=False):\n        super().__init__()\n        try:\n            self.model = torch.hub.load('facebookresearch/deit:main', model_name, pretrained=pretrained)\n        except:\n            self.model = torch.hub.load('facebookresearch/deit:main', model_name, pretrained=False)\n        if pretrained:\n            self.model.load_state_dict(torch.load('/root/.cache/torch/hub/checkpoints/deit_base_patch16_224-b5f2ef4d.pth')['model'])\n        n_features = self.model.head.in_features\n        self.model.head = nn.Linear(n_features, CFG['target_size'])\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.040014,"end_time":"2021-01-13T08:46:54.714564","exception":false,"start_time":"2021-01-13T08:46:54.674550","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Training APIs"},{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:54.904421Z","iopub.status.busy":"2021-01-13T08:46:54.899341Z","iopub.status.idle":"2021-01-13T08:46:54.924507Z","shell.execute_reply":"2021-01-13T08:46:54.923930Z"},"papermill":{"duration":0.169059,"end_time":"2021-01-13T08:46:54.924630","exception":false,"start_time":"2021-01-13T08:46:54.755571","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def prepare_dataloader(df, trn_idx, val_idx, data_root='../input/cassava-leaf-disease-classification/train_images/'):\n    \n    from catalyst.data.sampler import BalanceClassSampler\n    \n    train_ = df.loc[trn_idx,:].reset_index(drop=True)\n    valid_ = df.loc[val_idx,:].reset_index(drop=True)\n        \n    train_ds = TrainDataset(train, transform=get_train_transforms() , randagument= True)\n    valid_ds = TrainDataset(valid_,transform=get_valid_transforms(), randagument=False)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n        #sampler=BalanceClassSampler(labels=train_['label'].values, mode=\"downsampling\")\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n    return train_loader, val_loader\n\ndef train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None, schd_batch_update=False):\n    model.train()\n\n    t = time.time()\n    running_loss = None\n\n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n\n        #print(image_labels.shape, exam_label.shape)\n        with autocast():\n            image_preds = model(imgs)   #output = model(input)\n            #print(image_preds.shape, exam_pred.shape)\n\n            loss = loss_fn(image_preds, image_labels)\n            \n            scaler.scale(loss).backward()\n\n            if running_loss is None:\n                running_loss = loss.item()\n            else:\n                running_loss = running_loss * .99 + loss.item() * .01\n\n            if ((step + 1) %  CFG['accum_iter'] == 0) or ((step + 1) == len(train_loader)):\n                # may unscale_ here if desired (e.g., to allow clipping unscaled gradients)\n\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad() \n                \n                if scheduler is not None and schd_batch_update:\n                    scheduler.step()\n\n            if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(train_loader)):\n                description = f'epoch {epoch} loss: {running_loss:.4f}'\n                \n                pbar.set_description(description)\n                \n    if scheduler is not None and not schd_batch_update:\n        scheduler.step()\n        \ndef valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    sample_num = 0\n    image_preds_all = []\n    image_targets_all = []\n    \n    pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n        \n        image_preds = model(imgs)   #output = model(input)\n        #print(image_preds.shape, exam_pred.shape)\n        image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n        image_targets_all += [image_labels.detach().cpu().numpy()]\n        \n        loss = loss_fn(image_preds, image_labels)\n        \n        loss_sum += loss.item()*image_labels.shape[0]\n        sample_num += image_labels.shape[0]  \n\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(val_loader)):\n            description = f'epoch {epoch} loss: {loss_sum/sample_num:.4f}'\n            pbar.set_description(description)\n    \n    image_preds_all = np.concatenate(image_preds_all)\n    image_targets_all = np.concatenate(image_targets_all)\n    print('validation multi-class accuracy = {:.4f}'.format((image_preds_all==image_targets_all).mean()))\n    \n    if scheduler is not None:\n        if schd_loss_update:\n            scheduler.step(loss_sum/sample_num)\n        else:\n            scheduler.step()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:55.017863Z","iopub.status.busy":"2021-01-13T08:46:55.015839Z","iopub.status.idle":"2021-01-13T08:46:55.018599Z","shell.execute_reply":"2021-01-13T08:46:55.019090Z"},"papermill":{"duration":0.054741,"end_time":"2021-01-13T08:46:55.019223","exception":false,"start_time":"2021-01-13T08:46:54.964482","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class SymmetricCrossEntropy(nn.Module):\n\n    def __init__(self, alpha=0.1, beta=1.0, num_classes=5):\n        super(SymmetricCrossEntropy, self).__init__()\n        self.alpha = alpha\n        self.beta = beta\n        self.num_classes = num_classes\n\n    def forward(self, logits, targets, reduction='mean'):\n        onehot_targets = torch.eye(self.num_classes)[targets].cuda()\n        ce_loss = F.cross_entropy(logits, targets, reduction=reduction)\n        rce_loss = (-onehot_targets*logits.softmax(1).clamp(1e-7, 1.0).log()).sum(1)\n        if reduction == 'mean':\n            rce_loss = rce_loss.mean()\n        elif reduction == 'sum':\n            rce_loss = rce_loss.sum()\n        return self.alpha * ce_loss + self.beta * rce_loss","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:55.113058Z","iopub.status.busy":"2021-01-13T08:46:55.111574Z","iopub.status.idle":"2021-01-13T08:46:55.116396Z","shell.execute_reply":"2021-01-13T08:46:55.115839Z"},"papermill":{"duration":0.056439,"end_time":"2021-01-13T08:46:55.116501","exception":false,"start_time":"2021-01-13T08:46:55.060062","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from warmup_scheduler import GradualWarmupScheduler\n\nclass GradualWarmupSchedulerV2(GradualWarmupScheduler):\n    def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None):\n        super(GradualWarmupSchedulerV2, self).__init__(optimizer, multiplier, total_epoch, after_scheduler)\n    def get_lr(self):\n        if self.last_epoch > self.total_epoch:\n            if self.after_scheduler:\n                if not self.finished:\n                    self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs]\n                    self.finished = True\n                return self.after_scheduler.get_lr()\n            return [base_lr * self.multiplier for base_lr in self.base_lrs]\n        if self.multiplier == 1.0:\n            return [base_lr * (float(self.last_epoch) / self.total_epoch) for base_lr in self.base_lrs]\n        else:\n            return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in self.base_lrs]","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.040338,"end_time":"2021-01-13T08:46:55.199253","exception":false,"start_time":"2021-01-13T08:46:55.158915","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Main Loop"},{"metadata":{"execution":{"iopub.execute_input":"2021-01-13T08:46:55.297841Z","iopub.status.busy":"2021-01-13T08:46:55.297099Z","iopub.status.idle":"2021-01-13T12:56:33.500268Z","shell.execute_reply":"2021-01-13T12:56:33.499640Z"},"papermill":{"duration":14978.26061,"end_time":"2021-01-13T12:56:33.500386","exception":false,"start_time":"2021-01-13T08:46:55.239776","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])\n        \n    for fold in range(CFG['fold_num']):\n        if fold == CFG['Fold']:\n            print('Training with {} started'.format(fold))\n\n            with open(f'../input/making-fold/fold{fold}_train_idx.pickle', 'rb') as handle:\n                trn_idx = pickle.load(handle)\n            with open(f'../input/making-fold/fold{fold}_val_idx.pickle', 'rb') as handle:\n                val_idx = pickle.load(handle)\n\n            print(len(trn_idx), len(val_idx))\n            train_loader, val_loader = prepare_dataloader(train, trn_idx, val_idx, data_root='../input/cassava-leaf-disease-classification/train_images/')\n\n            device = torch.device(CFG['device'])\n\n            model = CassvaImgClassifier(pretrained =True).to(device)\n            scaler = GradScaler()   \n            optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'])\n            scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)      \n            loss_tr = SymmetricCrossEntropy().to(device)\n            loss_fn = SymmetricCrossEntropy().to(device)\n\n            for epoch in range(CFG['epochs']):\n                train_one_epoch(epoch, model, loss_tr, optimizer, train_loader, device, scheduler=scheduler, schd_batch_update=False)\n\n                with torch.no_grad():\n                    valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False)\n\n                torch.save(model.state_dict(),'{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch))\n\n            del model, optimizer, train_loader, val_loader, scaler, scheduler\n            torch.cuda.empty_cache()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}