{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"pip install torchsummary","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.execute_input":"2021-01-11T21:03:50.779883Z","iopub.status.busy":"2021-01-11T21:03:50.779051Z","iopub.status.idle":"2021-01-11T21:04:00.716335Z","shell.execute_reply":"2021-01-11T21:04:00.715738Z"},"papermill":{"duration":9.953608,"end_time":"2021-01-11T21:04:00.716523","exception":false,"start_time":"2021-01-11T21:03:50.762915","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"pip install timm","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2021-01-11T21:04:00.75867Z","iopub.status.busy":"2021-01-11T21:04:00.755937Z","iopub.status.idle":"2021-01-11T21:04:04.596406Z","shell.execute_reply":"2021-01-11T21:04:04.595238Z"},"papermill":{"duration":3.86643,"end_time":"2021-01-11T21:04:04.596555","exception":false,"start_time":"2021-01-11T21:04:00.730125","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport cv2\nimport random\nimport os\nfrom tqdm import tqdm\nimport time\n\n\nimport imgaug as ia\nfrom imgaug import augmenters as iaa\n\n\nimport glob\nfrom PIL import Image\n\n\nimport torch\nimport torchvision.models as models\nimport torch.nn as nn\nimport torch.optim.lr_scheduler as lr_scheduler\nfrom torch.optim import Adam, SGD\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\nfrom torchsummary import summary\nimport torchvision\nimport torchvision.transforms as transforms\nfrom albumentations.pytorch import ToTensorV2\nimport timm\n\n\nfrom sklearn.metrics import accuracy_score,f1_score,precision_score,recall_score,confusion_matrix\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nfrom sklearn.utils.multiclass import unique_labels\n\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE,\n    RandomRotate90,Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion,\n    HueSaturationValue,IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine,\n    RandomResizedCrop,IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout,\n    CoarseDropout,ShiftScaleRotate, CenterCrop, Resize\n                            )\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T21:04:04.631317Z","iopub.status.busy":"2021-01-11T21:04:04.630628Z","iopub.status.idle":"2021-01-11T21:04:04.829144Z","shell.execute_reply":"2021-01-11T21:04:04.828006Z"},"papermill":{"duration":0.219198,"end_time":"2021-01-11T21:04:04.829274","exception":false,"start_time":"2021-01-11T21:04:04.610076","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.sample(5)\nsns.countplot(train.label, edgecolor = 'black',palette = sns.color_palette(\"viridis\", 5))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T21:04:04.8721Z","iopub.status.busy":"2021-01-11T21:04:04.86992Z","iopub.status.idle":"2021-01-11T21:04:04.873032Z","shell.execute_reply":"2021-01-11T21:04:04.873628Z"},"papermill":{"duration":0.029608,"end_time":"2021-01-11T21:04:04.873757","exception":false,"start_time":"2021-01-11T21:04:04.844149","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# ====================================================\n# -----CFG-------------------------------------------\n# ====================================================\nclass CFG:\n    seed=42\n    train_path='../input/cassava-leaf-disease-classification/train_images/'\n    test_path='../input/cassava-leaf-disease-classification/test_images/'\n    img_size=475 #256\n    target_col='label'\n    target_size=5\n    epochs=8\n    print_freq=100\n    n_fold=5\n    trn_fold=[0, 1, 2, 3, 4]\n    num_workers=4\n    batch_size=16\n    model_name='tf_efficientnet_b5_ns' # #  #resnext50_32x4d #tf_efficientnet_b4_ns googlenet \n## resnet18  googlenet 1-resnext50_32x4d 2-wide_resnet50_2\n    scheduler='CosineAnnealingWarmRestarts' # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts']    \n    #factor=0.2 # ReduceLROnPlateau\n    #patience=4 # ReduceLROnPlateau\n    #eps=1e-6 # ReduceLROnPlateau\n    #T_max=10 # CosineAnnealingLR\n    T_0=10 # CosineAnnealingWarmRestarts\n    lr=1e-4\n    min_lr=1e-6\n    weight_decay=1e-6\n    gradient_accumulation_steps=1\n    max_grad_norm=1000\n    train=True\n    inference=False\n    accum_iter= 2\n    verbose_step=1\n    debug=False\n    apex=False\n    \nif CFG.debug:\n    CFG.epochs = 3\n    train = train.sample(n=500, random_state=CFG.seed).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T21:04:05.012714Z","iopub.status.busy":"2021-01-11T21:04:05.011578Z","iopub.status.idle":"2021-01-11T21:04:05.315936Z","shell.execute_reply":"2021-01-11T21:04:05.31649Z"},"papermill":{"duration":0.427827,"end_time":"2021-01-11T21:04:05.316662","exception":false,"start_time":"2021-01-11T21:04:04.888835","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# ====================================================\n# -----Helper Functions-------------------------------\n# ====================================================\ndef performance_matrix(true,pred):\n    precision = precision_score(true,pred,average='macro')\n    recall = recall_score(true,pred,average='macro')\n    accuracy = accuracy_score(true,pred)\n    f1_sco = f1_score(true,pred,average='macro')\n    print('Confusion Matrix:\\n',confusion_matrix(true, pred))\n    print('Precision: {:.4f} Recall: {:.4f}, Accuracy: {:.4f}: ,f1_score: {:.4f}'.format(precision,recall,accuracy,f1_sco))\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    return im_rgb\n\ndef imshow(inp, title=None):\n    inp = inp.numpy().transpose((1, 2, 0))\n    mean = np.array([0.485, 0.456, 0.406])\n    std = np.array([0.229, 0.224, 0.225])\n    inp = std * inp + mean\n    inp = np.clip(inp, 0, 1)\n    plt.imshow(inp)\n    if title is not None:\n        plt.title(title)\n    plt.pause(0.001)  # pause a bit so that plots are updated\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/999998473.jpg')\nplt.imshow(img)\nplt.show()\n\n\ndef get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T21:04:05.370658Z","iopub.status.busy":"2021-01-11T21:04:05.365028Z","iopub.status.idle":"2021-01-11T21:04:05.841375Z","shell.execute_reply":"2021-01-11T21:04:05.841904Z"},"papermill":{"duration":0.505696,"end_time":"2021-01-11T21:04:05.842038","exception":false,"start_time":"2021-01-11T21:04:05.336342","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"sample = train.head(4)\nplt.figure(figsize=(15, 5))\nfor ind, (image_id, label) in enumerate(zip(sample.image_id, sample.label)):\n    plt.subplot(1, 4, ind + 1)\n    img = cv2.imread(os.path.join(CFG.train_path, image_id))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    plt.imshow(img)\n    plt.axis(\"off\")\n    \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T21:04:05.913997Z","iopub.status.busy":"2021-01-11T21:04:05.913058Z","iopub.status.idle":"2021-01-11T21:04:05.918093Z","shell.execute_reply":"2021-01-11T21:04:05.917235Z"},"papermill":{"duration":0.048017,"end_time":"2021-01-11T21:04:05.918216","exception":false,"start_time":"2021-01-11T21:04:05.870199","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# ====================================================\n# -----Image Augmentations---------------------------\n# ====================================================\ndef get_transforms(*, data):\n    \n    if data == 'train':\n        return Compose([\n            #Resize(CFG.img_size, CFG.img_size),\n            RandomResizedCrop(CFG.img_size, CFG.img_size),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2,sat_shift_limit=0.2,val_shift_limit=0.2,p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1),contrast_limit=(-0.1, 0.1),p=0.5),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n                max_pixel_value=255.0,\n                p=1.0\n            ),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(),\n        ])\n\n    elif data == 'valid':\n        return Compose([\n            CenterCrop(CFG.img_size, CFG.img_size, p=1.),\n            Resize(CFG.img_size, CFG.img_size),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T21:04:05.98736Z","iopub.status.busy":"2021-01-11T21:04:05.984014Z","iopub.status.idle":"2021-01-11T21:04:05.990094Z","shell.execute_reply":"2021-01-11T21:04:05.989591Z"},"papermill":{"duration":0.040689,"end_time":"2021-01-11T21:04:05.990223","exception":false,"start_time":"2021-01-11T21:04:05.949534","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, df, data_root, transforms=None, output_label=True):\n        super().__init__()\n        self.df = df.reset_index(drop=True).copy() # data\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n \n    def __len__(self):\n        return self.df.shape[0] # or len(self.df)\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        \n        img  = get_img(path)\n\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n           \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T21:04:06.05601Z","iopub.status.busy":"2021-01-11T21:04:06.055138Z","iopub.status.idle":"2021-01-11T21:04:06.059399Z","shell.execute_reply":"2021-01-11T21:04:06.058884Z"},"papermill":{"duration":0.042159,"end_time":"2021-01-11T21:04:06.059515","exception":false,"start_time":"2021-01-11T21:04:06.017356","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# reference: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/173733\nclass MyCrossEntropyLoss(_WeightedLoss):\n    def __init__(self, weight=None, reduction='mean'):\n        super().__init__(weight=weight, reduction=reduction)\n        self.weight = weight\n        self.reduction = reduction\n\n    def forward(self, inputs, targets):\n        lsm = F.log_softmax(inputs, -1)\n\n        if self.weight is not None:\n            lsm = lsm * self.weight.unsqueeze(0)\n\n        loss = -(targets * lsm).sum(-1)\n\n        if  self.reduction == 'sum':\n            loss = loss.sum()\n        elif  self.reduction == 'mean':\n            loss = loss.mean()\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassvaImgClassifierx(nn.Module): #resnext50_32x4d\n    def __init__(self, model_name, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T21:04:06.121966Z","iopub.status.busy":"2021-01-11T21:04:06.121133Z","iopub.status.idle":"2021-01-11T21:04:06.124952Z","shell.execute_reply":"2021-01-11T21:04:06.125464Z"},"papermill":{"duration":0.038352,"end_time":"2021-01-11T21:04:06.125599","exception":false,"start_time":"2021-01-11T21:04:06.087247","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module): #Eff\n    def __init__(self, model_name, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \n    \n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T21:04:06.206182Z","iopub.status.busy":"2021-01-11T21:04:06.185078Z","iopub.status.idle":"2021-01-11T21:04:06.217051Z","shell.execute_reply":"2021-01-11T21:04:06.21654Z"},"papermill":{"duration":0.065818,"end_time":"2021-01-11T21:04:06.217157","exception":false,"start_time":"2021-01-11T21:04:06.151339","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def Get_Dataloader(df, trn_idx, val_idx, data_root=CFG.train_path):\n    \n    from catalyst.data.sampler import BalanceClassSampler\n    \n    train_ = df.loc[trn_idx,:].reset_index(drop=True)\n    valid_ = df.loc[val_idx,:].reset_index(drop=True)\n\n    train_ds = CassavaDataset(train_, data_root, transforms=get_transforms(data='train'), output_label=True)\n    valid_ds = CassavaDataset(valid_, data_root, transforms=get_transforms(data='valid'), output_label=True)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG.batch_size,\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG.batch_size,\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG.batch_size,\n        num_workers=CFG.batch_size,\n        shuffle=False,\n        pin_memory=False,\n    )\n    return train_loader, val_loader\n\ndef train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None, schd_batch_update=False):\n    model.train()\n\n    t = time.time()\n    loss_sum = 0#1\n    sample_num = 0#2\n    image_preds_all = []#3\n    image_targets_all = []#4\n    \n    running_loss = None\n\n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n\n        #print(image_labels.shape, exam_label.shape)\n        with autocast():\n            image_preds = model(imgs)\n            #print(image_preds.shape, exam_pred.shape)\n            image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n            image_targets_all += [image_labels.detach().cpu().numpy()]\n        \n            loss = loss_fn(image_preds, image_labels)\n            \n            scaler.scale(loss).backward()\n           \n            running_loss = loss.item()\n\n            loss_sum += loss.item()*image_labels.shape[0]#1\n            sample_num += image_labels.shape[0]#2\n            loss_avg=loss_sum/sample_num#3\n            if ((step + 1) %  CFG.accum_iter == 0) or ((step + 1) == len(train_loader)):\n                # may unscale_ here if desired (e.g., to allow clipping unscaled gradients)\n\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad() \n                \n                if scheduler is not None and schd_batch_update:\n                    scheduler.step()\n\n            if ((step + 1) % CFG.verbose_step == 0) or ((step + 1) == len(train_loader)):\n                description = f'epoch {epoch} loss: {running_loss:.4f}'\n                pbar.set_description(description)\n                \n    image_preds_all = np.concatenate(image_preds_all)#\n    image_targets_all = np.concatenate(image_targets_all)#\n    score=(image_preds_all==image_targets_all).mean() #\n    \n    if scheduler is not None and not schd_batch_update:\n        scheduler.step()\n    return score,loss_avg\n        \ndef valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    sample_num = 0\n    image_preds_all = []\n    image_targets_all = []\n    \n    pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n        \n        image_preds = model(imgs)  \n        #print(image_preds.shape, exam_pred.shape)\n        image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n        image_targets_all += [image_labels.detach().cpu().numpy()]\n        \n        loss = loss_fn(image_preds, image_labels)\n        \n        loss_sum += loss.item()*image_labels.shape[0]\n        sample_num += image_labels.shape[0]  \n        loss_avg=loss_sum/sample_num\n        if ((step + 1) % CFG.verbose_step == 0) or ((step + 1) == len(val_loader)):\n            description = f'epoch {epoch} loss: {loss_avg:.4f}'\n            pbar.set_description(description)\n    \n    image_preds_all = np.concatenate(image_preds_all)\n    image_targets_all = np.concatenate(image_targets_all)\n    score=(image_preds_all==image_targets_all).mean()\n    performance_matrix(image_targets_all, image_preds_all)\n    if scheduler is not None:\n        if schd_loss_update:\n            scheduler.step(loss_sum/sample_num)\n        else:\n            scheduler.step()\n            \n    return score,loss_avg\n            \ndef Get_scheduler(optimizer):\n    if CFG.scheduler=='ReduceLROnPlateau':\n        scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=CFG.factor, patience=CFG.patience, verbose=True, eps=CFG.eps)\n    elif CFG.scheduler=='CosineAnnealingLR':\n        scheduler = CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr, last_epoch=-1)\n    elif CFG.scheduler=='CosineAnnealingWarmRestarts':\n        scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr, last_epoch=-1)\n    return scheduler","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T21:04:06.289253Z","iopub.status.busy":"2021-01-11T21:04:06.288287Z","iopub.status.idle":"2021-01-12T04:22:03.501759Z","shell.execute_reply":"2021-01-12T04:22:02.980796Z"},"papermill":{"duration":26277.256807,"end_time":"2021-01-12T04:22:03.501917","exception":false,"start_time":"2021-01-11T21:04:06.24511","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n  \n    seed_torch(seed=CFG.seed)\n    # ====================================================\n    # -----Train loop-------------------------------------\n    # ====================================================\n    folds = StratifiedKFold(n_splits=CFG.n_fold,\n                            shuffle=True,\n                            random_state=CFG.seed).split(np.arange(train.shape[0]),train.label.values)\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):       \n        # we'll train fold 0 first\n        if fold > 2:\n            break \n\n        print(f\"========== fold: {fold} training ==========\")\n        print(len(trn_idx), len(val_idx))\n        \n        # ====================================================\n        # -----loader-----------------------------------------\n        # ====================================================\n        train_loader, val_loader = Get_Dataloader(train, trn_idx, val_idx)\n        \n        # ====================================================\n        # model & optimizer\n        # ====================================================\n        model = CassvaImgClassifier(CFG.model_name, train.label.nunique(), pretrained=True)\n        model.to(device)\n        #---------------------\n        #model = models.wide_resnet50_2(pretrained=True) # resnet18  googlenet 1-resnext50_32x4d 2-wide_resnet50_2\n        #num_ftrs = model.fc.in_features\n        #model.fc = nn.Linear(num_ftrs, 5)\n        #model = model.to(device)\n        summary(model, (3, CFG.img_size, CFG.img_size))\n        #--------------------\n        scaler = GradScaler() \n        optimizer = Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)\n        scheduler = Get_scheduler(optimizer)\n   \n\n        loss_tr = nn.CrossEntropyLoss().to(device) \n        loss_fn = nn.CrossEntropyLoss().to(device)\n        \n        best_score = 0.\n        dresults = pd.DataFrame(columns=['Epoch','Train_Acc', 'Train_loss','Val_Acc', 'Val_loss'])\n        for epoch in range(CFG.epochs):\n            \n            train_acc,train_loss=train_one_epoch(epoch, model, loss_tr, optimizer, train_loader, device, scheduler=scheduler, schd_batch_update=False)\n            with torch.no_grad():\n                val_acc,val_loss= valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False)\n                #----------------------------------------------\n                dresults = dresults.append({'Epoch': epoch+1,'Train_Acc': train_acc, 'Train_loss':train_loss,'Val_Acc': val_acc, 'Val_loss':val_loss},ignore_index=True)\n                #----------------------------------------------\n                print('Epoch= {}'.format(epoch+1))\n                print('Train_Acc  = {:.5f},  Train_loss={:.5f}'.format(train_acc,train_loss))\n                print('Val_Acc  = {:.5f},    Val_loss={:.5f}'.format(val_acc,val_loss))\n                #----------------------------------------------\n                if val_acc > 0.89: #best_score\n                    best_score = val_acc\n                    torch.save(model.state_dict(),'{}_fold_best_{}_{}'.format(CFG.model_name,fold,epoch))\n                #----------------------------------------------\n        dresults.to_csv('dresults.csv', index=False)         \n        del model, optimizer, train_loader, val_loader, scaler, scheduler\n        torch.cuda.empty_cache()\n\n\n             \n        ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}