{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# 参考 https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-train-amp-aug  训练部分\n# 参考 https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-inference-tta  预测部分\n# 比赛 https://www.kaggle.com/c/cassava-leaf-disease-classification\n\n# 其他待模仿 https://www.kaggle.com/sachinprabhu/pytorch-resnet50-snapmix-train-pipeline\n# 其他待模仿 https://www.kaggle.com/ar2017/pytorch-resnet200d-train-cutmix-fmix\n\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = 'all'\n\n\nimport os\nimport sys\nimport gc\nimport math\nimport pickle\nimport random\nimport time\nimport psutil\nimport pytz\nfrom datetime import datetime\nfrom collections import defaultdict\nfrom contextlib import contextmanager\n\nimport warnings\nwarnings.filterwarnings('ignore')  # warnings.filterwarnings(action='once')\n\nfrom tqdm import tqdm, tqdm_notebook\n\nimport numpy as np\nimport pandas as pd\n_ = np.seterr(divide='ignore', invalid='ignore')\n\npd.set_option('display.max_columns', None)\n# pd.set_option('display.max_columns', 100)\npd.set_option('display.max_rows', None)\n# pd.set_option('display.max_rows', 100)\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.style as style\nstyle.use('fivethirtyeight')\nimport seaborn as sns\n\n# 直接在cell中显示图片，支持jpg、png、jpeg等格式，Image('./2.JPG')\nfrom IPython.display import Image  \n\nimport lightgbm as lgb\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import log_loss\n\nimport tensorflow as tf\nfrom tensorflow import keras\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\ndef show_process_mem_usage(info_str=''):    ## 显示当前进程占用内存大小\n    process = psutil.Process(os.getpid())\n    memory_usage = process.memory_info().rss\n    percent = psutil.virtual_memory().percent\n    \n    tz = pytz.timezone('Asia/Shanghai')\n    now = datetime.now(tz)\n    dt_str = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n    \n    if memory_usage >= 2.**30:\n        print(f'{info_str} current process memory usage: {memory_usage/2.**30:.3f} GB, percentage: {percent:.2f}% 【{dt_str}】')\n    elif memory_usage >= 2.**20:\n        print(f'{info_str} current process memory usage: {memory_usage/2.**20:.3f} MB, percentage: {percent:.2f}% 【{dt_str}】')\n    elif memory_usage >= 2.**10:\n        print(f'{info_str} current process memory usage: {memory_usage/2.**10:.3f} KB, percentage: {percent:.2f}% 【{dt_str}】')\n    else:\n        print(f'{info_str} current process memory usage: {memory_usage} B, percentage: {percent:.2f}% 【{dt_str}】')\n\ndef logging(*info, file_name='./running_log.txt'):\n    log_info = ' '.join([str(s) for s in info])\n    with open(file_name, 'a') as f:\n        f.write(log_info + '\\n')\n\n@contextmanager\ndef trace(trace_msg):    ## 追踪内存变化和运行时间\n    t0 = time.time()\n    p = psutil.Process(os.getpid())\n    m0 = p.memory_info()[0] / 2. ** 30\n    yield\n    m1 = p.memory_info()[0] / 2. ** 30\n    delta = m1 - m0\n    sign = '+' if delta >= 0 else '-'\n    delta = math.fabs(delta)\n    trace_msg = str(trace_msg)\n    \n    tz = pytz.timezone('Asia/Shanghai')\n    now = datetime.now(tz)\n    dt_str = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n    print(f\"[{m1:.3f}GB({sign}{delta:.3f}GB):{time.time() - t0:.3f}sec] {trace_msg} 【{dt_str}】\", file=sys.stdout)\n    \ndef seed_all(random_seed=42):\n    os.environ['PYTHONHASHSEED'] = str(random_seed)\n    random.seed(random_seed)\n    np.random.seed(random_seed)\n    tf.random.set_seed(random_seed)\n    torch.manual_seed(random_seed)\n    torch.cuda.manual_seed(random_seed)\n    torch.backends.cudnn.deterministic = True\n\ndef keepbusy(num=10000):\n    start_t = time.time()\n    for i in range(num):\n        ftpt(f'i: {i}, taken time: {time.time() - start_t:.7f}')\n        time.sleep(60)\n\ndef ftpt(msg = 'having run this cell'):  # foot_print   \n    tz = pytz.timezone('Asia/Shanghai')\n    now = datetime.now(tz)\n    dt_string = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n    print(f'{dt_string}: {msg}')\n    \nRANDOM_SEED = 53113\nseed_all(RANDOM_SEED)\n\nglobal_start_t = time.time()\n\nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def run_one_time(cnt_lst):\n#     arr_idx_lst = [i for i in range(len(cnt_lst)) if cnt_lst[i]!=0]\n#     random.shuffle(arr_idx_lst)\n#     for i in range(int(len(arr_idx_lst)*0.5)):\n#         a, b = arr_idx_lst[2*i], arr_idx_lst[2*i+1]\n#         if random.randint(0, 1)==0:\n#             a, b = b, a\n#         if cnt_lst[a]>0 and cnt_lst[b]>0:\n#             cnt_lst[a] = cnt_lst[a]-1\n#             cnt_lst[b] = cnt_lst[b]+1\n#     return cnt_lst\n        \n# NUM = 100\n# cnt_lst = [100]*100\n\n# for i in range(5000000000):\n#     zero_num = len([i for i in cnt_lst if i==0])\n#     if i%1000==0:\n#         print('i: ', i, cnt_lst, 'zero_num: ', zero_num)\n#     if zero_num==(NUM-1):\n#         print('i: ', i, cnt_lst, 'zero_num: ', zero_num)\n#         print('final i: ', i)\n#         break\n#     cnt_lst = run_one_time(cnt_lst)\n    \n##############################################################################\n    \n# cnt_lst\n\n# i:  25675000 [0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] zero_num:  98\n# final i:  25675301\n\n# i:  48013164 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] zero_num:  99\n# final i:  48013164\n\nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"package_paths = [\n    '../input/pytorch-image-models/pytorch-image-models-master/', \n    #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\n    '../input/image-fmix/FMix-master'\n]\nimport sys\n\nfor pth in package_paths:\n    sys.path.append(pth)\n    \nfrom fmix import sample_mask, make_low_freq_image, binarise_mask\n\nfrom glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\n\nimport timm\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\n#from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom\n\nCFG = {\n    'fold_num': 5,\n    'seed': 719,\n    'model_arch': 'tf_efficientnet_b4_ns',\n    'img_size': 512,\n    'epochs': 10,\n    'train_bs': 16,\n    'valid_bs': 32,\n    'T_0': 10,\n    'lr': 1e-4,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_workers': 4,\n    'accum_iter': 2, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1, \n    'device': 'cuda:0' if torch.cuda.is_available() else 'cpu',\n}\n\nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.shape\ntrain.head()\ntrain.label.value_counts()\n\n# train = train.sample(n=500)\n# train = train.reset_index(drop=True)\n# train.shape\n\nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n#     print('im_bgr.shape: ', im_bgr.shape)\n    im_rgb = im_bgr[:, :, ::-1]\n    return im_rgb\n\nImage('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\n# Image('../input/cassava-leaf-disease-classification/test_images/2216849948.jpg')\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rand_bbox(size, lam):\n    W = size[0]\n    H = size[1]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n    \n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n    \n    bbx1 = np.clip(cx-cut_w//2, 0, W)\n    bby1 = np.clip(cy-cut_h//2, 0, H)\n    bbx2 = np.clip(cx+cut_w//2, 0, W)\n    bby2 = np.clip(cy+cut_h//2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\n\nclass CassavaDataset(Dataset):\n    def __init__(self, df, data_root, \n                 transforms=None, \n                 output_label=True,\n                 one_hot_label=False, \n                 do_fmix=False,\n                 fmix_params={\n                     'alpha': 1.,\n                     'decay_power': 3.,\n                     'shape': (CFG['img_size'], CFG['img_size']),\n                     'max_soft': True,\n                     'reformulate': False,\n                 }, \n                 do_cutmix=False,\n                 cutmix_params={\n                     'alpha': 1.,\n                 }):\n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.do_fmix = do_fmix\n        self.fmix_params = fmix_params\n        self.do_cutmix = do_cutmix\n        self.cutmix_params = cutmix_params\n        \n        self.output_label = output_label\n        self.one_hot_label = one_hot_label\n        \n        if output_label==True:\n            self.labels = self.df['label'].values\n            if one_hot_label:\n                self.labels = np.eye(self.df['label'].max()+1)[self.labels]\n                \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        if self.output_label:\n            target = self.labels[index]\n            \n        img = get_img('{}/{}'.format(self.data_root, self.df.loc[index]['image_id']))\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        if self.do_fmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            with torch.no_grad():\n                lam = np.clip(np.random.beta(self.fmix_param['alpha'], self.fmix_params['alpha'], 0.6, 0.7))\n                \n                # Make mask, get mean / std\n                mask = make_low_freq_image(self.fmix_params['decay_power'], self.fmix_params['shape'])\n                mask = binarise_mask(mask, lam, self.fmix_params['shape'], self.fmix_params['max_soft'])\n                \n                fmix_ix = np.random.choice(self.df.index, size=1)[0]\n                fmix_img = get_img('{}/{}'.format(self.data_root, self.df.iloc[fmix_ix]['image_id']))\n                \n                if self.transforms:\n                    fmix_img = self.transforms(image=fmix_img)['image']\n                    \n                mask_torch = torch.from_numpy(mask)\n                img = mask_torch*img + (1.-mask_torch)*fmix_img\n                rate = mask.sum()/CFG['img_size']/CFG['img_size']\n                target = rate*target + (1.-rate)*self.labels[fmix_ix]\n                \n        if self.do_cutmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            with torch.no_grad():\n                cmix_ix = np.random.choice(self.df.index, size=1)[0]\n                cmix_img = get_img('{}/{}'.format(self.data_root, self.df.iloc[cmix_ix]['image_id']))\n                if self.transforms:\n                    cmix_img = self.transforms(image=cmix_img)['image']\n                    \n                lam = np.clip(np.random.beta(self.cutmix_params['alpha'], self.cutmix_params['alpha'], 0.3, 0.4))\n                bbx1, bby1, bbx2, bby2 = rand_bbox((CFG['image_size'], CFG['image_size']), lam)\n                img[:, bbx1:bbx2, bby1:bby2] = cmix_img[:, bbx1:bbx2, bby1:bby2]\n                rate = 1 - (bbx2-bbx2)*(bby2-bby1)/(CFG['img_size']*CFG['img_size'])\n                target = rate*target + (1.-rate)*self.labels[cmix_ix]\n                \n        if self.output_label:\n            return img, target\n        else:\n            return img\n                \nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n        RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n        Transpose(p=0.5),\n        HorizontalFlip(p=0.5),\n        VerticalFlip(p=0.5),\n        ShiftScaleRotate(p=0.5),\n        HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n        RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        CoarseDropout(p=0.5),\n        Cutout(p=0.5),\n        ToTensorV2(p=1.0),\n    ], p=1.0)\n\ndef get_valid_transforms():\n    return Compose([\n        CenterCrop(CFG['img_size'], CFG['img_size'], p=1.0),\n        Resize(CFG['img_size'], CFG['img_size']),\n        Normalize(mean=[0.485, 0.556, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        ToTensorV2(p=1.0),\n    ], p=1.0)\n\nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \ndef prepare_dataloader(df, trn_idx, val_idx, data_root='../input/cassava-leaf-disease-classification/train_images/'):\n    from catalyst.data.sampler import BalanceClassSampler\n    train_ = df.loc[trn_idx, :].reset_index(drop=True)\n    valid_ = df.loc[val_idx, :].reset_index(drop=True)\n    train_ds = CassavaDataset(train_, data_root, transforms=get_train_transforms(),\n                              output_label=True, one_hot_label=False, do_fmix=False, \n                              do_cutmix=False)\n    valid_ds = CassavaDataset(valid_, data_root, transforms=get_valid_transforms(),\n                              output_label=True)\n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,\n        num_workers=CFG['num_workers'],\n#         sampler=BalanceClassSampler(labels=train_['label'].values, mode='downsampling')\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds,\n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n    return train_loader, val_loader\n\ndef train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, \n                    scheduler=None, schd_batch_update=False):\n    model.train()\n    \n    t = time.time()\n    running_loss = None\n    \n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n        \n        with autocast():\n            image_preds = model(imgs)\n            loss = loss_fn(image_preds, image_labels)\n            scaler.scale(loss).backward()\n            \n            if running_loss is None:\n                running_loss = loss.item()\n            else:\n                running_loss = running_loss * .99 + loss.item() * .01\n                \n            if ((step+1)%CFG['accum_iter']==0) or ((step+1)==len(train_loader)):\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()\n                \n                if scheduler is not None and schd_batch_update:\n                    description = f'epoch {epoch} loss: {running_loss:.4f}'\n                    pbar.set_description(description)\n    if scheduler is not None and not schd_batch_update:\n        scheduler.step()\n        \ndef valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):\n    print('in valid_one_epoch()')\n    model.eval()\n    \n    t = time.time()\n    loss_sum = 0\n    sample_num = 0\n    image_preds_prob_all = []\n    image_preds_all = []\n    image_targets_all = []\n    \n    pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n        \n        image_preds = model(imgs)\n        image_preds_prob_all += [image_preds.detach().cpu().numpy()]\n        image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n        image_targets_all += [image_labels.detach().cpu().numpy()]\n        loss = loss_fn(image_preds, image_labels)\n        loss_sum += loss.item()*image_labels.shape[0]\n        sample_num += image_labels.shape[0]\n        \n        if ((step+1)%CFG['verbose_step']==0) or ((step+1)==len(val_loader)):\n            description = f'epoch {epoch} loss: {loss_sum/sample_num:.4f}'\n            pbar.set_description(description)\n            \n    image_preds_prob_all = np.concatenate(image_preds_prob_all)\n    image_preds_all = np.concatenate(image_preds_all)\n    image_targets_all = np.concatenate(image_targets_all)\n    print('validation multi-class log_loss = {:.4f}'.format(log_loss(image_targets_all, image_preds_prob_all)))\n    print('validation multi-class accuracy = {:.4f}'.format((image_preds_all==image_targets_all).mean()))\n    \n    if scheduler is not None:\n        if schd_loss_update:\n            scheduler.step(loss_sum/sample_num)\n        else:\n            scheduler.step()\n           \n        \nclass MyCrossEntropyLoss(_WeightedLoss):\n    def __init__(self, weight=None, reduction='mean'):\n        super().__init__(weight=weight, reduction=reduction)\n        self.weight = weight\n        self.reduction = reduction\n        \n    def forward(self, inputs, targets):\n        lsm = F.log_softmax(inputs, -1)\n        \n        if self.weight is not None:\n            lsm = lsm * self.weight.unsqueeze(0)\n            \n        loss = -(targets * lsm).sum(-1)\n        if self.reduction == 'sum':\n            loss = loss.sum()\n        elif self.reduction == 'mean':\n            loss = loss.mean()\n            \n        return loss\n    \nftpt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__=='__main__':\n    seed_everything(CFG['seed'])\n    folds = StratifiedKFold(n_splits=CFG['fold_num'], shuffle=True, random_state=CFG['seed']).split(\n        np.arange(train.shape[0]), train.label.values)\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        fold_start_t = time.time()\n        print('in loop, fold: ', fold)\n        # we'll train fold 0 and fold 1 first\n        if fold > 1:\n            break\n            \n        print(f'Training with {fold} started')\n        print(f'len(trn_idx): {len(trn_idx)} len(val_idx): {len(val_idx)}')\n        train_loader, val_loader = prepare_dataloader(train, trn_idx, val_idx, \n                                                      data_root='../input/cassava-leaf-disease-classification/train_images/')\n        device = torch.device(CFG['device'])\n        model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique(), pretrained=True).to(device)\n        scaler = GradScaler()\n        optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'])\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)\n        \n        loss_tr = nn.CrossEntropyLoss().to(device)\n        loss_fn = nn.CrossEntropyLoss().to(device)\n        \n        for epoch in range(CFG['epochs']):\n            train_one_epoch(epoch, model, loss_tr, optimizer, train_loader, device, scheduler=scheduler, schd_batch_update=False)\n            \n            with torch.no_grad():\n                valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False)\n                \n            torch.save(model.state_dict(), '{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch))\n            \n        del model, optimizer, train_loader, val_loader, scaler, scheduler\n        torch.cuda.empty_cache()\n        fold_start_t = time.time()\n        print(f'fold: {fold}, cost time: {time.time()-fold_start_t:.5f}')\n        \ntest = pd.DataFrame()\ntest['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\ntest['label'] = 1\nprint('test.shape: ', test.shape)\ntest.to_csv('submission.csv', index=False)\n\nprint('this run total cost time: ', time.time() - global_start_t)\n\nftpt()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}