{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":13836,"databundleVersionId":1718836,"sourceType":"competition"},{"sourceId":1490626,"sourceType":"datasetVersion","datasetId":875062},{"sourceId":1669366,"sourceType":"datasetVersion","datasetId":988673},{"sourceId":2660070,"sourceType":"datasetVersion","datasetId":686792}],"dockerImageVersionId":30034,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"package_paths = [\n    '../input/pytorch-image-models/pytorch-image-models-master', #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\n    '../input/image-fmix/FMix-master'\n]\nimport sys; \n\nfor pth in package_paths:\n    sys.path.append(pth)\n    \nfrom fmix import sample_mask, make_low_freq_image, binarise_mask","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:38:52.394144Z","iopub.execute_input":"2024-10-20T01:38:52.394488Z","iopub.status.idle":"2024-10-20T01:38:52.399385Z","shell.execute_reply.started":"2024-10-20T01:38:52.39446Z","shell.execute_reply":"2024-10-20T01:38:52.398522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install efficientnet_pytorch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:38:52.404692Z","iopub.execute_input":"2024-10-20T01:38:52.404958Z","iopub.status.idle":"2024-10-20T01:38:59.863915Z","shell.execute_reply.started":"2024-10-20T01:38:52.404933Z","shell.execute_reply":"2024-10-20T01:38:59.862946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\nmodel = EfficientNet.from_pretrained('efficientnet-b0')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:38:59.866711Z","iopub.execute_input":"2024-10-20T01:38:59.867144Z","iopub.status.idle":"2024-10-20T01:38:59.996217Z","shell.execute_reply.started":"2024-10-20T01:38:59.867099Z","shell.execute_reply":"2024-10-20T01:38:59.995345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\n\nimport timm\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\nfrom scipy.ndimage.interpolation import zoom","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:38:59.997559Z","iopub.execute_input":"2024-10-20T01:38:59.997807Z","iopub.status.idle":"2024-10-20T01:39:00.008033Z","shell.execute_reply.started":"2024-10-20T01:38:59.997783Z","shell.execute_reply":"2024-10-20T01:39:00.007185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 719,\n    'model_arch': 'tf_efficientnet_b4_ns',\n    'img_size': 512,\n    'epochs': 10,\n    'train_bs': 16,\n    'valid_bs': 32,\n    'T_0': 10,\n    'lr': 1e-4,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_workers': 4,\n    'accum_iter': 2, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0'\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:39:00.009354Z","iopub.execute_input":"2024-10-20T01:39:00.009727Z","iopub.status.idle":"2024-10-20T01:39:00.019806Z","shell.execute_reply.started":"2024-10-20T01:39:00.00969Z","shell.execute_reply":"2024-10-20T01:39:00.018948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:39:00.022986Z","iopub.execute_input":"2024-10-20T01:39:00.023351Z","iopub.status.idle":"2024-10-20T01:39:00.051771Z","shell.execute_reply.started":"2024-10-20T01:39:00.023324Z","shell.execute_reply":"2024-10-20T01:39:00.051079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.label.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:39:00.053132Z","iopub.execute_input":"2024-10-20T01:39:00.053402Z","iopub.status.idle":"2024-10-20T01:39:00.060833Z","shell.execute_reply.started":"2024-10-20T01:39:00.05337Z","shell.execute_reply":"2024-10-20T01:39:00.059904Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> We could do stratified validation split in each fold to make each fold's train and validation set looks like the whole train set in target distributions.","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:39:00.062098Z","iopub.execute_input":"2024-10-20T01:39:00.062363Z","iopub.status.idle":"2024-10-20T01:39:00.07533Z","shell.execute_reply.started":"2024-10-20T01:39:00.062337Z","shell.execute_reply":"2024-10-20T01:39:00.074424Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nplt.imshow(img)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:39:00.076342Z","iopub.execute_input":"2024-10-20T01:39:00.076645Z","iopub.status.idle":"2024-10-20T01:39:00.30136Z","shell.execute_reply.started":"2024-10-20T01:39:00.076613Z","shell.execute_reply":"2024-10-20T01:39:00.300653Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"def rand_bbox(size, lam):\n    W = size[0]\n    H = size[1]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\n\nclass CassavaDataset(Dataset):\n    def __init__(self, df, data_root, \n                 transforms=None, \n                 output_label=True, \n                 one_hot_label=False,\n                 do_fmix=False, \n                 fmix_params={\n                     'alpha': 1., \n                     'decay_power': 3., \n                     'shape': (CFG['img_size'], CFG['img_size']),\n                     'max_soft': True, \n                     'reformulate': False\n                 },\n                 do_cutmix=False,\n                 cutmix_params={\n                     'alpha': 1,\n                 }\n                ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.do_fmix = do_fmix\n        self.fmix_params = fmix_params\n        self.do_cutmix = do_cutmix\n        self.cutmix_params = cutmix_params\n        \n        self.output_label = output_label\n        self.one_hot_label = one_hot_label\n        \n        if output_label == True:\n            self.labels = self.df['label'].values\n            #print(self.labels)\n            \n            if one_hot_label is True:\n                self.labels = np.eye(self.df['label'].max()+1)[self.labels]\n                #print(self.labels)\n            \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.labels[index]\n          \n        img  = get_img(\"{}/{}\".format(self.data_root, self.df.loc[index]['image_id']))\n\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n        if self.do_fmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            with torch.no_grad():\n                #lam, mask = sample_mask(**self.fmix_params)\n                \n                lam = np.clip(np.random.beta(self.fmix_params['alpha'], self.fmix_params['alpha']),0.6,0.7)\n                \n                # Make mask, get mean / std\n                mask = make_low_freq_image(self.fmix_params['decay_power'], self.fmix_params['shape'])\n                mask = binarise_mask(mask, lam, self.fmix_params['shape'], self.fmix_params['max_soft'])\n    \n                fmix_ix = np.random.choice(self.df.index, size=1)[0]\n                fmix_img  = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[fmix_ix]['image_id']))\n\n                if self.transforms:\n                    fmix_img = self.transforms(image=fmix_img)['image']\n\n                mask_torch = torch.from_numpy(mask)\n                \n                # mix image\n                img = mask_torch*img+(1.-mask_torch)*fmix_img\n\n                #print(mask.shape)\n\n                #assert self.output_label==True and self.one_hot_label==True\n\n                # mix target\n                rate = mask.sum()/CFG['img_size']/CFG['img_size']\n                target = rate*target + (1.-rate)*self.labels[fmix_ix]\n                #print(target, mask, img)\n                #assert False\n        \n        if self.do_cutmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            #print(img.sum(), img.shape)\n            with torch.no_grad():\n                cmix_ix = np.random.choice(self.df.index, size=1)[0]\n                cmix_img  = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[cmix_ix]['image_id']))\n                if self.transforms:\n                    cmix_img = self.transforms(image=cmix_img)['image']\n                    \n                lam = np.clip(np.random.beta(self.cutmix_params['alpha'], self.cutmix_params['alpha']),0.3,0.4)\n                bbx1, bby1, bbx2, bby2 = rand_bbox((CFG['img_size'], CFG['img_size']), lam)\n\n                img[:, bbx1:bbx2, bby1:bby2] = cmix_img[:, bbx1:bbx2, bby1:bby2]\n\n                rate = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (CFG['img_size'] * CFG['img_size']))\n                target = rate*target + (1.-rate)*self.labels[cmix_ix]\n                \n            #print('-', img.sum())\n            #print(target)\n            #assert False\n                            \n        # do label smoothing\n        #print(type(img), type(target))\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:39:00.302469Z","iopub.execute_input":"2024-10-20T01:39:00.302762Z","iopub.status.idle":"2024-10-20T01:39:00.338571Z","shell.execute_reply.started":"2024-10-20T01:39:00.302734Z","shell.execute_reply":"2024-10-20T01:39:00.337676Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Define Train\\Validation Image Augmentations","metadata":{}},{"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:39:00.339595Z","iopub.execute_input":"2024-10-20T01:39:00.339944Z","iopub.status.idle":"2024-10-20T01:39:00.35465Z","shell.execute_reply.started":"2024-10-20T01:39:00.339901Z","shell.execute_reply":"2024-10-20T01:39:00.353875Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        '''\n        self.model.classifier = nn.Sequential(\n            nn.Dropout(0.3),\n            #nn.Linear(n_features, hidden_size,bias=True), nn.ELU(),\n            nn.Linear(n_features, n_class, bias=True)\n        )\n        '''\n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:39:00.355771Z","iopub.execute_input":"2024-10-20T01:39:00.356096Z","iopub.status.idle":"2024-10-20T01:39:00.367223Z","shell.execute_reply.started":"2024-10-20T01:39:00.356069Z","shell.execute_reply":"2024-10-20T01:39:00.366502Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training APIs","metadata":{}},{"cell_type":"code","source":"def prepare_dataloader(df, trn_idx, val_idx, data_root='../input/cassava-leaf-disease-classification/train_images/'):\n    \n    from catalyst.data.sampler import BalanceClassSampler\n    \n    train_ = df.loc[trn_idx,:].reset_index(drop=True)\n    valid_ = df.loc[val_idx,:].reset_index(drop=True)\n        \n    train_ds = CassavaDataset(train_, data_root, transforms=get_train_transforms(), output_label=True, one_hot_label=False, do_fmix=False, do_cutmix=False)\n    valid_ds = CassavaDataset(valid_, data_root, transforms=get_valid_transforms(), output_label=True)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n        #sampler=BalanceClassSampler(labels=train_['label'].values, mode=\"downsampling\")\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n    return train_loader, val_loader\n\ndef train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None, schd_batch_update=False):\n    model.train()\n\n    t = time.time()\n    running_loss = None\n\n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n\n        #print(image_labels.shape, exam_label.shape)\n        with autocast():\n            image_preds = model(imgs)   #output = model(input)\n            #print(image_preds.shape, exam_pred.shape)\n\n            loss = loss_fn(image_preds, image_labels)\n            \n            scaler.scale(loss).backward()\n\n            if running_loss is None:\n                running_loss = loss.item()\n            else:\n                running_loss = running_loss * .99 + loss.item() * .01\n\n            if ((step + 1) %  CFG['accum_iter'] == 0) or ((step + 1) == len(train_loader)):\n                # may unscale_ here if desired (e.g., to allow clipping unscaled gradients)\n\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad() \n                \n                if scheduler is not None and schd_batch_update:\n                    scheduler.step()\n\n            if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(train_loader)):\n                description = f'epoch {epoch} loss: {running_loss:.4f}'\n                \n                pbar.set_description(description)\n                \n    if scheduler is not None and not schd_batch_update:\n        scheduler.step()\n        \ndef valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    sample_num = 0\n    image_preds_all = []\n    image_targets_all = []\n    \n    pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n        \n        image_preds = model(imgs)   #output = model(input)\n        #print(image_preds.shape, exam_pred.shape)\n        image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n        image_targets_all += [image_labels.detach().cpu().numpy()]\n        \n        loss = loss_fn(image_preds, image_labels)\n        \n        loss_sum += loss.item()*image_labels.shape[0]\n        sample_num += image_labels.shape[0]  \n\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(val_loader)):\n            description = f'epoch {epoch} loss: {loss_sum/sample_num:.4f}'\n            pbar.set_description(description)\n    \n    image_preds_all = np.concatenate(image_preds_all)\n    image_targets_all = np.concatenate(image_targets_all)\n    print('validation multi-class accuracy = {:.4f}'.format((image_preds_all==image_targets_all).mean()))\n    \n    if scheduler is not None:\n        if schd_loss_update:\n            scheduler.step(loss_sum/sample_num)\n        else:\n            scheduler.step()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:39:00.368384Z","iopub.execute_input":"2024-10-20T01:39:00.368714Z","iopub.status.idle":"2024-10-20T01:39:00.400221Z","shell.execute_reply.started":"2024-10-20T01:39:00.368686Z","shell.execute_reply":"2024-10-20T01:39:00.39924Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# reference: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/173733\nclass MyCrossEntropyLoss(_WeightedLoss):\n    def __init__(self, weight=None, reduction='mean'):\n        super().__init__(weight=weight, reduction=reduction)\n        self.weight = weight\n        self.reduction = reduction\n\n    def forward(self, inputs, targets):\n        lsm = F.log_softmax(inputs, -1)\n\n        if self.weight is not None:\n            lsm = lsm * self.weight.unsqueeze(0)\n\n        loss = -(targets * lsm).sum(-1)\n\n        if  self.reduction == 'sum':\n            loss = loss.sum()\n        elif  self.reduction == 'mean':\n            loss = loss.mean()\n\n        return loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:39:00.401573Z","iopub.execute_input":"2024-10-20T01:39:00.401974Z","iopub.status.idle":"2024-10-20T01:39:00.412236Z","shell.execute_reply.started":"2024-10-20T01:39:00.401938Z","shell.execute_reply":"2024-10-20T01:39:00.411532Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Main Loop","metadata":{}},{"cell_type":"code","source":"if __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num'], shuffle=True, random_state=CFG['seed']).split(np.arange(train.shape[0]), train.label.values)\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        # we'll train fold 0 first\n        if fold > 0:\n            break \n\n        print('Training with {} started'.format(fold))\n\n        print(len(trn_idx), len(val_idx))\n        train_loader, val_loader = prepare_dataloader(train, trn_idx, val_idx, data_root='../input/cassava-leaf-disease-classification/train_images/')\n\n        device = torch.device(CFG['device'])\n        \n        model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique(), pretrained=True).to(device)\n        scaler = GradScaler()   \n        optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'])\n        #scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=CFG['epochs']-1)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)\n        #scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=25, \n        #                                                max_lr=CFG['lr'], epochs=CFG['epochs'], steps_per_epoch=len(train_loader))\n        \n        loss_tr = nn.CrossEntropyLoss().to(device) #MyCrossEntropyLoss().to(device)\n        loss_fn = nn.CrossEntropyLoss().to(device)\n        \n        for epoch in range(CFG['epochs']):\n            train_one_epoch(epoch, model, loss_tr, optimizer, train_loader, device, scheduler=scheduler, schd_batch_update=False)\n\n            with torch.no_grad():\n                valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False)\n\n            torch.save(model.state_dict(),'{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch))\n            \n        #torch.save(model.cnn_model.state_dict(),'{}/cnn_model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag']))\n        del model, optimizer, train_loader, val_loader, scaler, scheduler\n        torch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T01:39:00.413491Z","iopub.execute_input":"2024-10-20T01:39:00.413837Z","iopub.status.idle":"2024-10-20T03:49:03.477126Z","shell.execute_reply.started":"2024-10-20T01:39:00.413807Z","shell.execute_reply":"2024-10-20T03:49:03.476162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df[[\"image_id\",\"label\"]].to_csv(\"submission.csv\", index=False)\n!head submission.csv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-20T04:09:00.816599Z","iopub.execute_input":"2024-10-20T04:09:00.81696Z","iopub.status.idle":"2024-10-20T04:09:00.851761Z","shell.execute_reply.started":"2024-10-20T04:09:00.816923Z","shell.execute_reply":"2024-10-20T04:09:00.850396Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inferece part is here: https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-inference-tta","metadata":{}}]}