{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Training pipline with ViT using Pytorch \nThis is a pipeline on training with ViT using PyTorch. If anyone finds any improvement, please comment in the notebook!\n\nReferences:\n\n[paper](https://arxiv.org/abs/2010.11929)\n\n[Github](https://github.com/rwightman/pytorch-image-models)","metadata":{"editable":false}},{"cell_type":"markdown","source":"# Install Timm","metadata":{"editable":false}},{"cell_type":"code","source":"# !/opt/conda/bin/python3.7 -m pip install --upgrade pip\n# !c ../input/timm031py3noneanywhl/timm-0.3.1-py3-none-any.whl","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:50.872053Z","iopub.execute_input":"2022-12-26T11:51:50.872576Z","iopub.status.idle":"2022-12-26T11:51:50.87641Z","shell.execute_reply.started":"2022-12-26T11:51:50.872541Z","shell.execute_reply":"2022-12-26T11:51:50.875141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/pytorchimagemodel/pytorch-image-models-main')\nimport timm","metadata":{"execution":{"iopub.status.busy":"2022-12-26T11:51:50.878277Z","iopub.execute_input":"2022-12-26T11:51:50.878894Z","iopub.status.idle":"2022-12-26T11:51:52.777604Z","shell.execute_reply.started":"2022-12-26T11:51:50.878857Z","shell.execute_reply":"2022-12-26T11:51:52.77654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Print GPU info","metadata":{"editable":false}},{"cell_type":"code","source":"# gpu_info = !nvidia-smi\n# gpu_info = '\\n'.join(gpu_info)\n# if gpu_info.find('failed') >= 0:\n#     print('Select the Runtime > \"Change runtime type\" menu to enable a GPU accelerator, ')\n#     print('and then re-execute this cell.')\n# else:\n#     print(gpu_info)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:52.781285Z","iopub.execute_input":"2022-12-26T11:51:52.781592Z","iopub.status.idle":"2022-12-26T11:51:52.785521Z","shell.execute_reply.started":"2022-12-26T11:51:52.781557Z","shell.execute_reply":"2022-12-26T11:51:52.784313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Import 3rdparty**","metadata":{"editable":false}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport random\nimport cv2\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn.functional as F\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport timm\nfrom matplotlib import pyplot as plt\nfrom sklearn.model_selection import StratifiedKFold\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom tqdm import tqdm","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:52.788435Z","iopub.execute_input":"2022-12-26T11:51:52.789058Z","iopub.status.idle":"2022-12-26T11:51:54.020135Z","shell.execute_reply.started":"2022-12-26T11:51:52.789009Z","shell.execute_reply":"2022-12-26T11:51:54.01937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm as lgb","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    seed = 42\n    data_dir = '/kaggle/input/cassava-leaf-disease-classification/'\n    train_data_dir = data_dir + 'train_images/'\n    train_csv_path = data_dir + 'train.csv'\n    test_data_dir = data_dir + 'test_images/'\n    test_csv_path = data_dir + 'test.csv'\n    arch = 'nasnetalarge' ## model name\n    device = 'cuda'\n    debug = True                 ##\n    \n    image_size = 331    \n    train_batch_size = 16\n    test_batch_size = 32\n    epochs = 10                 ## total train epochs\n    freeze_bn_epochs = 5        ## freeze bn weights before epochs\n    \n    lr=1e-4                     ## init learning rate\n    min_lr = 1e-6               ## min learning rate\n    weight_decay = 1e-6\n    num_workers = 4\n    n_splits = 5             ## numbers splits\n    num_classes = 5            ## numbers classes\n    T_0 = 10\n    T_mult = 1\n    accum_iter = 2\n    verbose_step = 1\n    tta = 3\n    \n    criterion = 'CrossEntropy' ## CrossEntropy, LabelSmoothingCrossEntropy\n    label_smoothing = 0.3\n    \n    train_id = [0,1,2,3,4]\n    \n    lgb_folds       = 5    # no. folds for stacking\n    lgb_stop_rounds = 200  # no. early stopping rounds\n    lgb_params      = {'objective':         'multiclass',\n                       'metrics':           'multi_error',\n                       'n_estimators':      10000,\n                       'learning_rate':     0.01,\n                       'num_leaves':        10,\n                       'max_depth':         5,\n                       'min_child_samples': 20,\n                       'subsample':         0.3,\n                       'colsample_bytree':  0.3,\n                       'reg_alpha':         0.15,\n                       'reg_lambda':        0.15,\n                       'silent':            True,\n                       'verbosity':         -1,\n                       'n_jobs' :           2,\n                       'random_state':      13353}\nCFG = Config()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:54.022429Z","iopub.execute_input":"2022-12-26T11:51:54.022764Z","iopub.status.idle":"2022-12-26T11:51:54.028609Z","shell.execute_reply.started":"2022-12-26T11:51:54.022728Z","shell.execute_reply":"2022-12-26T11:51:54.027496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Global config**","metadata":{"editable":false}},{"cell_type":"markdown","source":"# **Load Image**","metadata":{"editable":false}},{"cell_type":"code","source":"def load_image(image_path):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    return img","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:54.041553Z","iopub.execute_input":"2022-12-26T11:51:54.041922Z","iopub.status.idle":"2022-12-26T11:51:54.052009Z","shell.execute_reply.started":"2022-12-26T11:51:54.041889Z","shell.execute_reply":"2022-12-26T11:51:54.051055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **CassavaDataset**","metadata":{"editable":false}},{"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, data_dir, df, transforms=None, output_label=True):\n        self.data_dir = data_dir\n        self.df = df\n        self.transforms = transforms\n        self.output_label = output_label\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, index):\n        image_infos = self.df.iloc[index]\n        image_path = self.data_dir + image_infos.image_id\n\n        image = load_image(image_path)\n\n        if image is None:\n            raise FileNotFoundError(image_path)\n\n        ### augment\n        if self.transforms is not None:\n            image = self.transforms(image=image)['image']\n        else:\n            image = torch.from_numpy(image)\n\n        if self.output_label:\n            return image, image_infos.label\n        else:\n            return image","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:54.054867Z","iopub.execute_input":"2022-12-26T11:51:54.055123Z","iopub.status.idle":"2022-12-26T11:51:54.064993Z","shell.execute_reply.started":"2022-12-26T11:51:54.055099Z","shell.execute_reply":"2022-12-26T11:51:54.064385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **CassavaClassifier**","metadata":{"editable":false}},{"cell_type":"code","source":"class CassavaClassifier(nn.Module):\n    def __init__(self, model_arch, num_classes, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        ### vit\n\n        \n        '''\n        self.model.classifier = nn.Sequential(\n            nn.Dropout(0.3),\n            #nn.Linear(num_features, hidden_size,bias=True), nn.ELU(),\n            nn.Linear(num_features, num_classes, bias=True)\n        )\n        '''\n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:54.066516Z","iopub.execute_input":"2022-12-26T11:51:54.066904Z","iopub.status.idle":"2022-12-26T11:51:54.075108Z","shell.execute_reply.started":"2022-12-26T11:51:54.066867Z","shell.execute_reply":"2022-12-26T11:51:54.074394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Train and Val transforms**","metadata":{"editable":false}},{"cell_type":"code","source":"def get_train_transforms(CFG):\n    return A.Compose([\n            A.RandomResizedCrop(height=CFG.image_size, width=CFG.image_size, p=0.5),\n            A.Transpose(p=0.5),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.RandomRotate90(p=0.5),\n            A.ShiftScaleRotate(p=0.5),\n            A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            A.RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            A.CenterCrop(CFG.image_size, CFG.image_size),\n            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            A.CoarseDropout(p=0.5),\n            A.Cutout(p=0.5),\n            ToTensorV2(),\n        ],p=1.0)\n\ndef get_val_transforms(cfg):\n    return A.Compose([\n            A.CenterCrop(CFG.image_size, CFG.image_size, p=0.5),\n            A.Resize(CFG.image_size, CFG.image_size),\n            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(),\n        ],p=1.0)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:54.076616Z","iopub.execute_input":"2022-12-26T11:51:54.077017Z","iopub.status.idle":"2022-12-26T11:51:54.091272Z","shell.execute_reply.started":"2022-12-26T11:51:54.076983Z","shell.execute_reply":"2022-12-26T11:51:54.090392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_inference_transforms(CFG):\n    return A.Compose([\n            A.RandomResizedCrop(CFG.image_size, CFG.image_size),\n            A.Transpose(p=0.5),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            A.RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.0)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:54.092613Z","iopub.execute_input":"2022-12-26T11:51:54.092993Z","iopub.status.idle":"2022-12-26T11:51:54.102364Z","shell.execute_reply.started":"2022-12-26T11:51:54.092959Z","shell.execute_reply":"2022-12-26T11:51:54.101566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Train and Val data loader**","metadata":{"editable":false}},{"cell_type":"markdown","source":"# **Train one epoch**","metadata":{"editable":false}},{"cell_type":"code","source":"def train_one_epoch(epoch,model,loss_fn,optimizer,train_loader,device,scheduler=None,schd_batch_update=False):\n    model.train()\n    lr = optimizer.state_dict()['param_groups'][0]['lr']   ##不同\n    \n    running_loss = None\n    pbar = tqdm(enumerate(train_loader),total=len(train_loader))\n    for step,(images,targets) in pbar:\n        images = images.to(device).float()\n        targets = targets.to(device).long()\n        \n        with autocast():\n            preds = model(images)\n            loss = loss_fn(preds,targets)\n        \n            scaler.scale(loss).backward()\n            if running_loss is None:\n                running_loss = loss.item()\n            else:\n                running_loss = running_loss* 0.99 + loss.item()*0.01\n                \n            if ((step + 1) % CFG.accum_iter == 0) or ((step + 1) == len(train_loader)):\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()\n                \n                if scheduler is not None and schd_batch_update:\n                    scheduler.step()\n            if ((step + 1) % CFG.accum_iter == 0) or ((step + 1) == len(train_loader)):\n                description = f'Train epoch {epoch} loss: {running_loss:.5f}'\n                pbar.set_description(description)\n                \n    if scheduler is not None and schd_batch_update:\n        scheduler.step()","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:54.103474Z","iopub.execute_input":"2022-12-26T11:51:54.105411Z","iopub.status.idle":"2022-12-26T11:51:54.118845Z","shell.execute_reply.started":"2022-12-26T11:51:54.105373Z","shell.execute_reply":"2022-12-26T11:51:54.11806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dataloader(CFG, df, train_idx, mode='train'):\n    df_train = df.loc[train_idx,:].reset_index(drop=True)\n#     df_val = df.loc[val_idx,:].reset_index(drop=True)\n    if mode == 'train':\n        train_dataset = CassavaDataset(\n            CFG.train_data_dir,\n            df_train,\n            transforms=get_train_transforms(CFG),\n            output_label=False)\n\n        train_loader = torch.utils.data.DataLoader(\n            train_dataset,\n            batch_size=CFG.train_batch_size,\n            pin_memory=False,\n    #         drop_last=False,\n            shuffle=False,\n            num_workers=CFG.num_workers,\n            #sampler=BalanceClassSampler(labels=train_['label'].values, mode=\"downsampling\")\n        )\n        return train_loader\n    else:\n        test_dataset = CassavaDataset(\n            CFG.test_data_dir,\n            df_train,\n            transforms=get_train_transforms(CFG),\n            output_label=False)\n\n        test_loader = torch.utils.data.DataLoader(\n            test_dataset,\n            batch_size=CFG.test_batch_size,\n            pin_memory=False,\n            #         drop_last=False,\n            shuffle=False,\n            num_workers=CFG.num_workers,\n            # sampler=BalanceClassSampler(labels=train_['label'].values, mode=\"downsampling\")\n        )\n        return test_loader","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Valid one epoch**","metadata":{"editable":false}},{"cell_type":"code","source":"def valid_one_epoch(epoch,model,loss_fn,val_loader,device,scheduler=None,schd_loss_update=False):\n    model.eval()\n    \n    loss_sum = 0\n    sample_num = 0\n    preds_all = []\n    targets_all = []\n    scores = []\n    \n    pbar = tqdm(enumerate(val_loader),total=len(val_loader))\n    for step,(images,targets) in pbar:\n        images = images.to(device).float()\n        targets = targets.to(device).long()\n        preds = model(images)\n            \n        preds_all += [torch.argmax(preds,1).detach().cpu().numpy()]\n        targets_all += [targets.detach().cpu().numpy()]\n\n        loss = loss_fn(preds,targets)\n        loss_sum += loss.item()*targets.shape[0]\n        sample_num += targets.shape[0]\n           \n        if ((step + 1) % CFG.accum_iter == 0) or ((step + 1) == len(train_loader)):\n            description = f'Val epoch {epoch} loss: {loss_sum/sample_num:.5f}'\n            pbar.set_description(description)\n            \n    preds_all = np.concatenate(preds_all)\n    targets_all = np.concatenate(targets_all)\n    accuracy = (preds_all == targets_all).mean()\n    print(f'Validation multi-class accuracy = {accuracy:.5f}')\n    \n    if scheduler is not None:\n        if schd_loss_update:\n            scheduler.step(loss_sum/sample_num)\n        else:\n            scheduler.step()\n    \n    return accuracy","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:54.131063Z","iopub.execute_input":"2022-12-26T11:51:54.131466Z","iopub.status.idle":"2022-12-26T11:51:54.144445Z","shell.execute_reply.started":"2022-12-26T11:51:54.131431Z","shell.execute_reply":"2022-12-26T11:51:54.143585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)   #output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:54.146073Z","iopub.execute_input":"2022-12-26T11:51:54.146476Z","iopub.status.idle":"2022-12-26T11:51:54.156324Z","shell.execute_reply.started":"2022-12-26T11:51:54.146419Z","shell.execute_reply":"2022-12-26T11:51:54.155576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Freeze bn weights**","metadata":{"editable":false}},{"cell_type":"code","source":"################ freeze bn \ndef freeze_batchnorm_stats(net):\n    try:\n        for m in net.modules():\n            if isinstance(m,nn.BatchNorm2d) or isinstance(m,nn.LayerNorm):\n                m.eval()\n    except ValuError:\n        print('error with batchnorm2d or layernorm')\n        return","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:54.15741Z","iopub.execute_input":"2022-12-26T11:51:54.157757Z","iopub.status.idle":"2022-12-26T11:51:54.166336Z","shell.execute_reply.started":"2022-12-26T11:51:54.157725Z","shell.execute_reply":"2022-12-26T11:51:54.165304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Label Smoothing Cross Entropy Loss**","metadata":{"editable":false}},{"cell_type":"code","source":"class LabelSmoothingCrossEntropy(nn.Module):\n    \"\"\"\n    NLL loss with label smoothing.\n    \"\"\"\n    def __init__(self, smoothing=0.1):\n        \"\"\"\n        Constructor for the LabelSmoothing module.\n        :param smoothing: label smoothing factor\n        \"\"\"\n        super(LabelSmoothingCrossEntropy, self).__init__()\n        assert smoothing < 1.0\n        self.smoothing = smoothing\n        self.confidence = 1. - smoothing\n\n    def forward(self, x, target):\n        logprobs = F.log_softmax(x, dim=-1)\n        nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1))\n        nll_loss = nll_loss.squeeze(1)\n        smooth_loss = -logprobs.mean(dim=-1)\n        loss = self.confidence * nll_loss + self.smoothing * smooth_loss\n        return loss.mean()","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:54.16767Z","iopub.execute_input":"2022-12-26T11:51:54.168006Z","iopub.status.idle":"2022-12-26T11:51:54.177903Z","shell.execute_reply.started":"2022-12-26T11:51:54.167973Z","shell.execute_reply":"2022-12-26T11:51:54.177305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asd=['../input/nasnet-ckpt/nasnetalarge_fold0_best_se_false.ckpt',\n     '../input/nasnet-ckpt/nasnetalarge_fold1_best_se_false.ckpt',\n     '../input/nasnet-ckpt/nasnetalarge_fold2_best_se_false.ckpt',\n     '../input/nasnet-ckpt/nasnetalarge_fold3_best_se_false.ckpt',\n     '../input/nasnet-ckpt/nasnetalarge_fold4_best_se_false.ckpt']","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:54.179452Z","iopub.execute_input":"2022-12-26T11:51:54.179834Z","iopub.status.idle":"2022-12-26T11:51:54.186741Z","shell.execute_reply.started":"2022-12-26T11:51:54.179799Z","shell.execute_reply":"2022-12-26T11:51:54.186081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:51:54.190884Z","iopub.execute_input":"2022-12-26T11:51:54.191355Z","iopub.status.idle":"2022-12-26T11:51:54.226968Z","shell.execute_reply.started":"2022-12-26T11:51:54.191328Z","shell.execute_reply":"2022-12-26T11:51:54.225952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tst_preds = []\ntrain_preds = pd.DataFrame()\ntrain_preds['image_id'] = list(os.listdir('/kaggle/input/cassava-leaf-disease-classification/train_images'))\ntest_preds = pd.DataFrame()\ntest_preds['image_id'] = list(os.listdir('/kaggle/input/cassava-leaf-disease-classification/test_images'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Main Loop**","metadata":{"editable":false}},{"cell_type":"code","source":"if __name__ == '__main__':\n    seed_everything(CFG.seed)\n    \n    folds = StratifiedKFold(n_splits=CFG.n_splits).split(np.arange(train.shape[0]), train.label.values)\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('/kaggle/input/cassava-leaf-disease-classification/test_images/'))\n\n        train_loader = load_dataloader(CFG, train,val_idx,'train')\n        tst_loader = load_dataloader(CFG, test,np.array(list(test.index)),'test')\n        \n\n        device = torch.device(CFG.device)\n        model = CassavaClassifier(CFG.arch, train.label.nunique(), pretrained=False).to(device)\n        model.load_state_dict(torch.load(asd[fold],map_location=device))\n        \n        tst_pred_nas = []\n        with torch.no_grad():\n            train_pred_nas = inference_one_epoch(model, train_loader, device)\n            for _ in range(CFG.tta):\n                tst_pred_nas += [inference_one_epoch(model, tst_loader, device)]\n            tst_pred_nas = np.mean(tst_pred_nas, axis=0) \n\n        if fold == 0:\n            for i in range(1,5):\n                train_preds['nasnetRank' + str(i)] = 0\n                test_preds['nasnetRank' + str(i)] = 0\n        for i in range(1,5):\n            train_preds['nasnetRank' + str(i)].loc[val_idx] = np.argsort(train_pred_nas,axis=1).T[-i]\n            test_preds['nasnetRank' + str(i)] = np.argsort(tst_pred_nas,axis=1).T[-i]\n        \n\n        del model\n        torch.cuda.empty_cache()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG.arch = 'maxxvit_rmlp_nano_rw_256'\nCFG.image_size = 256\nasd=['../input/maxxvit-ckpt/maxxvit_rmlp_nano_rw_256_fold0_best_se_false.ckpt',\n     '../input/maxxvit-ckpt/maxxvit_rmlp_nano_rw_256_fold1_best_se_false.ckpt',\n     '../input/maxxvit-ckpt/maxxvit_rmlp_nano_rw_256_fold2_best_se_false.ckpt',\n     '../input/maxxvit-ckpt/maxxvit_rmlp_nano_rw_256_fold3_best_se_false.ckpt',\n     '../input/maxxvit-ckpt/maxxvit_rmlp_nano_rw_256_fold4_best_se_false.ckpt']","metadata":{"execution":{"iopub.status.busy":"2022-12-26T11:52:27.560745Z","iopub.execute_input":"2022-12-26T11:52:27.561092Z","iopub.status.idle":"2022-12-26T11:52:27.568298Z","shell.execute_reply.started":"2022-12-26T11:52:27.561048Z","shell.execute_reply":"2022-12-26T11:52:27.567291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    seed_everything(CFG.seed)\n    \n    folds = StratifiedKFold(n_splits=CFG.n_splits).split(np.arange(train.shape[0]), train.label.values)\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('/kaggle/input/cassava-leaf-disease-classification/test_images/'))\n\n        train_loader = load_dataloader(CFG, train,val_idx,'train')\n        tst_loader = load_dataloader(CFG, test,np.array(list(test.index)),'test')\n        \n\n        device = torch.device(CFG.device)\n        model = CassavaClassifier(CFG.arch, train.label.nunique(), pretrained=False).to(device)\n        model.load_state_dict(torch.load(asd[fold],map_location=device))\n        \n        tst_pred_max = []\n        with torch.no_grad():\n            train_pred_max = inference_one_epoch(model, train_loader, device)\n            for _ in range(CFG.tta):\n                tst_pred_max += [inference_one_epoch(model, tst_loader, device)]\n            tst_pred_max = np.mean(tst_pred_max, axis=0) \n\n        if fold == 0:\n            for i in range(1,5):\n                train_preds['maxvvitRank' + str(i)] = 0\n                test_preds['maxvvitRank' + str(i)] = 0\n        for i in range(1,5):\n            train_preds['maxvvitRank' + str(i)].loc[val_idx] = np.argsort(train_pred_max,axis=1).T[-i]\n            test_preds['maxvvitRank' + str(i)] = np.argsort(tst_pred_max,axis=1).T[-i]\n        \n\n        del model\n        torch.cuda.empty_cache()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG.arch = 'convnext_pico'\nCFG.image_size = 256 \nasd=['../input/coatnext-pico-ckpt/convnext_pico_fold0_best_ce.ckpt',\n     '../input/coatnext-pico-ckpt/convnext_pico_fold1_best_ce.ckpt',\n     '../input/coatnext-pico-ckpt/convnext_pico_fold2_best_ce.ckpt',\n     '../input/coatnext-pico-ckpt/convnext_pico_fold3_best_ce.ckpt',\n     '../input/coatnext-pico-ckpt/convnext_pico_fold4_best_ce.ckpt']","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:52:36.127585Z","iopub.execute_input":"2022-12-26T11:52:36.127957Z","iopub.status.idle":"2022-12-26T11:52:36.134486Z","shell.execute_reply.started":"2022-12-26T11:52:36.12791Z","shell.execute_reply":"2022-12-26T11:52:36.133321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    seed_everything(CFG.seed)\n    \n    folds = StratifiedKFold(n_splits=CFG.n_splits).split(np.arange(train.shape[0]), train.label.values)\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('/kaggle/input/cassava-leaf-disease-classification/test_images/'))\n\n        train_loader = load_dataloader(CFG, train,val_idx,'train')\n        tst_loader = load_dataloader(CFG, test,np.array(list(test.index)),'test')\n        \n\n        device = torch.device(CFG.device)\n        model = CassavaClassifier(CFG.arch, train.label.nunique(), pretrained=False).to(device)\n        model.load_state_dict(torch.load(asd[fold],map_location=device))\n        \n        tst_pred_pico = []\n        with torch.no_grad():\n            train_pred_pico = inference_one_epoch(model, train_loader, device)\n            for _ in range(CFG.tta):\n                tst_pred_pico += [inference_one_epoch(model, tst_loader, device)]\n            tst_pred_pico = np.mean(tst_pred_pico, axis=0) \n\n        if fold == 0:\n            for i in range(1,5):\n                train_preds['picoRank' + str(i)] = 0\n                test_preds['picoRank' + str(i)] = 0\n        for i in range(1,5):\n            train_preds['picoRank' + str(i)].loc[val_idx] = np.argsort(train_pred_pico,axis=1).T[-i]\n            test_preds['picoRank' + str(i)] = np.argsort(tst_pred_pico,axis=1).T[-i]\n        \n\n        del model\n        torch.cuda.empty_cache()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG.arch = 'convnext_atto'\nCFGimage_size = 256    \nasd=['../input/convnext-atto-ckpt/convnext_atto_fold0_best_se_false.ckpt',\n     '../input/convnext-atto-ckpt/convnext_atto_fold1_best_se_false.ckpt',\n     '../input/convnext-atto-ckpt/convnext_atto_fold2_best_se_false.ckpt',\n     '../input/convnext-atto-ckpt/convnext_atto_fold3_best_se_false.ckpt',\n     '../input/convnext-atto-ckpt/convnext_atto_fold4_best_se_false.ckpt']","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-12-26T11:52:41.478912Z","iopub.execute_input":"2022-12-26T11:52:41.479307Z","iopub.status.idle":"2022-12-26T11:52:41.486724Z","shell.execute_reply.started":"2022-12-26T11:52:41.479261Z","shell.execute_reply":"2022-12-26T11:52:41.48562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    seed_everything(CFG.seed)\n    \n    folds = StratifiedKFold(n_splits=CFG.n_splits).split(np.arange(train.shape[0]), train.label.values)\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('/kaggle/input/cassava-leaf-disease-classification/test_images/'))\n\n        train_loader = load_dataloader(CFG, train,val_idx,'train')\n        tst_loader = load_dataloader(CFG, test,np.array(list(test.index)),'test')\n        \n\n        device = torch.device(CFG.device)\n        model = CassavaClassifier(CFG.arch, train.label.nunique(), pretrained=False).to(device)\n        model.load_state_dict(torch.load(asd[fold],map_location=device))\n        \n        tst_pred_atto = []\n        with torch.no_grad():\n            train_pred_atto = inference_one_epoch(model, train_loader, device)\n            for _ in range(CFG.tta):\n                tst_pred_atto += [inference_one_epoch(model, tst_loader, device)]\n            tst_pred_atto = np.mean(tst_pred_atto, axis=0) \n\n        if fold == 0:\n            for i in range(1,5):\n                train_preds['attoRank' + str(i)] = 0\n                test_preds['attoRank' + str(i)] = 0\n        for i in range(1,5):\n            train_preds['attoRank' + str(i)].loc[val_idx] = np.argsort(train_pred_atto,axis=1).T[-i]\n            test_preds['attoRank' + str(i)] = np.argsort(tst_pred_atto,axis=1).T[-i]\n        \n\n        del model\n        torch.cuda.empty_cache()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_preds['label'] = train['label']\ntrain_preds.to_csv('/kaggle/working/train_preds.csv')\ntest_preds.to_csv('/kaggle/working/test_preds.csv')\n\ndrop_features = ['image_id', 'label']\nfeatures      = [f for f in train_preds.columns if f not in drop_features]\n\ntrain_preds = pd.read_csv('/kaggle/working/train_preds.csv')\ntest_preds = pd.read_csv('/kaggle/working/test_preds.csv')\ndf = pd.read_csv('/kaggle/input/cassava-leaf-disease-classification/sample_submission.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Please Upvote if you liked the kernel ! Cheers.\n\nReferences :\nhttps://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-train-amp-aug. \nPlease Upvote too.","metadata":{"editable":false}},{"cell_type":"code","source":"drop_features = ['image_id', 'label']\nfeatures      = [f for f in train_preds.columns if f not in drop_features]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    folds = StratifiedKFold(n_splits = CFG.lgb_folds).split(np.arange(train_preds.shape[0]), train_preds.label.values)\n    \n    # placeholders\n    oof_preds   = np.zeros((len(train_preds), CFG.num_classes))\n    importances = pd.DataFrame()\n    \n    # cross-validation\n    print('-' * 30)    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        print(fold)\n        X_train, y_train = train_preds[features].loc[trn_idx], train_preds['label'].loc[trn_idx]\n        X_valid, y_valid = train_preds[features].loc[val_idx], train_preds['label'].loc[val_idx]\n        X_test           = test_preds[features]\n        \n        clf = lgb.LGBMClassifier(**CFG.lgb_params) \n        clf = clf.fit(X_train, y_train, \n                      eval_set              = [(X_valid, y_valid)],\n                      early_stopping_rounds = CFG.lgb_stop_rounds,\n                      verbose               = False)\n        \n        oof_preds[val_idx, :] = clf.predict_proba(X_valid)\n        lgb_preds             = clf.predict_proba(X_test)\n        \n        lgb_preds = pd.DataFrame(lgb_preds, columns = ['f' + str(fold) + '_' + str(c) for c in ['c0', 'c1', 'c2', 'c3', 'c4']])\n        if fold == 0:\n            all_lgb_preds = lgb_preds.copy()\n        else:\n            all_lgb_preds = pd.concat([all_lgb_preds, lgb_preds], axis = 1)\n            \n        # feature importance\n        fold_importance_df = pd.DataFrame()\n        fold_importance_df['Feature']    = features\n        fold_importance_df['Importance'] = clf.feature_importances_\n        fold_importance_df['Fold']       = fold + 1\n        importances = pd.concat([importances, fold_importance_df], axis = 0)\n    \n        # information\n        print('- FOLD {}: accuracy = {:.4f}'.format(fold + 1, (np.argmax(oof_preds[val_idx, :], axis = 1) == y_valid).sum() / len(y_valid)))\n        \n    # print performance\n    print('-' * 30)\n    print('OOF accuracy = {:.4f}'.format((np.argmax(oof_preds, axis = 1) == train_preds['label']).sum() / len(train_preds['label'])))\n    print('-' * 30)\n    \n    for c in ['c0', 'c1', 'c2', 'c3', 'c4']:\n        all_lgb_preds[c] = all_lgb_preds.filter(like = c).mean(axis = 1)\n    \n    df['label'] = np.argmax(all_lgb_preds[['c0', 'c1', 'c2', 'c3', 'c4']].values, axis = 1)\n    df.to_csv('/kaggle/working/submission.csv', index = False)","metadata":{},"execution_count":null,"outputs":[]}]}