{"cells":[{"metadata":{},"cell_type":"markdown","source":"- model1 : eff2020\n- model2 : eff2019+2020\n- model3 : reg2020\n- model4 : reg2019+2020\n- model5 : eff(seed:720)\n- model6 : reg distillation"},{"metadata":{"trusted":true},"cell_type":"code","source":"package_paths = [\n    '../input/pytorch-image-models/pytorch-image-models-master', #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\n    '../input/adamp-optimizer/AdamP-master/adamp'\n]\nimport sys; \n\nfor pth in package_paths:\n    sys.path.append(pth)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\n\nimport timm\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\n#from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'valid': False, \n    'fold_num': 5,\n    'seed': 719,\n    'model_arch1': 'tf_efficientnet_b4_ns',\n    'model_arch2': 'tf_efficientnet_b4_ns',\n    'model_arch3' : 'regnety_040', \n    'model_arch4' : 'regnety_040', \n    'model_arch5': 'tf_efficientnet_b4_ns',\n    'model_arch6': 'regnety_040',\n    'ckpt_path2': 'regnety4noresetadamp',\n    'ckpt_path3': 'regnety4nocv',\n    'weight' : [1/6 for _ in range(6)],\n    'img_size1': 512,\n    'img_size2': 512,\n    'img_size3': 512,\n    'img_size4': 512,\n    'epochs': 10,\n    'tta_num' : 3,\n    'train_bs': 64,\n    'valid_bs': 64,\n    'T_0': 10,\n    'lr': 1e-4,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_workers': 4,\n    'accum_iter': 2, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0', \n    'used_epochs': [5, 6, 7, 8, 9] # Last 5 Epoch \n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rand_bbox(size, lam):\n    W = size[0]\n    H = size[1]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\n\nclass CassavaDataset(Dataset):\n    def __init__(self, df, data_root, \n                 transforms=None, \n                 output_label=True, \n                ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        \n        self.output_label = output_label\n        if self.output_label == True: \n            self.labels = self.df['label'].values\n\n            \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.labels[index]\n          \n        img  = get_img(\"{}/{}\".format(self.data_root, self.df.loc[index]['image_id']))\n\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize, Rotate,\n    ShiftScaleRotate, CenterCrop, Resize, Rotate, RandomShadow, RandomSizedBBoxSafeCrop,\n    ChannelShuffle, MotionBlur\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size1'], CFG['img_size1']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(CFG['img_size1'], CFG['img_size1'], p=1.),\n            Resize(CFG['img_size1'], CFG['img_size1']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\n# def get_inference_transforms():\n#     return Compose([\n#             CenterCrop(CFG['img_size1'], CFG['img_size1'], p=1.),\n#             Transpose(p=0.5),\n#             HorizontalFlip(p=0.5),\n#             VerticalFlip(p=0.5),\n#             Resize(CFG['img_size1'], CFG['img_size1']),\n#             Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n#             ToTensorV2(p=1.0),\n#         ], p=1.)\n\ndef get_inference_transforms1():\n    return Compose([\n            OneOf([\n                Resize(CFG['img_size1'], CFG['img_size1'], p=1.),\n                CenterCrop(CFG['img_size1'], CFG['img_size1'], p=1.),\n                RandomResizedCrop(CFG['img_size1'], CFG['img_size1'], p=1.)\n            ], p=1.), \n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            Resize(CFG['img_size1'], CFG['img_size1']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\ndef get_inference_transforms2():\n    return Compose([\n            OneOf([\n                Resize(CFG['img_size2'], CFG['img_size2'], p=1.),\n                CenterCrop(CFG['img_size2'], CFG['img_size2'], p=1.),\n                RandomResizedCrop(CFG['img_size2'], CFG['img_size2'], p=1.)\n            ], p=1.), \n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            Resize(CFG['img_size2'], CFG['img_size2']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        if model_arch == 'regnety_040':\n            self.model.head = nn.Sequential(\n                                nn.AdaptiveAvgPool2d((1,1)),\n                                nn.Flatten(),\n                                nn.Linear(1088, n_class)\n            )\n        elif model_arch == 'regnety_320':\n            self.model.head = nn.Sequential(\n                                nn.AdaptiveAvgPool2d((1,1)),\n                                nn.Flatten(),\n                                nn.Linear(3712, n_class)\n            )\n        elif model_arch == 'regnety_080':\n            self.model.head = nn.Sequential(\n                                nn.AdaptiveAvgPool2d((1,1)),\n                                nn.Flatten(),\n                                nn.Linear(2016, n_class)\n            )\n            \n        elif model_arch == 'regnety_160':\n            self.model.head = nn.Sequential(\n                                nn.AdaptiveAvgPool2d((1,1)),\n                                nn.Flatten(),\n                                nn.Linear(3024, n_class)\n            )\n            \n        else:\n            n_features = self.model.classifier.in_features\n            self.model.classifier = nn.Linear(n_features, n_class)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassvaImgClassifier_ViT(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        #if pretrained:\n        #    self.model.load_state_dict(torch.load(MODEL_PATH))\n\n        self.model.head = nn.Linear(self.model.head.in_features, n_class)\n        \n        for module in self.model.modules():\n            #print(module)\n            if isinstance(module, nn.BatchNorm2d):\n                if hasattr(module, 'weight'):\n                    module.weight.requires_grad_(False)\n                if hasattr(module, 'bias'):\n                    module.bias.requires_grad_(False)\n                #module.eval()\n\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_dataloader(df, trn_idx, val_idx, data_root='../input/cassava-leaf-disease-classification/train_images/'):\n    \n    # from catalyst.data.sampler import BalanceClassSampler\n    \n    train_ = df.loc[trn_idx,:].reset_index(drop=True)\n    valid_ = df.loc[val_idx,:].reset_index(drop=True)\n        \n    train_ds = CassavaDataset(train_, data_root, transforms=get_train_transforms(), output_label=True)\n    valid_ds = CassavaDataset(valid_, data_root, transforms=get_valid_transforms(), output_label=True)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n        #sampler=BalanceClassSampler(labels=train_['label'].values, mode=\"downsampling\")\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n    return train_loader, val_loader\n\ndef train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None, schd_batch_update=False):\n    model.train()\n\n    t = time.time()\n    running_loss = None\n\n    # pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    for step, (imgs, image_labels) in enumerate(train_loader):\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n\n        with autocast():\n            image_preds = model(imgs)   #output = model(input)\n            loss = loss_fn(image_preds, image_labels)\n            \n            scaler.scale(loss).backward()\n\n            if running_loss is None:\n                running_loss = loss.item()\n            else:\n                running_loss = running_loss * .99 + loss.item() * .01\n\n            if ((step + 1) %  CFG['accum_iter'] == 0) or ((step + 1) == len(train_loader)):\n\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad() \n                \n                if scheduler is not None and schd_batch_update:\n                    scheduler.step()\n                \n    if scheduler is not None and not schd_batch_update:\n        scheduler.step()\n        \ndef valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    sample_num = 0\n    image_preds_all = []\n    image_targets_all = []\n    \n    # pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (imgs, image_labels) in enumerate(val_loader):\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n        \n        image_preds = model(imgs)   #output = model(input)\n        image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n        image_targets_all += [image_labels.detach().cpu().numpy()]\n        \n        loss = loss_fn(image_preds, image_labels)\n        \n        loss_sum += loss.item()*image_labels.shape[0]\n        sample_num += image_labels.shape[0]  \n\n        # if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(val_loader)):\n        #     description = f'epoch {epoch} loss: {loss_sum/sample_num:.4f}'\n        #     pbar.set_description(description)\n    \n    image_preds_all = np.concatenate(image_preds_all)\n    image_targets_all = np.concatenate(image_targets_all)\n    print('epoch = {}'.format(epoch+1), 'validation multi-class accuracy = {:.4f}'.format((image_preds_all==image_targets_all).mean()))\n    \n    if scheduler is not None:\n        if schd_loss_update:\n            scheduler.step(loss_sum/sample_num)\n        else:\n            scheduler.step()\n            \n            \ndef inference_one_epoch(model, data_loader, device):\n    model.eval()\n    image_preds_all = []\n    # pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    with torch.no_grad():\n        for step, (imgs) in enumerate(data_loader):\n            imgs = imgs.to(device).float()\n\n            image_preds = model(imgs)   #output = model(input)\n            image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def freeze_batchnorm_stats(net):\n    try:\n        for m in net.modules():\n            if isinstance(m,nn.BatchNorm2d) or isinstance(m,nn.LayerNorm):\n                m.eval()\n    except ValuError:\n        print('error with batchnorm2d or layernorm')\n        return\n    \ndef unfreeze_batchnorm_stats(net):\n    try:\n        for m in net.modules():\n            if isinstance(m,nn.BatchNorm2d) or isinstance(m,nn.LayerNorm):\n                m.train()\n    except ValuError:\n        print('error with batchnorm2d or layernorm')\n        return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LabelSmoothingCrossEntropy(nn.Module):\n    \"\"\"\n    NLL loss with label smoothing.\n    \"\"\"\n    def __init__(self, smoothing=0.1):\n        \"\"\"\n        Constructor for the LabelSmoothing module.\n        :param smoothing: label smoothing factor\n        \"\"\"\n        super(LabelSmoothingCrossEntropy, self).__init__()\n        assert smoothing < 1.0\n        self.smoothing = smoothing\n        self.confidence = 1. - smoothing\n\n    def forward(self, x, target):\n        logprobs = torch.nn.functional.log_softmax(x, dim=-1)\n        nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1))\n        nll_loss = nll_loss.squeeze(1)\n        smooth_loss = -logprobs.mean(dim=-1)\n        loss = self.confidence * nll_loss + self.smoothing * smooth_loss\n        return loss.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])    \n    oof_preds = np.zeros(len(train))\n    \n    ## model 1\n    print('Model 1 Start')\n    sub1 = [] \n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n\n        print('Inference fold {} started'.format(fold))\n\n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds2 = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2(), output_label=False)\n\n        tst_loader2 = torch.utils.data.DataLoader(\n            test_ds2, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n        \n        device = torch.device(CFG['device'])\n        \n        model1 = CassvaImgClassifier(CFG['model_arch1'], train.label.nunique()).to(device) # efficientnet\n        \n        tst_preds = []\n        \n        model1.load_state_dict(torch.load('../input/leaf-weight-v9-2/model9_2/swa_{}_fold_{}_{}'.format(CFG['model_arch1'], fold, '9')))\n        \n        for tta in range(CFG['tta_num']):\n            tst_preds += [inference_one_epoch(model1, tst_loader2, device)]\n        \n        sub1 += [np.mean(tst_preds, axis=0)] \n\n        del model1; \n        torch.cuda.empty_cache()\n        \n    ## model 2\n    print('Model 2 Start')\n    sub2 = []\n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n\n        print('Inference fold {} started'.format(fold))\n\n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds2 = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2(), output_label=False)\n\n        tst_loader2 = torch.utils.data.DataLoader(\n            test_ds2, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n        \n        device = torch.device(CFG['device'])\n    \n        model2 = CassvaImgClassifier(CFG['model_arch2'], train.label.nunique()).to(device) # EFF-2019+2020\n        \n        tst_preds = []\n        \n        model2.load_state_dict(torch.load('../input/905-training-efficientnetb4-merged-bs32/swa_{}_fold_{}_{}'.format(CFG['model_arch2'], fold, '9')))\n        \n        for tta in range(CFG['tta_num']):\n            tst_preds += [inference_one_epoch(model2, tst_loader2, device)]\n        \n        sub2 += [np.mean(tst_preds, axis=0)] \n\n        del model2;\n        torch.cuda.empty_cache()\n        \n    ## model 3\n    print('Model 3 Start')\n    sub3 = []\n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n\n        print('Inference fold {} started'.format(fold))\n\n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds2 = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2(), output_label=False)\n\n        tst_loader2 = torch.utils.data.DataLoader(\n            test_ds2, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n        \n        device = torch.device(CFG['device'])\n    \n        model3 = CassvaImgClassifier(CFG['model_arch3'], train.label.nunique()).to(device) # regnet-2020\n        \n        tst_preds = []\n        \n        model3.load_state_dict(torch.load('../input/regnety4noresetadamp/swa_{}_fold_{}_{}'.format(CFG['model_arch3'], fold, '19')))\n        \n        for tta in range(CFG['tta_num']):\n            tst_preds += [inference_one_epoch(model3, tst_loader2, device)]\n        \n        sub3 += [np.mean(tst_preds, axis=0)] \n\n        del model3;\n        torch.cuda.empty_cache()\n    \n    ## model 4\n    print('Model 4 Start')\n    sub4 = []\n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n\n        print('Inference fold {} started'.format(fold))\n\n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds2 = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2(), output_label=False)\n\n        tst_loader2 = torch.utils.data.DataLoader(\n            test_ds2, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n        \n        device = torch.device(CFG['device'])\n    \n        model4 = CassvaImgClassifier(CFG['model_arch4'], train.label.nunique()).to(device) # regnet-2019+2020\n        \n        tst_preds = []\n        \n        model4.load_state_dict(torch.load('../input/0214v1-hwkim-regnet-40-reset-swalr-swastep-ep24/swa_{}_fold_{}_{}'.format(CFG['model_arch4'], fold, '23')))\n        \n        for tta in range(CFG['tta_num']):\n            tst_preds += [inference_one_epoch(model4, tst_loader2, device)]\n        \n        sub4 += [np.mean(tst_preds, axis=0)] \n\n        del model4;\n        torch.cuda.empty_cache()    \n\n    ## model 5\n    print('Model 5 Start')\n    sub5 = []\n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n\n        print('Inference fold {} started'.format(fold))\n\n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds2 = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2(), output_label=False)\n\n        tst_loader2 = torch.utils.data.DataLoader(\n            test_ds2, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n        \n        device = torch.device(CFG['device'])\n    \n        model5 = CassvaImgClassifier(CFG['model_arch5'], train.label.nunique()).to(device) # EFF-SEED720\n        \n        tst_preds = []\n        \n        model5.load_state_dict(torch.load('../input/905-training-efficientnetb4-seed720/swa_{}_fold_{}_{}'.format(CFG['model_arch5'], fold, '9')))\n        \n        for tta in range(CFG['tta_num']):\n            tst_preds += [inference_one_epoch(model5, tst_loader2, device)]\n        \n        sub5 += [np.mean(tst_preds, axis=0)] \n\n        del model5;\n        torch.cuda.empty_cache()\n        \n    ## model 6\n    print('Model 6 Start')\n    sub6 = []\n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n\n        print('Inference fold {} started'.format(fold))\n\n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds2 = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2(), output_label=False)\n\n        tst_loader2 = torch.utils.data.DataLoader(\n            test_ds2, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n        \n        device = torch.device(CFG['device'])\n    \n        model6 = CassvaImgClassifier(CFG['model_arch6'], train.label.nunique()).to(device) # reg-distillation\n        \n        tst_preds = []\n        \n        model6.load_state_dict(torch.load('../input/reg-distill/swa_{}_fold_{}_{}'.format(CFG['model_arch6'], fold, '19')))\n        \n        for tta in range(CFG['tta_num']):\n            tst_preds += [inference_one_epoch(model6, tst_loader2, device)]\n        \n        sub6 += [np.mean(tst_preds, axis=0)] \n\n        del model6;\n        torch.cuda.empty_cache()\n        \n    sub1 = [e * CFG['weight'][0] for e in sub1]\n    sub2 = [e * CFG['weight'][1] for e in sub2]\n    sub3 = [e * CFG['weight'][2] for e in sub3]\n    sub4 = [e * CFG['weight'][3] for e in sub4]\n    sub5 = [e * CFG['weight'][4] for e in sub5]\n    sub6 = [e * CFG['weight'][5] for e in sub6]\n    \n    sub = [e1 + e2 + e3 + e4 + e5 + e6 for (e1, e2, e3, e4, e5, e6) in zip(sub1,sub2,sub3, sub4, sub5, sub6)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['label'] = np.argmax(np.mean(sub, axis=0) , axis=1)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}