{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install albumentations\n!pip install ../input/timm031/timm-0.3.1-py3-none-any.whl\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torchvision.datasets import MNIST\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom torchvision import transforms\nimport pytorch_lightning as pl\n\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nfrom tqdm.auto import tqdm\nimport timm\nfrom sklearn.metrics import accuracy_score\n\nimport albumentations as A\nfrom albumentations import Compose\nfrom albumentations.pytorch import ToTensorV2\n\nfrom PIL import Image, ImageOps, ImageEnhance, ImageChops\nimport pandas as pd\nimport numpy as np\nimport random\n\n\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom pytorch_lightning import loggers as pl_loggers\n\n    \n# Ignore warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\nTraining = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Configurations\n# ====================================================\n# ====================================================\n# Configurations\n# ====================================================\nclass CONFIGURATION:\n    def __init__(self): \n        \n        self.DEBUG = True\n\n        #Model Params\n        self.N_TTA = 8\n        self.N_FOLDS = 5\n        self.MODEL_NAME = 'tf_efficientnet_b4_ns' # Recommended : ['deit_base_patch16_384','vit_large_patch16_384','tf_efficientnet_b4_ns','resnext50_32x4d']\n        self.pretrained = True\n        self.N_CLASSES = 5\n        self.TRAIN_FOLDS = [0,1,2,3,4]\n        #self.TRAIN_FOLDS = [1] #Folds to be Trained\n\n        self.scheduler_name = 'GradualWarmupSchedulerV2' # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts', 'OneCycleLR', 'GradualWarmupSchedulerV2','LambdaLR']\n        self.scheduler_update = 'batch' #['batch','epoch']\n        self.criterion_name = 'TaylorSmoothedLoss'        # ['CrossEntropyLoss', 'LabelSmoothingLoss', 'FocalLoss','FocalCosineLoss', 'SymmetricCrossEntropyLoss', 'BiTemperedLoss', 'TaylorCrossEntropyLoss', 'TaylorSmoothedLoss']\n        self.optimizer_name = 'AdamW' #['Adam','AdamW','AdamP','Ranger'] -> AdamP doesn't work on TPUs\n        self.LR_RAMPUP_EPOCHS = 1\n        self.LR_SUSTAIN_EPOCHS = 0\n\n        self.FREEZE = True #If you fine tune after START_FREEZE epochs\n        self.START_FREEZE = 12\n\n        #Image Size\n        self.HEIGHT = 512 #If VIT or deit is chosen as model: need 384 x 384\n        self.WIDTH = 512\n        self.CHANNELS = 3\n        \n        \n        #Training Params\n        self.BATCH_SIZE = 16 # PER REPLICA FOR TPUS    #RECOMMENDED : effnet = 16 ; resnext = 8 ; vit = 4 ; deit = 4\n        self.EPOCHS = 25 # more is definitely plausible and recommended around 10\n        self.LR = 2e-5\n        self.LR_START =1e-5\n        self.LR_MIN = 5e-6\n        self.weight_decay = 0\n        self.eps = 1e-8\n        self.PATIENCE = 3\n\n        #BiTemperedLoss\n        self.T1 = 0.2\n        self.T2 = 1.1\n        self.LABEL_SMOOTH = 0.2\n\n        #CosineAnnealingWarmRestarts\n        self.T_0 = self.EPOCHS\n\n        #CosineAnnealingLR\n        self.T_max = self.EPOCHS\n\n        self.NUM_WORKERS = 4\n\n        \n        self.IMG_MEAN = [0.485, 0.456, 0.406] #Mean for normalization Transform cassava = [0.4303, 0.4967, 0.3134] imgnet = [0.485, 0.456, 0.406]\n        self.IMG_STD = [0.229, 0.224, 0.225] #STD for normalization Transform cassava = [0.2142, 0.2191, 0.1954] imgnet = [0.229, 0.224, 0.225]\n\n        self.USE_2019 = False #Use 2019 images?\n\n\n        self.SEED = 42\n        \n        Aug_Norm = A.Normalize(mean=self.IMG_MEAN, std=self.IMG_STD, max_pixel_value=255.0, p=1.0)\n        Drop_Rand = A.CoarseDropout(max_holes=12, max_height=int(0.11*self.HEIGHT), max_width=int(0.11*self.WIDTH),\n                                    min_holes=1, min_height=int(0.03*self.HEIGHT), min_width=int(0.03*self.WIDTH),\n                                    always_apply=False, p=0.5)\n        Rand_Crop = A.RandomCrop(height= self.HEIGHT, width = self.WIDTH,always_apply=True, p=1.0)\n        Resize_Crop = A.RandomResizedCrop(self.HEIGHT, self.WIDTH,p=1.0)\n        self.train_transforms = Compose([\n                    A.Transpose(p=0.5),\n                    A.HorizontalFlip(p=0.5),\n                    A.VerticalFlip(p=0.5),\n                    A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.5),\n                    A.HueSaturationValue(\n                        hue_shift_limit=0.2, \n                        sat_shift_limit=0.2, \n                        val_shift_limit=0.2, \n                        p=0.5\n                    ),\n                    A.RandomBrightnessContrast(\n                            brightness_limit=(-0.1,0.1), \n                            contrast_limit=(-0.1, 0.1), \n                            p=0.5\n                        ),\n                    Resize_Crop,\n                    Drop_Rand,           \n                    Aug_Norm,   \n                    ToTensorV2(p=1.0),\n                ], p=1.)\n\n        self.light_transforms = Compose([\n                    A.Transpose(p=0.5),\n                    A.HorizontalFlip(p=0.5),\n                    A.VerticalFlip(p=0.5),\n                    A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.5),\n                    A.HueSaturationValue(\n                                hue_shift_limit=0.2, \n                                sat_shift_limit=0.2, \n                                val_shift_limit=0.2, \n                                p=0.5),\n                    A.RandomBrightnessContrast(\n                                    brightness_limit=(-0.1,0.1), \n                                    contrast_limit=(-0.1, 0.1), \n                                    p=0.5),\n                    Resize_Crop,\n                    Aug_Norm,   \n                    ToTensorV2(p=1.0),\n                ], p=1.)\n\n        self.heavy_transforms = Compose([\n            A.HorizontalFlip(p=0.5),\n\n            A.Resize(self.HEIGHT, self.WIDTH),\n\n            A.Transpose(p=0.5),\n            A.VerticalFlip(p=0.5),\n            #A.augmentations.transforms.ColorJitter(brightness=0.10, contrast=0.2, saturation=0.2, hue=0.00, always_apply=False, p=0.5),\n            A.ShiftScaleRotate(p=0.5),\n            A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            A.RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            A.augmentations.transforms.RGBShift (r_shift_limit=20, g_shift_limit=20, b_shift_limit=20, always_apply=False, p=0.5),\n            A.augmentations.transforms.ChannelDropout (channel_drop_range=(1, 1), fill_value=0, always_apply=False, p=0.5),\n\n            A.augmentations.transforms.GridDistortion (num_steps=5, distort_limit=0.3, interpolation=1, border_mode=4, value=None, mask_value=None, always_apply=False, p=0.5),\n            A.CoarseDropout(p=0.5),\n            A.Cutout(p=0.5),\n            Aug_Norm,\n            ToTensorV2(p=1.0),])\n\n        self.valid_transforms = Compose([\n                    A.CenterCrop(self.HEIGHT, self.WIDTH),\n                    Aug_Norm,   \n                    ToTensorV2(p=1.0),\n                ], p=1.)\n\n        self.test_aug = Compose([\n                    A.HorizontalFlip(p=0.5),\n                    A.VerticalFlip(p=0.5),\n                    A.ShiftScaleRotate(p = 1.0),\n                    #A.ColorJitter(brightness=0.1, contrast=0.2, saturation=0.2, hue=0.00, always_apply=False, p=1.0),\n                    Rand_Crop,\n                    Aug_Norm,\n                    ToTensorV2(p=1.0)\n                ], p=1.)\n\n#         image_net_post = Compose([\n#                     Resize_Crop,\n#                     Drop_Rand,\n#                     Aug_Norm,    \n#                     ToTensorV2(p=1.0)\n#                 ], p=1.)\n\n        self.TRAIN_AUG_TYPE = self.light_transforms\n        self.VALID_AUG_TYPE = self.valid_transforms\n\nCFG = CONFIGURATION()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/effnet-resnet-weights\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# CV Split\n# ====================================================\nDATA_PATH = '../input/cassava-leaf-disease-classification/'\nTRAIN_DIR = DATA_PATH + 'train_images/'\n\nDATA_PATH_2019 = './'\nTRAIN_DIR_2019 = DATA_PATH_2019 + 'train/'\nTEST_DIR = DATA_PATH + 'test_images/'\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if Training:\n    \n    #This guarantees that no images from 2019 contaminate the validation split\n    if CFG.USE_2019:\n        train_df_merged = pd.read_csv(DATA_PATH_2019 + 'merged.csv')\n        train_df = train_df_merged.loc[train_df_merged.source == 2020]\n        if CFG.DEBUG:\n            train_df = train_df.sample(500).reset_index(drop=True)\n        train_df_2019 = train_df_merged.loc[train_df_merged.source == 2019]\n        skf = StratifiedKFold(n_splits=CFG.N_FOLDS, shuffle=True, random_state=CFG.SEED)\n        skf.get_n_splits(np.arange(train_df.shape[0]), train_df['label'])\n        folds = [(idxT,idxV) for i,(idxT,idxV) in enumerate(skf.split(np.arange(train_df.shape[0]), train_df['label']))]\n        if not CFG.DEBUG:\n            folds_2019 = [np.concatenate((idxT,idxV)) for i,(idxT,idxV) in enumerate(skf.split(np.arange(train_df_2019.shape[0]), train_df_2019['label']))]\n            for i in range(CFG.N_FOLDS):\n                (idxT,idxV) = folds[i]\n                folds[i] = (np.concatenate((idxT,train_df_2019.iloc[folds_2019[i]].index)),idxV)\n                (idxT,idxV) = folds[i]\n                print(np.bincount(train_df_merged['label'].iloc[idxT]),np.bincount(train_df['label'].iloc[idxV]))\n        DATA_FOLD = TRAIN_DIR_2019\n        del train_df_2019\n    else:\n        print(\"not using 2019\")\n        train_df = pd.read_csv(DATA_PATH + 'train.csv')\n        if CFG.DEBUG:\n            train_df = train_df.sample(500).reset_index(drop=True)\n        skf = StratifiedKFold(n_splits=CFG.N_FOLDS, shuffle=True, random_state=CFG.SEED)\n        skf.get_n_splits(np.arange(train_df.shape[0]), train_df['label'])\n        folds = [(idxT,idxV) for i,(idxT,idxV) in enumerate(skf.split(np.arange(train_df.shape[0]), train_df['label']))]\n        for i in range(CFG.N_FOLDS):\n            (idxT,idxV) = folds[i]\n            print(np.bincount(train_df['label'].iloc[idxT]),np.bincount(train_df['label'].iloc[idxV]))\n\n        train_df_merged = train_df\n        DATA_FOLD = TRAIN_DIR","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" def _startify_and_save(data):\n        # New column to hold the fold number\n        data.loc[:, \"kfold\"] = -1\n\n        # Shuffle the dataframe\n        data = data.sample(frac=1).reset_index(drop=True)        \n        \n        # 5 Folds\n        skf = StratifiedKFold(n_splits=CFG.N_FOLDS, shuffle=False, random_state=CFG.SEED) \n                                                                                    \n        for fold_, (trn_,val_) in enumerate(skf.split(np.arange(data.shape[0]), data['label'])): \n            # We are just filling the vaidation indices. \n            # All other data are for training (trn indices are not required)\n            data.loc[val_, \"kfold\"] = fold_\n    \n        # We are saving the result to the disk so that other GPUs can pick it from there. \n        # Rather if we do \"self.startified_data = train_targets_scored\", \n        # other GPUs will not be able to read this \n        data.to_csv(\"train_folds.csv\", index=False)   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if Training:\n    _startify_and_save(train_df_merged)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LabelSmoothingLoss(nn.Module): \n    def __init__(self, classes=5, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        if CFG.criterion_name == 'LabelSmoothingLoss':\n            pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))\n    \nclass TaylorSoftmax(nn.Module):\n    '''\n    This is the autograd version\n    '''\n    def __init__(self, dim=1, n=2):\n        super(TaylorSoftmax, self).__init__()\n        assert n % 2 == 0\n        self.dim = dim\n        self.n = n\n\n    def forward(self, x):\n        '''\n        usage similar to nn.Softmax:\n            >>> mod = TaylorSoftmax(dim=1, n=4)\n            >>> inten = torch.randn(1, 32, 64, 64)\n            >>> out = mod(inten)\n        '''\n        fn = torch.ones_like(x)\n        denor = 1.\n        for i in range(1, self.n+1):\n            denor *= i\n            fn = fn + x.pow(i) / denor\n        out = fn / fn.sum(dim=self.dim, keepdims=True)\n        return out\n    \nclass TaylorCrossEntropyLoss(nn.Module):\n    '''\n    This is the autograd version\n    '''\n    def __init__(self, n=2, ignore_index=-1, reduction='mean'):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n\n    def forward(self, logits, labels):\n        '''\n        usage similar to nn.CrossEntropyLoss:\n            >>> crit = TaylorCrossEntropyLoss(n=4)\n            >>> inten = torch.randn(1, 10, 64, 64)\n            >>> label = torch.randint(0, 10, (1, 64, 64))\n            >>> out = crit(inten, label)\n        '''\n        log_probs = self.taylor_softmax(logits).log()\n        loss = F.nll_loss(log_probs, labels, reduction=self.reduction,\n                ignore_index=self.ignore_index)\n        return loss\n    \nclass TaylorSmoothedLoss(nn.Module):\n\n    def __init__(self, n=2, ignore_index=-1, reduction='mean', smoothing=0.2):\n        super(TaylorSmoothedLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n        self.lab_smooth = LabelSmoothingLoss(CFG.N_CLASSES, smoothing=CFG.LABEL_SMOOTH)\n\n    def forward(self, logits, labels):\n\n        log_probs = self.taylor_softmax(logits).log()\n        #loss = F.nll_loss(log_probs, labels, reduction=self.reduction,\n        #        ignore_index=self.ignore_index)\n        loss = self.lab_smooth(log_probs, labels)\n        return loss\n    \n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\n#Choose Criterions for the Training Loop\ndef GetCriterion(criterion_name):\n    if criterion_name == 'BiTemperedLoss':\n        criterion = BiTemperedLogistic()\n    elif criterion_name == 'SymmetricCrossEntropyLoss':\n        criterion = SymmetricCrossEntropy()\n    elif criterion_name == 'CrossEntropyLoss':\n        criterion = nn.CrossEntropyLoss()\n    elif criterion_name == 'LabelSmoothingLoss':\n        criterion = LabelSmoothingLoss()\n    elif criterion_name == 'FocalLoss':\n        criterion = FocalLoss()\n    elif criterion_name == 'FocalCosineLoss':\n        criterion = FocalCosineLoss()\n    elif criterion_name == 'TaylorCrossEntropyLoss':\n        criterion = TaylorCrossEntropyLoss()\n    elif criterion_name == 'TaylorSmoothedLoss':\n        criterion = TaylorSmoothedLoss()\n    elif criterion_name == 'CutMix':\n        criterion = CutMixCriterion(GetCriterion(CFG.criterion_name))\n    #print(criterion_name)\n    return criterion\n    \n    \ndef GetScheduler(scheduler_name,optimizer):\n    #['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts', 'OneCycleLR', 'GradualWarmupSchedulerV2']\n#     if scheduler_name == 'OneCycleLR':\n#         return torch.optim.lr_scheduler.OneCycleLR(optimizer,max_lr = 1e-2,epochs = CFG.EPOCHS,steps_per_epoch = batches+1,pct_start = 0.1)\n#     elif scheduler_name == 'CosineAnnealingWarmRestarts':\n#         return torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0 = CFG.T_0, T_mult=1, eta_min=CFG.LR_MIN, last_epoch=-1)\n#     el\n    if scheduler_name == 'CosineAnnealingLR':\n        return torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max = CFG.T_max, eta_min=0, last_epoch=-1)\n    elif scheduler_name == 'ReduceLROnPlateau':\n        return torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,factor=0.1, patience=1, threshold=0.0001, cooldown=0, min_lr=CFG.LR_MIN, eps=CFG.eps)\n    elif scheduler_name == 'GradualWarmupSchedulerV2':\n        return GradualWarmupSchedulerV2(optimizer=optimizer)\n    \ndef GetOptimizer(optimizer_name,parameters):\n    #['Adam','Ranger']\n    if optimizer_name == 'Adam':\n        if CFG.scheduler_name == 'GradualWarmupSchedulerV2':\n            return torch.optim.Adam(parameters, lr=CFG.LR_START, weight_decay=CFG.weight_decay, amsgrad=False)\n        else:\n            return torch.optim.Adam(parameters, lr=CFG.LR, weight_decay=CFG.weight_decay, amsgrad=False)\n    elif optimizer_name == 'AdamW':\n        if CFG.scheduler_name == 'GradualWarmupSchedulerV2':\n            return torch.optim.AdamW(parameters, lr=CFG.LR_START, weight_decay=CFG.weight_decay, amsgrad=False)\n        else:\n            return torch.optim.Adam(parameters, lr=CFG.LR, weight_decay=CFG.weight_decay, amsgrad=False)\n    elif optimizer_name == 'AdamP':\n        if CFG.scheduler_name == 'GradualWarmupSchedulerV2':\n            return AdamP(parameters, lr=CFG.LR_START, weight_decay=CFG.weight_decay)\n        else:\n            return AdamP(parameters, lr=CFG.LR, weight_decay=CFG.weight_decay)\n    elif optimizer_name == 'Ranger':\n        return Ranger(parameters,lr = CFG.LR,alpha = 0.5, k = 6,N_sma_threshhold = 5,betas = (0.95,0.999),eps=CFG.eps,weight_decay=CFG.weight_decay)\n    \n\nSEED = CFG.SEED\nseed_everything(SEED) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Datasets\n# ====================================================\nclass CasavaDataset(Dataset):\n    def __init__(self, directory, FNames, labels,transform):\n        self.dir = directory\n        self.fnames = FNames\n        self.labels = labels\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.fnames)\n    \n \n\n    def __getitem__(self, index):\n#         print(self.fnames[index])\n        img = Image.open(os.path.join(self.dir, self.fnames[index]))\n        img =  np.asarray(img,dtype = np.float32)\n        if self.transform is not None:\n            img = self.transform(image = img)['image']\n        return img, self.labels[index], self.fnames[index]\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaModel(pl.LightningModule):\n\n    def __init__(self,  model_name=CFG.MODEL_NAME, pretrained=CFG.pretrained, fold=None):\n        super().__init__()\n        self.accuracy = pl.metrics.Accuracy()\n        self.val_accuracy = pl.metrics.Accuracy()\n        self.model_name = model_name\n        self.fold = fold\n        if model_name == 'deit_base_patch16_224' or model_name == 'deit_base_patch16_384':\n            self.model = torch.hub.load('facebookresearch/deit:main', model_name, pretrained=pretrained)\n        else:\n            self.model = timm.create_model(model_name, pretrained=pretrained)\n        if 'efficientnet' in model_name:\n            self.n_features = self.model.classifier.in_features\n            self.model.classifier = nn.Linear(self.n_features, CFG.N_CLASSES)\n        elif model_name == 'vit_large_patch16_384' or model_name == 'deit_base_patch16_224' or model_name == 'deit_base_patch16_384':\n            self.n_features = self.model.head.in_features\n            self.model.head = nn.Linear(self.n_features, CFG.N_CLASSES)\n        elif 'resnext' in model_name:\n            self.n_features = self.model.fc.in_features\n            self.model.fc = nn.Linear(self.n_features, CFG.N_CLASSES)\n        self.criterion = GetCriterion(CFG.criterion_name)\n        self.val_criterion = GetCriterion(CFG.criterion_name)\n    \n    def forward(self, x):\n        return self.model(x)\n    \n    \n    def freeze(self):\n        # To freeze the residual layers\n        for param in self.model.parameters():\n            param.requires_grad = False\n            \n        if 'efficientnet' in self.model_name:\n            for param in self.model.classifier.parameters():\n                param.requires_grad = True\n        elif self.model_name == 'vit_large_patch16_384' or 'deit_base_patch16_224':\n            for param in self.model.head.parameters():\n                param.requires_grad = True\n        elif 'resnext' in self.model_name:\n            for param in self.model.fc.parameters():\n                param.requires_grad = True\n            \n    def unfreeze(self):\n        # Unfreeze all layers\n        for param in self.model.parameters():\n            param.requires_grad = True\n        \n    def prepare_data(self):\n        # Even in multi-GPU training. this method is called only from a single GPU. \n        # So this method ideal for download, stratification etc.     \n        pass\n        \n    \n    def setup(self, stage=None):\n        # In multi-GPU training, this method is run on each GPU. \n        # So ideal for each training/valid split\n        data = pd.read_csv(\"train_folds.csv\")\n        \n        training_data = data[data.kfold != self.fold]\n        training_data = training_data.drop(['kfold'], axis=1)\n        validation_data = data[data.kfold == self.fold]\n        validation_data = validation_data.drop(['kfold'], axis=1)\n        self.train_dataset = CasavaDataset(DATA_FOLD, training_data['image_id'].values, training_data['label'].values, self.train_transforms)\n        \n        self.valid_dataset =  CasavaDataset(DATA_FOLD, validation_data['image_id'].values, validation_data['label'].values, self.train_transforms)\n    \n    def training_step(self, batch, batch_index):\n        images, labels = batch\n        if self.trainer.current_epoch == CFG.START_FREEZE:\n            self.freeze()\n            print('freeze the model')\n        # forward pass on a batch\n        logits = self(images)\n        \n        # identifying number of correct predections in a given batch\n        correct=logits.argmax(dim=1).eq(labels).sum().item()\n        \n        # identifying total number of labels in a given batch\n        total=len(labels)\n\n        # calculating the loss\n        train_loss = self.criterion(logits, labels)\n        \n        \n        # logs- a dictionary\n        logs={\"train_loss\": train_loss}\n\n        batch_dictionary={\n            #REQUIRED: It ie required for us to return \"loss\"\n            \"loss\": train_loss,\n             \n            #optional for batch logging purposes\n#             \"log\": logs,\n            # info to be used at epoch end\n            \"correct\": correct,\n            \"total\": total,\n            \"progress_bar\": logs\n        }\n\n        return batch_dictionary\n    \n    def validation_step(self, batch, batch_index):\n        images, labels = batch\n        logits = self(images)\n        val_loss = self.val_criterion(logits, labels)\n\n         # identifying number of correct predections in a given batch\n        correct=logits.argmax(dim=1).eq(labels).sum().item()\n        \n        # identifying total number of labels in a given batch\n        total=len(labels)\n          # logs- a dictionary\n        logs={\"val_loss\": val_loss}\n\n        batch_dictionary={\n            #REQUIRED: It ie required for us to return \"loss\"\n            \"loss\": val_loss,   \n            #optional for batch logging purposes\n#             \"log\": logs,\n            # info to be used at epoch end\n            \"correct\": correct,\n            \"total\": total,\n            \"progress_bar\": logs\n        }\n\n        return batch_dictionary\n    \n\n    def configure_optimizers(self):\n        optimizer = GetOptimizer(CFG.optimizer_name, self.model.parameters())\n        return optimizer\n    \n    def train_dataloader(self):\n        return DataLoader(self.train_dataset, CFG.BATCH_SIZE, num_workers=CFG.NUM_WORKERS, shuffle=True)\n    \n    def val_dataloader(self):\n        return DataLoader(self.valid_dataset, CFG.BATCH_SIZE, num_workers=CFG.NUM_WORKERS, shuffle=False) \n    \n    def training_epoch_end(self, outputs):\n        #  the function is called after every epoch is completed\n        \n        # calculating average loss \n        avg_loss = torch.stack([x['loss'] for x in outputs]).mean()\n        \n        # calculating correect and total predictions\n        correct=sum([x[\"correct\"] for  x in outputs])\n        total=sum([x[\"total\"] for  x in outputs])\n\n        # creating log dictionary\n         # logging using tensorboard logger\n        self.logger.experiment.add_scalar(\"Loss/Train\",\n                                            avg_loss,\n                                            self.current_epoch)\n         \n        self.logger.experiment.add_scalar(\"Accuracy/Train\",\n                                            correct/total,\n                                            self.current_epoch)\n\n\n        epoch_dictionary={\n            # required\n            'loss': avg_loss\n        }\n        \n        return epoch_dictionary\n\n    \n    def validation_epoch_end(self, outputs):\n         # calculating average loss \n        avg_loss = torch.stack([x['loss'] for x in outputs]).mean()\n        \n        # calculating correect and total predictions\n        correct=sum([x[\"correct\"] for  x in outputs])\n        total=sum([x[\"total\"] for  x in outputs])\n\n        # creating log dictionary\n         # logging using tensorboard logger\n        self.logger.experiment.add_scalar(\"Loss/Val\",\n                                            avg_loss,\n                                            self.current_epoch)\n         \n        self.logger.experiment.add_scalar(\"Accuracy/Val\",\n                                            correct/total,\n                                            self.current_epoch)\n\n\n        epoch_dictionary={\n            # required\n            'loss': avg_loss\n        }\n        \n        return epoch_dictionary","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Five fold training. \nif Training:\n    \n    for fold in range(5):  \n\n        checkpoint_callback = ModelCheckpoint(\n            filepath='./ng_models_effnet/'+str(fold)+'/model_fold-{epoch:02d}-loss-{val_loss}', \n            monitor='val_loss', verbose=True, \n            save_last=False, save_top_k=1, save_weights_only=False, \n            mode='min', period=1, prefix='')\n\n        tb_logger = pl_loggers.TensorBoardLogger('ng_logs_effnet/'+str(fold)+'/')\n\n        trainer = pl.Trainer(logger=tb_logger, gpus=1, max_epochs=15, checkpoint_callback=checkpoint_callback) #, accelerator ='ddp')\n    #     dm = MoADataModule(fold=k)\n        model = CassavaModel(fold=fold) # Input Features, Output Targets\n        trainer.fit(model)\n\n        print(checkpoint_callback.best_model_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL_PATH = '../input/effnet-resnet-weights/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Inference\nif not Training:\n    # ====================================================\n    # Model Loading\n    # ====================================================\n    models = []\n    count = 0\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    MODEL_PATHS = ['tf_efficientnet_b4_ns_fold_0.ckpt','tf_efficientnet_b4_ns_fold_1.ckpt','tf_efficientnet_b4_ns_fold_2.ckpt','tf_efficientnet_b4_ns_fold_3.ckpt','tf_efficientnet_b4_ns_fold_4.ckpt']\n    for model_fpath in MODEL_PATHS: #os.listdir(MODEL_PATH):\n#         if count in CFG.TRAIN_FOLDS:\n        print(\"Model Loaded:\",model_fpath)\n        model_name_split = model_fpath.split('_f')[0]\n        model = CassavaModel(model_name=model_name_split,  pretrained=False) \n#         model = CassavaNet(model_name_split,pretrained = False)\n        info = torch.load(MODEL_PATH + model_fpath,map_location = torch.device(device))\n        print(info.keys())\n#             model.load_from_checkpoint(checkpoint_path=MODEL_PATH + model_fpath)\n        model.load_state_dict(info['state_dict'])\n        models.append(model)\n        count+=1\n\n    \n#     submission = pd.DataFrame()\n#     list_files = os.listdir(TRAIN_DIR) #TEST_DIR)\n#     submission['image_id'] = pd.Series(list_files)\n#     submission['label'] =0\n\n    test_df =  pd.read_csv(DATA_PATH + 'sample_submission.csv')\n   \n   \n\n    print(test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission = submission.sample(50).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not Training:\n    # ====================================================\n    # TTA\n    # ====================================================\n    BATCH_SIZE = 1\n    test_set = CasavaDataset(TEST_DIR, test_df['image_id'].values, test_df['label'].values, None)\n    image_ids = []\n    predicts = []\n    # test_set = GetData(TEST_DIR,submission['image_id'], submission['label'], Type = 'test')\n    test_loader = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=False, num_workers=1,pin_memory = True)\n    with torch.no_grad():\n        for i, (images,labels, image_id) in enumerate(tqdm(test_loader)):\n            voting = np.zeros((len(models),CFG.N_TTA,CFG.N_CLASSES))\n            aug_images = np.zeros((CFG.N_TTA,CFG.CHANNELS,CFG.HEIGHT,CFG.WIDTH))\n            for aug_no in range(CFG.N_TTA):\n                img_np = images.numpy()\n\n                aug_data = CFG.heavy_transforms(image = np.reshape(img_np,(600,800,CFG.CHANNELS)))\n               \n            aug_images[aug_no,:,:,:] = aug_data['image'].numpy()\n            aug_images = torch.from_numpy(aug_images).to(torch.float32).to(device)\n            for model_no in range(len(models)):\n                model = models[model_no]\n                model = model.to(device)\n                model.eval()            \n\n                logits = model(aug_images)\n                voting[model_no,:,:] = F.softmax(logits).cpu().numpy()\n\n            voting = np.sum(voting,axis = 1) / CFG.N_TTA\n            voting = np.sum(voting,axis = 0) / len(models)\n            label = np.argmax(voting)\n            image_ids.extend(image_id)\n            predicts.append(label)\n    submission_df = pd.DataFrame(\n                {'image_id': image_ids,\n                 'label': predicts\n                })\n#             submission['predict'].loc[submission['image_id'] == labels[0]] = label\n\n\n    submission_df.to_csv('submission.csv',index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cat submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}