{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is a notebook makes it easy to test with the following implementations:\nOptimizer : Adam, Ranger\nScheduler : ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts', OneCycleLR]\nModels    : Any Timm Model\nLoss Fn   : ['CrossEntropyLoss', LabelSmoothing', 'FocalLoss','FocalCosineLoss', 'SymmetricCrossEntropyLoss', 'BiTemperedLoss', 'TaylorCrossEntropyLoss']\n\nthanks to https://www.kaggle.com/piantic/train-cassava-starter-using-various-loss-funcs/data and @piantic for sharing all the loss functions\n\nAny feedback is appreciated!"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Required Installations\n# ====================================================\n!pip install --quiet timm\n!pip install --quiet '../input/pytorch-ranger'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Library"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport random\nimport os\nimport math\nimport time\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import imread\nimport numpy as np\nimport cv2\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\n\nimport torch\nimport torch.nn as nn\nimport torchvision\nfrom torchvision import models as tvmodels\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, utils\nimport torch.nn.functional as F\nfrom tqdm import tqdm\nimport timm\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import KFold,StratifiedKFold\n\nimport albumentations as A\nfrom albumentations import Compose\nfrom albumentations.pytorch import ToTensorV2\n\n\nfrom PIL import Image, ImageOps, ImageEnhance, ImageChops\n\nfrom pytorch_ranger import Ranger\n\n# Ignore warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Run Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Configurations\n# ====================================================\nclass CFG:\n    #Model Params\n    N_FOLDS = 5\n    MODEL_NAME = 'tf_efficientnet_b4_ns' # Recommended : ['tf_efficientnet_b3_ns','tf_efficientnet_b4_ns',resnext50_32x4d']\n    pretrained = True   \n    N_CLASSES = 5\n    \n    N_TTA = 4\n    TEST_TTA = True\n    \n    #Image Size\n    HEIGHT = 256\n    WIDTH = 256\n    CHANNELS = 3\n    \n    #Training Params\n    BATCH_SIZE = 32\n    EPOCHS = 3\n    LR = 5e-4\n    LR_MIN = 1e-6\n    weight_decay = 1e-6\n    eps = 1e-8\n    PATIENCE = 4\n    \n    #BiTemperedLoss\n    T1 = 0.8\n    T2 = 1.3\n    LABEL_SMOOTH = 0.2\n    \n    #CosineAnnealingWarmRestarts\n    T_0 = 10\n    \n    #CosineAnnealingLR\n    T_max = 10\n    \n    NUM_WORKERS = os.cpu_count()\n    \n    scheduler_name = 'OneCycleLR' # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts', OneCycleLR]\n    criterion_name = 'BiTemperedLoss' # ['CrossEntropyLoss', LabelSmoothing', 'FocalLoss','FocalCosineLoss', 'SymmetricCrossEntropyLoss', 'BiTemperedLoss', 'TaylorCrossEntropyLoss']\n    optimizer_name = 'Ranger' #['Adam','Ranger']\n    \n    #TRAIN_FOLDS = [0,1,2,3,4]\n    TRAIN_FOLDS = [0] #Folds to be Trained\n    model_print = False #If the model architecture is printed\n    TRAIN_AUG_TYPE = 'autoaugment' #['train','lightaug','autoaugment']\n    VALID_AUG_TYPE = 'valid' #['valid']\n    \n    IMG_MEAN = [0.485, 0.456, 0.406] #Mean for normalization Transform\n    IMG_STD = [0.229, 0.224, 0.225] #STD for normalization Transform\n    SEED = 1234","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss Functions"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def log_t(u, t):\n    \"\"\"Compute log_t for `u'.\"\"\"\n    if t==1.0:\n        return u.log()\n    else:\n        return (u.pow(1.0 - t) - 1.0) / (1.0 - t)\n\ndef exp_t(u, t):\n    \"\"\"Compute exp_t for `u'.\"\"\"\n    if t==1:\n        return u.exp()\n    else:\n        return (1.0 + (1.0-t)*u).relu().pow(1.0 / (1.0 - t))\n\ndef compute_normalization_fixed_point(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t > 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same shape as activation with the last dimension being 1.\n    \"\"\"\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations_step_0 = activations - mu\n\n    normalized_activations = normalized_activations_step_0\n\n    for _ in range(num_iters):\n        logt_partition = torch.sum(\n                exp_t(normalized_activations, t), -1, keepdim=True)\n        normalized_activations = normalized_activations_step_0 * \\\n                logt_partition.pow(1.0-t)\n\n    logt_partition = torch.sum(\n            exp_t(normalized_activations, t), -1, keepdim=True)\n    normalization_constants = - log_t(1.0 / logt_partition, t) + mu\n\n    return normalization_constants\n\ndef compute_normalization_binary_search(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t < 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (< 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations = activations - mu\n\n    effective_dim = \\\n        torch.sum(\n                (normalized_activations > -1.0 / (1.0-t)).to(torch.int32),\n            dim=-1, keepdim=True).to(activations.dtype)\n\n    shape_partition = activations.shape[:-1] + (1,)\n    lower = torch.zeros(shape_partition, dtype=activations.dtype, device=activations.device)\n    upper = -log_t(1.0/effective_dim, t) * torch.ones_like(lower)\n\n    for _ in range(num_iters):\n        logt_partition = (upper + lower)/2.0\n        sum_probs = torch.sum(\n                exp_t(normalized_activations - logt_partition, t),\n                dim=-1, keepdim=True)\n        update = (sum_probs < 1.0).to(activations.dtype)\n        lower = torch.reshape(\n                lower * update + (1.0-update) * logt_partition,\n                shape_partition)\n        upper = torch.reshape(\n                upper * (1.0 - update) + update * logt_partition,\n                shape_partition)\n\n    logt_partition = (upper + lower)/2.0\n    return logt_partition + mu\n\nclass ComputeNormalization(torch.autograd.Function):\n    \"\"\"\n    Class implementing custom backward pass for compute_normalization. See compute_normalization.\n    \"\"\"\n    @staticmethod\n    def forward(ctx, activations, t, num_iters):\n        if t < 1.0:\n            normalization_constants = compute_normalization_binary_search(activations, t, num_iters)\n        else:\n            normalization_constants = compute_normalization_fixed_point(activations, t, num_iters)\n\n        ctx.save_for_backward(activations, normalization_constants)\n        ctx.t=t\n        return normalization_constants\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        activations, normalization_constants = ctx.saved_tensors\n        t = ctx.t\n        normalized_activations = activations - normalization_constants \n        probabilities = exp_t(normalized_activations, t)\n        escorts = probabilities.pow(t)\n        escorts = escorts / escorts.sum(dim=-1, keepdim=True)\n        grad_input = escorts * grad_output\n        \n        return grad_input, None, None\n\ndef compute_normalization(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example. \n    Backward pass is implemented.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    return ComputeNormalization.apply(activations, t, num_iters)\n\ndef tempered_sigmoid(activations, t, num_iters = 5):\n    \"\"\"Tempered sigmoid function.\n    Args:\n      activations: Activations for the positive class for binary classification.\n      t: Temperature tensor > 0.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_probabilities = tempered_softmax(internal_activations, t, num_iters)\n    return internal_probabilities[..., 0]\n\n\ndef tempered_softmax(activations, t, num_iters=5):\n    \"\"\"Tempered softmax function.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature > 1.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    if t == 1.0:\n        return activations.softmax(dim=-1)\n\n    normalization_constants = compute_normalization(activations, t, num_iters)\n    return exp_t(activations - normalization_constants, t)\n\ndef bi_tempered_binary_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing = 0.0,\n        num_iters=5,\n        reduction='mean'):\n\n    \"\"\"Bi-Tempered binary logistic loss.\n    Args:\n      activations: A tensor containing activations for class 1.\n      labels: A tensor with shape as activations, containing probabilities for class 1\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A loss tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_labels = torch.stack([labels.to(activations.dtype),\n        1.0 - labels.to(activations.dtype)],\n        dim=-1)\n    return bi_tempered_logistic_loss(internal_activations, \n            internal_labels,\n            t1,\n            t2,\n            label_smoothing = label_smoothing,\n            num_iters = num_iters,\n            reduction = reduction)\n\ndef bi_tempered_logistic_loss(activations,\n        labels,\n        t1 = CFG.T1,\n        t2 = CFG.T2,\n        label_smoothing=CFG.LABEL_SMOOTH,\n        num_iters=5,\n        reduction = 'mean'):\n\n    \"\"\"Bi-Tempered Logistic Loss.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      labels: A tensor with shape and dtype as activations (onehot), \n        or a long tensor of one dimension less than activations (pytorch standard)\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing parameter between [0, 1). Default 0.0.\n      num_iters: Number of iterations to run the method. Default 5.\n      reduction: ``'none'`` | ``'mean'`` | ``'sum'``. Default ``'mean'``.\n        ``'none'``: No reduction is applied, return shape is shape of\n        activations without the last dimension.\n        ``'mean'``: Loss is averaged over minibatch. Return shape (1,)\n        ``'sum'``: Loss is summed over minibatch. Return shape (1,)\n    Returns:\n      A loss tensor.\n    \"\"\"\n\n    if len(labels.shape)<len(activations.shape): #not one-hot\n        labels_onehot = torch.zeros_like(activations)\n        labels_onehot.scatter_(1, labels[..., None], 1)\n    else:\n        labels_onehot = labels\n\n    if label_smoothing > 0:\n        num_classes = labels_onehot.shape[-1]\n        labels_onehot = ( 1 - label_smoothing * num_classes / (num_classes - 1) ) \\\n                * labels_onehot + \\\n                label_smoothing / (num_classes - 1)\n\n    probabilities = tempered_softmax(activations, t2, num_iters)\n\n    loss_values = labels_onehot * log_t(labels_onehot + 1e-10, t1) \\\n            - labels_onehot * log_t(probabilities, t1) \\\n            - labels_onehot.pow(2.0 - t1) / (2.0 - t1) \\\n            + probabilities.pow(2.0 - t1) / (2.0 - t1)\n    loss_values = loss_values.sum(dim = -1) #sum over classes\n\n    if reduction == 'none':\n        return loss_values\n    if reduction == 'sum':\n        return loss_values.sum()\n    if reduction == 'mean':\n        return loss_values.mean()\n\nclass BiTemperedLogistic(nn.Module):\n    def __init__(self, T1 = CFG.T1, T2 = CFG.T2, LABEL_SMOOTH = CFG.LABEL_SMOOTH):\n        super().__init__()\n        self.T1 = T1\n        self.T2 = T2\n        self.LABEL_SMOOTH = LABEL_SMOOTH\n\n    def forward(self, logits,labels):\n        return bi_tempered_logistic_loss(logits, labels,t1 = self.T1,t2 = self.T2, label_smoothing = self.LABEL_SMOOTH)\n    \nclass SymmetricCrossEntropy(nn.Module):\n\n    def __init__(self, alpha=0.1, beta=1.0, num_classes= 5):\n        super(SymmetricCrossEntropy, self).__init__()\n        self.alpha = alpha\n        self.beta = beta\n        self.num_classes = num_classes\n\n    def forward(self, logits, targets, reduction='mean'):\n        onehot_targets = torch.eye(self.num_classes)[targets].cuda()\n        ce_loss = F.cross_entropy(logits, targets, reduction=reduction)\n        rce_loss = (-onehot_targets*logits.softmax(1).clamp(1e-7, 1.0).log()).sum(1)\n        if reduction == 'mean':\n            rce_loss = rce_loss.mean()\n        elif reduction == 'sum':\n            rce_loss = rce_loss.sum()\n        return self.alpha * ce_loss + self.beta * rce_loss\n    \nclass LabelSmoothingLoss(nn.Module): \n    def __init__(self, classes=5, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))\n    \nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, reduce=True):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduce = reduce\n\n    def forward(self, inputs, targets):\n        BCE_loss = nn.CrossEntropyLoss()(inputs, targets)\n\n        pt = torch.exp(-BCE_loss)\n        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n\n        if self.reduce:\n            return torch.mean(F_loss)\n        else:\n            return F_loss\n        \nclass FocalCosineLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, xent=.1):\n        super(FocalCosineLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n        self.xent = xent\n\n        self.y = torch.Tensor([1]).cuda()\n\n    def forward(self, input, target, reduction=\"mean\"):\n        cosine_loss = F.cosine_embedding_loss(input, F.one_hot(target, num_classes=input.size(-1)), self.y, reduction=reduction)\n\n        cent_loss = F.cross_entropy(F.normalize(input), target, reduce=False)\n        pt = torch.exp(-cent_loss)\n        focal_loss = self.alpha * (1-pt)**self.gamma * cent_loss\n\n        if reduction == \"mean\":\n            focal_loss = torch.mean(focal_loss)\n\n        return cosine_loss + self.xent * focal_loss\n    \nclass TaylorSoftmax(nn.Module):\n    '''\n    This is the autograd version\n    '''\n    def __init__(self, dim=1, n=2):\n        super(TaylorSoftmax, self).__init__()\n        assert n % 2 == 0\n        self.dim = dim\n        self.n = n\n\n    def forward(self, x):\n        '''\n        usage similar to nn.Softmax:\n            >>> mod = TaylorSoftmax(dim=1, n=4)\n            >>> inten = torch.randn(1, 32, 64, 64)\n            >>> out = mod(inten)\n        '''\n        fn = torch.ones_like(x)\n        denor = 1.\n        for i in range(1, self.n+1):\n            denor *= i\n            fn = fn + x.pow(i) / denor\n        out = fn / fn.sum(dim=self.dim, keepdims=True)\n        return out\n    \nclass TaylorCrossEntropyLoss(nn.Module):\n    '''\n    This is the autograd version\n    '''\n    def __init__(self, n=2, ignore_index=-1, reduction='mean'):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n\n    def forward(self, logits, labels):\n        '''\n        usage similar to nn.CrossEntropyLoss:\n            >>> crit = TaylorCrossEntropyLoss(n=4)\n            >>> inten = torch.randn(1, 10, 64, 64)\n            >>> label = torch.randint(0, 10, (1, 64, 64))\n            >>> out = crit(inten, label)\n        '''\n        log_probs = self.taylor_softmax(logits).log()\n        loss = F.nll_loss(log_probs, labels, reduction=self.reduction,\n                ignore_index=self.ignore_index)\n        return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef retrieve_df(df,name,idx):\n    series = df[name].iloc[idx]\n    series.reset_index(drop=True,inplace=True)\n    return series\n\ndef accuracy_metric(input, targs):\n    return accuracy_score(targs.cpu(), input.cpu())\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \n#Choose Criterions for the Training Loop\nif CFG.criterion_name == 'BiTemperedLoss':\n    CFG.criterion = BiTemperedLogistic()\n    CFG.val_criterion = BiTemperedLogistic()\nelif CFG.criterion_name == 'SymmetricCrossEntropyLoss':\n    CFG.criterion = SymmetricCrossEntropy()\n    CFG.val_criterion = SymmetricCrossEntropy()\nelif CFG.criterion_name == 'CrossEntropyLoss':\n    CFG.criterion = nn.CrossEntropyLoss()\n    CFG.val_criterion = nn.CrossEntropyLoss()\nelif CFG.criterion_name == 'LabelSmoothingLoss':\n    CFG.criterion = LabelSmoothingLoss()\n    CFG.val_criterion = LabelSmoothingLoss()\nelif CFG.criterion_name == 'FocalLoss':\n    CFG.criterion = FocalLoss()\n    CFG.val_criterion = FocalLoss()\nelif CFG.criterion_name == 'FocalCosineLoss':\n    CFG.criterion = FocalCosineLoss()\n    CFG.val_criterion = FocalCosineLoss()\nelif CFG.criterion_name == 'TaylorCrossEntropyLoss':\n    CFG.criterion = TaylorCrossEntropyLoss()\n    CFG.val_criterion = TaylorCrossEntropyLoss()\n    \ndef GetScheduler(scheduler_name,optimizer):\n    #['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts', 'OneCycleLR']\n    if scheduler_name == 'OneCycleLR':\n        return torch.optim.lr_scheduler.OneCycleLR(optimizer,max_lr = 1e-2,epochs = CFG.EPOCHS,steps_per_epoch = batches+1,pct_start = 0.3)\n    elif scheduler_name == 'CosineAnnealingWarmRestarts':\n        return torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0 = CFG.T_0, T_mult=1, eta_min=0, last_epoch=-1)\n    elif scheduler_name == 'CosineAnnealingLR':\n        return torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max = CFG.T_max, eta_min=0, last_epoch=-1)\n    elif scheduler_name == 'ReduceLROnPlateau':\n        return torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,factor=0.1, patience=1, threshold=0.0001, cooldown=0, min_lr=CFG.LR_MIN, eps=CFG.eps)\n    \ndef GetOptimizer(optimizer_name,parameters):\n    #['Adam','Ranger']\n    if optimizer_name == 'Adam':\n        return torch.optim.Adam(parameters, lr=CFG.LR, weight_decay=CFG.weight_decay, amsgrad=False)\n    elif optimizer_name == 'Ranger':\n        return Ranger(parameters,lr = CFG.LR,alpha = 0.5, k = 6,N_sma_threshhold = 5,betas = (0.95,0.999),eps=CFG.eps,weight_decay=CFG.weight_decay)\n\nSEED = CFG.SEED\nseed_everything(SEED)  \nDEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Datasets\n# ====================================================\nclass GetData(Dataset):\n    def __init__(self, Dir, FNames, labels,Type):\n        self.dir = Dir\n        self.fnames = FNames\n        self.lbs = labels\n        self.type = Type\n        self.auto_augment = timm.data.auto_augment.auto_augment_transform('originalr',None)\n        \n    def __len__(self):\n        return len(self.fnames)\n\n    def __getitem__(self, index):\n        if \"train\" in self.type:\n            x = imread(os.path.join(self.dir, self.fnames[index]))\n            aug_data = train_transforms(image = x)\n            return aug_data['image'], self.lbs[index]\n        if \"lightaug\" in self.type:\n            x = imread(os.path.join(self.dir, self.fnames[index]))\n            aug_data = light_transforms(image = x)\n            return aug_data['image'], self.lbs[index]\n        elif \"autoaugment\" in self.type:\n            x = Image.open(os.path.join(self.dir, self.fnames[index]))\n            aug_image = self.auto_augment(x)\n            aug_data = image_net_post(image = np.asarray(aug_image,dtype = np.float32))\n            return aug_data['image'], self.lbs[index]\n        elif \"valid\" in self.type:\n            x = imread(os.path.join(self.dir, self.fnames[index]))\n            aug_data = valid_transforms(image = x)\n            return aug_data['image'], self.lbs[index]\n        elif \"tr-tst\" in self.type:\n            x = imread(os.path.join(self.dir, self.fnames[index]))\n            return x, self.lbs[index]\n        elif \"test\" in self.type:\n            x = imread(os.path.join(self.dir, self.fnames[index]))\n            return x, self.fnames[index]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CV SPLIT"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# CV Split\n# ====================================================\nDATA_PATH = '../input/cassava-leaf-disease-classification/'\nTRAIN_DIR = DATA_PATH + 'train_images/'\nDATA_PATH_2019 = '../input/cassava-leaf-disease-merged/'\nTRAIN_DIR_2019 = DATA_PATH_2019 + 'train/'\nTEST_DIR = DATA_PATH + 'test_images/'\n\n#This guarantees that no images from 2019 contaminate the validation split\ntrain_df = pd.read_csv(DATA_PATH + 'train.csv')\ntrain_df_merged = pd.read_csv(DATA_PATH_2019 + 'merged.csv')\ntrain_df_2019 = train_df_merged.loc[train_df_merged.source == 2019]\nskf = StratifiedKFold(n_splits=CFG.N_FOLDS, shuffle=True, random_state=CFG.SEED)\nskf.get_n_splits(np.arange(train_df.shape[0]), train_df['label'])\nfolds = [(idxT,idxV) for i,(idxT,idxV) in enumerate(skf.split(np.arange(train_df.shape[0]), train_df['label']))]\nfolds_2019 = [np.concatenate((idxT,idxV)) for i,(idxT,idxV) in enumerate(skf.split(np.arange(train_df_2019.shape[0]), train_df_2019['label']))]\nfor i in range(CFG.N_FOLDS):\n    (idxT,idxV) = folds[i]\n    folds[i] = (np.concatenate((idxT,folds_2019[i])),idxV)\n    (idxT,idxV) = folds[i]\n    print(np.bincount(train_df_merged['label'].iloc[idxT]),np.bincount(train_df['label'].iloc[idxV]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Augmentations\n# ====================================================\nAug_Norm = A.Normalize(mean=CFG.IMG_MEAN, std=CFG.IMG_STD, max_pixel_value=255.0, p=1.0)\nDrop_Rand = A.CoarseDropout(max_holes=16, max_height=int(0.11*CFG.HEIGHT), max_width=int(0.11*CFG.WIDTH),\n                            min_holes=4, min_height=int(0.08*CFG.HEIGHT), min_width=int(0.08*CFG.WIDTH),\n                            fill_value=0, mask_fill_value=0, always_apply=False, p=0.4)\nRand_Crop = A.RandomCrop(height= CFG.HEIGHT, width = CFG.WIDTH,always_apply=True, p=1.0)\nResize_Crop = A.RandomResizedCrop(CFG.HEIGHT, CFG.WIDTH,p=1.0)\ntrain_transforms = Compose([\n            A.Transpose(p=0.5),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.ColorJitter(brightness=0.10, contrast=0.2, saturation=0.2, hue=0.00, always_apply=False, p=0.5),\n            A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.5),\n            Resize_Crop,\n            Drop_Rand,\n            Aug_Norm,   \n            ToTensorV2(p=1.0),\n        ], p=1.)\n\nlight_transforms = Compose([\n            A.Transpose(p=0.5),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.5),\n            Resize_Crop,\n            Aug_Norm,   \n            ToTensorV2(p=1.0),\n        ], p=1.)\n\nvalid_transforms = Compose([\n            Resize_Crop,\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            Aug_Norm,   \n            ToTensorV2(p=1.0),\n        ], p=1.)\n\ntest_aug = Compose([\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.ShiftScaleRotate(p = 1.0),\n            A.ColorJitter(brightness=0.1, contrast=0.2, saturation=0.2, hue=0.00, always_apply=False, p=1.0),\n            Rand_Crop,\n            Aug_Norm,\n            ToTensorV2(p=1.0)\n        ], p=1.)\n\nimage_net_post = Compose([\n            Rand_Crop,\n            Drop_Rand,\n            Aug_Norm,    \n            ToTensorV2(p=1.0)\n        ], p=1.)\n\nclass UnNormalize(object):\n    def __init__(self, mean, std):\n        self.mean = mean\n        self.std = std\n\n    def __call__(self, tensor):\n        \"\"\"\n        Args:\n            tensor (Tensor): Tensor image of size (C, H, W) to be normalized.\n        Returns:\n            Tensor: Normalized image.\n        \"\"\"\n        for t, m, s in zip(tensor, self.mean, self.std):\n            t.mul_(s).add_(m)\n            # The normalize code -> t.sub_(m).div_(s)\n        return tensor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 10))\ninv_normalize = UnNormalize(mean = CFG.IMG_MEAN,std = CFG.IMG_STD)\nfor i in range(16):\n    ax = plt.subplot(4, 4, i + 1)\n    image = imread(TRAIN_DIR+train_df['image_id'].iloc[i])\n    aug_image = train_transforms(image = image)['image']\n    aug_image = inv_normalize(aug_image)\n    aug_image = np.transpose(aug_image.numpy(),[1,2,0])\n    plt.imshow(aug_image)\n    label = train_df['label'].iloc[i]\n    plt.title(label)\n    plt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Model\n# ====================================================\nclass CassavaNet(nn.Module):\n    def __init__(self, model_name=CFG.MODEL_NAME, pretrained=CFG.pretrained):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        try:\n            self.n_features = self.model.fc.in_features\n        except:\n            self.n_features = list(self.model.children())[-1].in_features\n        self.model = torch.nn.Sequential(*(list(self.model.children())[:-1]))\n        self.drop_fc = nn.Dropout(p = 0.3)\n        self.out = nn.Linear(self.n_features, CFG.N_CLASSES)\n        \n    def forward(self, x):\n        x = self.model(x)\n        x = self.drop_fc(x)\n        return self.out(x)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"model = CassavaNet()\nif CFG.model_print:\n    print(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Train Loop\n# ====================================================\ntorch.cuda.empty_cache()\nfor fold,(idxT, idxV) in enumerate(folds):\n    if fold not in CFG.TRAIN_FOLDS:\n        continue\n    print(fold)\n    #______INSTANTIATE TRAINING DATASETS_____\n    x_train = retrieve_df(train_df_merged,'image_id',idxT)\n    y_train = retrieve_df(train_df_merged,'label',idxT)\n    x_val = retrieve_df(train_df,'image_id',idxV)\n    y_val = retrieve_df(train_df,'label',idxV)\n    train_set = GetData(TRAIN_DIR, x_train, y_train, Type = CFG.TRAIN_AUG_TYPE)\n    train_loader = DataLoader(train_set, batch_size=CFG.BATCH_SIZE, shuffle=True, num_workers=CFG.NUM_WORKERS,pin_memory = True)\n    val_set = GetData(TRAIN_DIR, x_val, y_val, Type = CFG.VALID_AUG_TYPE)\n    val_loader = DataLoader(val_set, batch_size=CFG.BATCH_SIZE, shuffle=True, num_workers=CFG.NUM_WORKERS,pin_memory = True)\n    batches = math.floor(len(x_train)/CFG.BATCH_SIZE)\n    val_batches = math.floor(len(x_val)/CFG.BATCH_SIZE)\n    \n    #INSTANTIATE FOLD MODEL\n    model = CassavaNet(model_name=CFG.MODEL_NAME, pretrained=True).to(DEVICE)\n    criterion = CFG.criterion.to(DEVICE)\n    val_criterion = CFG.val_criterion.to(DEVICE)\n    \n    optimizer = GetOptimizer(CFG.optimizer_name, model.parameters())\n    scheduler = GetScheduler(CFG.scheduler_name, optimizer)\n    \n    saved_model = None\n    best_val_acc = 0.0\n    best_val_loss = 1e3\n    fold_patience = 0.0\n    for epoch in range(CFG.EPOCHS):\n        #______TRAINING______\n        tr_loss = 0.0\n        cur_step = 0\n        scores = []\n        model.train()\n        progress = tqdm(enumerate(train_loader), desc=\"Loss: \", total=batches)\n        for i, (images,labels) in progress:\n            images = images.to(DEVICE)\n            labels = labels.to(DEVICE)\n\n            logits = model(images)\n            loss = criterion(logits, labels)\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            \n            preds = F.softmax(logits).argmax(axis = 1)\n            scores.append(accuracy_metric(preds,labels))\n\n            tr_loss += loss.detach().item()\n            cur_step = i+1\n            trn_epoch_result = dict()\n            trn_epoch_result['Epoch'] = epoch\n            trn_epoch_result['train_loss'] = round(tr_loss/cur_step, 4)\n            trn_epoch_result['train_acc'] = round(np.mean(scores), 4)\n            \n            scheduler.step()\n            progress.set_description(str(trn_epoch_result))\n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()\n        #______VALIDATION_______\n        val_loss = 0.0\n        val_bi_loss = 0.0\n        cur_step = 0\n        scores = []\n        model.eval()\n        val_progress = tqdm(enumerate(val_loader), desc=\"Loss: \", total=val_batches)\n        with torch.no_grad():\n            for i, (images,labels) in val_progress:\n                images = images.to(DEVICE)\n                labels = labels.to(DEVICE)\n\n                logits = model(images)\n                val_loss_value = val_criterion(logits,labels)\n                val_loss += val_loss_value.detach().item()\n\n                preds = F.softmax(logits).argmax(axis = 1)\n                scores.append(accuracy_metric(preds,labels))\n\n                cur_step = i + 1\n                val_epoch_result = dict()\n                val_epoch_result['Epoch'] = epoch\n                val_epoch_result['val_loss'] = round(val_loss/cur_step, 4)\n\n                val_epoch_result['val_acc'] = round(np.mean(scores), 4)\n                val_progress.set_description(str(val_epoch_result))\n        \n        if val_loss/cur_step < best_val_loss or np.mean(scores) > best_val_acc:\n            fold_patience = 0\n            best_val_loss = val_loss/cur_step\n            best_val_acc = np.mean(scores)\n            torch.save({'model': model.state_dict(), \n                        'preds': preds},\n                        f'{CFG.MODEL_NAME}_f{fold}_b{round(np.mean(scores), 4)}.pth')\n            if saved_model is not None:\n                os.remove(\"./\"+saved_model)\n            saved_model = f'{CFG.MODEL_NAME}_f{fold}_b{round(np.mean(scores), 4)}.pth'\n            print(f'Model Saved at {round(np.mean(scores), 5)} accuracy')\n        else:\n            fold_patience += 1\n            if fold_patience >= CFG.PATIENCE:\n                print(f'Early stopping due to model not improving for {CFG.PATIENCE} epochs')\n                '''\n                torch.save({'model': model.state_dict(), \n                'preds': preds},\n                f'ResNext50_fold{fold}_ES.pth')\n                '''\n                break\n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# VTA Validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# TTA Validation\n# ====================================================\nif CFG.TEST_TTA:\n    submission = pd.DataFrame()\n    list_files = os.listdir(TRAIN_DIR)\n    submission['image_id'] = pd.Series(list_files)\n    submission['label'] = 0\n    models= []\n    best_model = CassavaNet()\n    info = torch.load(saved_model,map_location = torch.device(DEVICE))\n    best_model.load_state_dict(info['model'])\n    models.append(best_model)\n\n    #IF TTA AVERAGED\n    TEST = False\n    if not TEST:\n        start_time = time.time()\n        BATCH_SIZE = 1\n        val_set = GetData(TRAIN_DIR, x_val, y_val, Type = 'tr-tst')\n        val_loader = DataLoader(val_set, batch_size=BATCH_SIZE, shuffle=False, num_workers=8,pin_memory = True)\n\n        scores = []\n        count = {0:0,1:0,2:0,3:0,4:0}\n        N_TTA = CFG.N_TTA\n        right_count = {0:0,1:0,2:0,3:0,4:0}\n        val_tta = {\"Acc_0\":0,\"Acc_1\":0,\"Acc_2\":0,\"Acc_3\":0,\"Acc_4\":0,\"Acc_T\":0}\n        tta_progress = tqdm(enumerate(val_loader), desc=\"Loss: \", total=len(x_val))\n        confusion_matrix = np.zeros((CFG.N_CLASSES,CFG.N_CLASSES))\n        with torch.no_grad():\n            for i, (images,labels) in tta_progress:\n                voting = np.zeros((len(models),N_TTA,CFG.N_CLASSES))\n                aug_images = np.zeros((N_TTA,CFG.CHANNELS,CFG.HEIGHT,CFG.WIDTH))\n                for aug_no in range(N_TTA):\n                    img_np = images.numpy()\n\n                    aug_data = test_aug(image = np.reshape(img_np,(600,800,CFG.CHANNELS)))\n                    aug_images[aug_no,:,:,:] = aug_data['image'].numpy()\n                aug_images = torch.from_numpy(aug_images).to(torch.float32).to(DEVICE)\n                for model_no in range(len(models)):\n                    model = models[model_no]\n                    model = model.to(DEVICE)\n                    model.eval()            \n\n                    labels = labels.to(DEVICE)\n\n                    logits = model(aug_images)\n                    voting[model_no,:,:] = F.softmax(logits).cpu().numpy()\n\n                voting = np.sum(voting,axis = 1) / N_TTA\n                voting = np.sum(voting,axis = 0) / len(models)\n                label = np.argmax(voting)\n                count[int(labels[0].cpu().numpy())] += 1\n                confusion_matrix[labels[0].cpu().numpy(),label] += 1\n                if label == labels[0].cpu().numpy():\n                    right_count[label] += 1\n                try:\n                    val_tta['Acc_'+str(label)] = round(right_count[label]/count[label],3)\n                except:\n                    val_tta['Acc_'+str(label)] = 0\n                val_tta['Acc_T'] = round(sum([right_count[x] for x in range(5)])/sum([count[x] for x in range(5)]),4)\n                tta_progress.set_description(str(val_tta))\n        print(time.time()-start_time)\n        print(round(sum([right_count[x] for x in range(5)])/sum([count[x] for x in range(5)]),4))\n        new_cm = confusion_matrix\n        for i in range(5):\n            new_cm[i,:] = new_cm[i,:]/np.sum(new_cm[i,:]) \n        print(\"Confusion Matrix:\")\n        print(new_cm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}