{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Preliminaries\nfrom pathlib import Path\nfrom tqdm import tqdm\ntqdm.pandas()\nimport json\nimport random\nimport os\nimport pandas as pd\nimport numpy as np\n\n# Visuals and CV2\nimport seaborn as sn\nimport matplotlib.pyplot as plt\nimport cv2\n\n# albumentations for augs\nimport albumentations\nfrom albumentations.pytorch.transforms import ToTensorV2\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score\n\n\n#torch\nimport torch\nimport timm\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configuration"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DIM = (512,512)\n\nNUM_WORKERS = 4\nTRAIN_BATCH_SIZE = 16\nVALID_BATCH_SIZE = 16\nEPOCHS = 10\nSEED = 2020\nLR = 1e-4\n\nBASE_DIR = Path('../input/cassava-leaf-disease-merged')\n\nDEVICE = \"cuda\"\n\nMEAN = [0.485, 0.456, 0.406]\nSTD = [0.229, 0.224, 0.225]\n\nUSE_FULL_DATA = True\n\n\n################################################# MODEL ####################################################################\n\nMODEL_NAME = 'se_resnext50_32x4d' #resnext50_32x4d #resnext101_32x4d #efficientnet_b3 #efficientnetb5 #efficientnetb7\n\n\n################################################ Loss and its params #######################################################\nLOSS = 'bi_tempered_loss' #'label_smoothing' #'focal_cosine_loss' #'bi_tempered_loss' #'CE'\nSMOOTHING = 0.05\nALPHA = 1\nGAMMA = 2\nXENT = 0.1\nt1=0.3 # bi-tempered-loss https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/202017\nt2=1.0 # bi-tempered-loss https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/202017\nSMOOTHING_TEMP = 0.0\n\n####################################### Scheduler and its params ############################################################\nSCHEDULER = 'CosineAnnealingWarmRestarts' #'CosineAnnealingLR'\nfactor=0.2 # ReduceLROnPlateau\npatience=4 # ReduceLROnPlateau\neps=1e-6 # ReduceLROnPlateau\nT_max=10 # CosineAnnealingLR\nT_0=10 # CosineAnnealingWarmRestarts\nmin_lr=1e-6","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Seed"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter(object):\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n    \n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AccuracyMeter(object):\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.prediction = []\n        self.target = []\n        self.accuracy = []\n        \n    def update(self, y_true, y_pred):  \n        y_true = y_true.detach().cpu().numpy().astype(int)\n        y_pred = F.softmax(y_pred, dim=1).argmax(axis=1).detach().cpu().numpy().astype(int)\n        \n        self.prediction.append(y_pred)\n        self.target.append(y_true)\n        \n        y_true = np.concatenate(self.prediction, axis=0)\n        y_pred = np.concatenate(self.target, axis=0)\n        \n        self.accuracy = accuracy_score(y_true,y_pred)\n        \n    @property\n    def avg(self):\n        return self.accuracy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fetch_scheduler(optimizer):\n        if SCHEDULER =='ReduceLROnPlateau':\n            scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=factor, patience=patience, verbose=True, eps=eps)\n        elif SCHEDULER =='CosineAnnealingLR':\n            scheduler = CosineAnnealingLR(optimizer, T_max=T_max, eta_min=min_lr, last_epoch=-1)\n        elif SCHEDULER =='CosineAnnealingWarmRestarts':\n            scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=T_0, T_mult=1, eta_min=min_lr, last_epoch=-1)\n        return scheduler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fetch_loss():\n        if LOSS=='label_smoothing':\n            loss = LabelSmoothingLoss(smoothing=SMOOTHING)\n        elif LOSS=='bi_tempered_loss':\n            loss = BiTemperedLogisticLoss(t1=t1, t2=t2, smoothing=SMOOTHING_TEMP)\n        elif LOSS=='focal_cosine_loss':\n            loss = FocalCosineLoss(alpha=ALPHA,gamma=GAMMA,xent=XENT)\n        elif LOSS =='CE':\n            loss = nn.CrossEntropyLoss()\n        return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transforms():\n    return albumentations.Compose(\n        [\n            albumentations.HorizontalFlip(p=0.5),\n            albumentations.VerticalFlip(p=0.5),\n            albumentations.Rotate(limit=120, p=0.8),\n            albumentations.RandomBrightness(limit=(0.09, 0.6), p=0.5),\n            albumentations.CenterCrop (384, 384, always_apply=False, p=0.5),\n            albumentations.Resize(DIM[0],DIM[1]),\n            albumentations.Cutout(num_holes=8, max_h_size=8, max_w_size=8, fill_value=0, always_apply=False, p=0.5),\n            albumentations.ShiftScaleRotate(\n                shift_limit=0.25, scale_limit=0.1, rotate_limit=0\n            ),\n            albumentations.Normalize(\n                MEAN, STD, max_pixel_value=255.0, always_apply=True\n            ),\n        \n            ToTensorV2(p=1.0),\n        ]\n    )\n\ndef get_valid_transforms():\n\n    return albumentations.Compose(\n        [albumentations.Normalize(MEAN, STD, max_pixel_value=255.0, always_apply=True),\n        ToTensorV2(p=1.0)\n        ]\n    )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self,image_ids,labels,dimension=None,augmentations=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.labels = labels\n        self.dim = dimension\n        self.augmentations = augmentations\n        \n    def __len__(self):\n        return len(self.image_ids)\n    \n    def __getitem__(self,idx):\n        \n        img = cv2.imread(str(BASE_DIR/'train'/self.image_ids[idx]))\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n                         \n        if self.dim:\n            img = cv2.resize(img,self.dim)\n        \n        if self.augmentations:\n            augmented = self.augmentations(image=img)\n            image = augmented['image']\n                         \n        return {\n            'image': image,\n            'target': torch.tensor(self.labels[idx],dtype=torch.long)\n        }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaModel(nn.Module):\n    def __init__(self, model_name='seresnext50_32x4d',out_features=5,pretrained=True):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        \n        if model_name in ['efficientnet_b3','efficientnet_b5','efficientnet_b7']:\n            n_features = self.model.classifier.in_features\n            self.model.classifier = nn.Linear(n_features, out_features)\n            \n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, out_features)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_fn(dataloader,model,criterion,optimizer,device,scheduler,epoch):\n    model.train()\n    loss_score = AverageMeter()\n    auccuracy = AccuracyMeter()\n    \n    tk0 = tqdm(enumerate(dataloader), total=len(dataloader))\n    for bi,d in tk0:\n        \n        images = d['image']\n        targets = d['target']\n                \n        images = images.to(device)\n        targets = targets.to(device)\n        \n        batch_size = images.shape[0]\n        \n        optimizer.zero_grad()\n        \n        outputs = model(images)\n        \n        loss = criterion(outputs,targets)\n        loss.backward()\n        optimizer.step()\n        \n        if scheduler is not None:\n            scheduler.step()\n        \n        \n        auccuracy.update(targets, outputs)\n        loss_score.update(loss.detach().item(), batch_size)\n        \n        \n        tk0.set_postfix(Train_Loss=loss_score.avg,Train_accuracy = auccuracy.avg,Epoch=epoch)\n        \n    return loss_score, auccuracy","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluation Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def eval_fn(data_loader,model,criterion,device):\n    \n    loss_score = AverageMeter()\n    auccuracy = AccuracyMeter()\n    \n    model.eval()\n    tk0 = tqdm(enumerate(data_loader), total=len(data_loader))\n    \n    with torch.no_grad():\n        \n        for bi, d in tk0:\n            images = d['image']\n            targets = d['target']\n            \n            images = images.to(device)\n            targets = targets.to(device)\n            \n            batch_size = images.shape[0]\n            \n            outputs = model(images)\n            loss = criterion(outputs,targets)\n            \n            auccuracy.update(targets, outputs)\n            loss_score.update(loss.detach().item(), batch_size)\n            \n            tk0.set_postfix(Valid_Loss=loss_score.avg,Valid_accuracy = auccuracy.avg)\n        \n    return loss_score, auccuracy","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalCosineLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, xent=.1):\n        super(FocalCosineLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n        self.xent = xent\n\n        self.y = torch.Tensor([1]).cuda()\n\n    def forward(self, input, target, reduction=\"mean\"):\n        cosine_loss = F.cosine_embedding_loss(input, F.one_hot(target, num_classes=input.size(-1)), self.y, reduction=reduction)\n\n        cent_loss = F.cross_entropy(F.normalize(input), target, reduce=False)\n        pt = torch.exp(-cent_loss)\n        focal_loss = self.alpha * (1-pt)**self.gamma * cent_loss\n\n        if reduction == \"mean\":\n            focal_loss = torch.mean(focal_loss)\n\n        return cosine_loss + self.xent * focal_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def log_t(u, t):\n    \"\"\"Compute log_t for `u'.\"\"\"\n    if t==1.0:\n        return u.log()\n    else:\n        return (u.pow(1.0 - t) - 1.0) / (1.0 - t)\n\ndef exp_t(u, t):\n    \"\"\"Compute exp_t for `u'.\"\"\"\n    if t==1:\n        return u.exp()\n    else:\n        return (1.0 + (1.0-t)*u).relu().pow(1.0 / (1.0 - t))\n\ndef compute_normalization_fixed_point(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t > 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same shape as activation with the last dimension being 1.\n    \"\"\"\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations_step_0 = activations - mu\n\n    normalized_activations = normalized_activations_step_0\n    for _ in range(num_iters):\n        logt_partition = torch.sum(\n                exp_t(normalized_activations, t), -1, keepdim=True)\n        normalized_activations = normalized_activations_step_0 * \\\n                logt_partition.pow(1.0-t)\n\n    logt_partition = torch.sum(\n            exp_t(normalized_activations, t), -1, keepdim=True)\n    normalization_constants = - log_t(1.0 / logt_partition, t) + mu\n\n    return normalization_constants\n\ndef compute_normalization_binary_search(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t < 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (< 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations = activations - mu\n\n    effective_dim = \\\n        torch.sum(\n                (normalized_activations > -1.0 / (1.0-t)).to(torch.int32),\n            dim=-1, keepdim=True).to(activations.dtype)\n\n    shape_partition = activations.shape[:-1] + (1,)\n    lower = torch.zeros(shape_partition, dtype=activations.dtype, device=activations.device)\n    upper = -log_t(1.0/effective_dim, t) * torch.ones_like(lower)\n\n    for _ in range(num_iters):\n        logt_partition = (upper + lower)/2.0\n        sum_probs = torch.sum(\n                exp_t(normalized_activations - logt_partition, t),\n                dim=-1, keepdim=True)\n        update = (sum_probs < 1.0).to(activations.dtype)\n        lower = torch.reshape(\n                lower * update + (1.0-update) * logt_partition,\n                shape_partition)\n        upper = torch.reshape(\n                upper * (1.0 - update) + update * logt_partition,\n                shape_partition)\n\n    logt_partition = (upper + lower)/2.0\n    return logt_partition + mu\n\nclass ComputeNormalization(torch.autograd.Function):\n    \"\"\"\n    Class implementing custom backward pass for compute_normalization. See compute_normalization.\n    \"\"\"\n    @staticmethod\n    def forward(ctx, activations, t, num_iters):\n        if t < 1.0:\n            normalization_constants = compute_normalization_binary_search(activations, t, num_iters)\n        else:\n            normalization_constants = compute_normalization_fixed_point(activations, t, num_iters)\n\n        ctx.save_for_backward(activations, normalization_constants)\n        ctx.t=t\n        return normalization_constants\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        activations, normalization_constants = ctx.saved_tensors\n        t = ctx.t\n        normalized_activations = activations - normalization_constants \n        probabilities = exp_t(normalized_activations, t)\n        escorts = probabilities.pow(t)\n        escorts = escorts / escorts.sum(dim=-1, keepdim=True)\n        grad_input = escorts * grad_output\n        \n        return grad_input, None, None\n\ndef compute_normalization(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example. \n    Backward pass is implemented.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    return ComputeNormalization.apply(activations, t, num_iters)\n\ndef tempered_sigmoid(activations, t, num_iters = 5):\n    \"\"\"Tempered sigmoid function.\n    Args:\n      activations: Activations for the positive class for binary classification.\n      t: Temperature tensor > 0.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_probabilities = tempered_softmax(internal_activations, t, num_iters)\n    return internal_probabilities[..., 0]\n\ndef tempered_softmax(activations, t, num_iters=5):\n    \"\"\"Tempered softmax function.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature > 1.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    if t == 1.0:\n        return activations.softmax(dim=-1)\n\n    normalization_constants = compute_normalization(activations, t, num_iters)\n    return exp_t(activations - normalization_constants, t)\n\ndef bi_tempered_binary_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing = 0.0,\n        num_iters=5,\n        reduction='mean'):\n\n    \"\"\"Bi-Tempered binary logistic loss.\n    Args:\n      activations: A tensor containing activations for class 1.\n      labels: A tensor with shape as activations, containing probabilities for class 1\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A loss tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_labels = torch.stack([labels.to(activations.dtype),\n        1.0 - labels.to(activations.dtype)],\n        dim=-1)\n    return bi_tempered_logistic_loss(internal_activations, \n            internal_labels,\n            t1,\n            t2,\n            label_smoothing = label_smoothing,\n            num_iters = num_iters,\n            reduction = reduction)\n\ndef bi_tempered_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing=0.0,\n        num_iters=5,\n        reduction = 'mean'):\n\n    \"\"\"Bi-Tempered Logistic Loss.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      labels: A tensor with shape and dtype as activations (onehot), \n        or a long tensor of one dimension less than activations (pytorch standard)\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing parameter between [0, 1). Default 0.0.\n      num_iters: Number of iterations to run the method. Default 5.\n      reduction: ``'none'`` | ``'mean'`` | ``'sum'``. Default ``'mean'``.\n        ``'none'``: No reduction is applied, return shape is shape of\n        activations without the last dimension.\n        ``'mean'``: Loss is averaged over minibatch. Return shape (1,)\n        ``'sum'``: Loss is summed over minibatch. Return shape (1,)\n    Returns:\n      A loss tensor.\n      \"\"\"\n\n    if len(labels.shape)<len(activations.shape): #not one-hot\n        labels_onehot = torch.zeros_like(activations)\n        labels_onehot.scatter_(1, labels[..., None], 1)\n    else:\n        labels_onehot = labels\n\n    if label_smoothing > 0:\n        num_classes = labels_onehot.shape[-1]\n        labels_onehot = ( 1 - label_smoothing * num_classes / (num_classes - 1) ) \\\n                * labels_onehot + \\\n                label_smoothing / (num_classes - 1)\n\n    probabilities = tempered_softmax(activations, t2, num_iters)\n\n    loss_values = labels_onehot * log_t(labels_onehot + 1e-10, t1) \\\n            - labels_onehot * log_t(probabilities, t1) \\\n            - labels_onehot.pow(2.0 - t1) / (2.0 - t1) \\\n            + probabilities.pow(2.0 - t1) / (2.0 - t1)\n    loss_values = loss_values.sum(dim = -1) #sum over classes\n\n    if reduction == 'none':\n        return loss_values\n    if reduction == 'sum':\n        return loss_values.sum()\n    if reduction == 'mean':\n        return loss_values.mean()\n    \nclass BiTemperedLogisticLoss(nn.Module): \n    def __init__(self, t1, t2, smoothing=0.0): \n        super(BiTemperedLogisticLoss, self).__init__() \n        self.t1 = t1\n        self.t2 = t2\n        self.smoothing = smoothing\n    def forward(self, logit_label, truth_label):\n        loss_label = bi_tempered_logistic_loss(\n            logit_label, truth_label,\n            t1=self.t1, t2=self.t2,\n            label_smoothing=self.smoothing,\n            reduction='none'\n        )\n        \n        loss_label = loss_label.mean()\n        return loss_label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LabelSmoothingLoss(nn.Module): \n    def __init__(self, classes=5, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plotter"},{"metadata":{"trusted":true},"cell_type":"code","source":"def print_history(fold,history,num_epochs=EPOCHS):\n        plt.figure(figsize=(15,5))\n        \n        plt.plot(\n            np.arange(num_epochs),\n            history['train_history_accuracy'],\n            '-o',\n            label='Train ACCURACY',\n            color='#ff7f0e'\n        )\n        \n        plt.plot(\n            np.arange(num_epochs),\n            history['val_history_accuracy'],\n            '-o',\n            label='Val ACCURACY',\n            color='#1f77b4'\n        )\n        \n        x = np.argmax(history['val_history_accuracy'])\n        y = np.max(history['val_history_accuracy'])\n        \n        xdist = plt.xlim()[1] - plt.xlim()[0]\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        \n        plt.scatter(x, y, s=200, color='#1f77b4')\n        \n        plt.text(\n            x-0.03*xdist,\n            y-0.13*ydist,\n            'max auc\\n%.2f'%y,\n            size=14\n        )\n        \n        plt.ylabel('ACCURACY', size=14)\n        plt.xlabel('Epoch', size=14)\n        \n        plt.legend(loc=2)\n        \n        plt2 = plt.gca().twinx()\n        \n        plt2.plot(\n            np.arange(num_epochs),\n            history['train_history_loss'],\n            '-o',\n            label='Train Loss',\n            color='#2ca02c'\n        )\n        \n        plt2.plot(\n            np.arange(num_epochs),\n            history['val_history_loss'],\n            '-o',\n            label='Val Loss',\n            color='#d62728'\n        )\n        \n        x = np.argmin(history['val_history_loss'])\n        y = np.min(history['val_history_loss'])\n        \n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        \n        plt.scatter(x, y, s=200, color='#d62728')\n        \n        plt.text(\n            x-0.03*xdist, \n            y+0.05*ydist, \n            'min loss', \n            size=14\n        )\n        \n        plt.ylabel('Loss', size=14)\n        \n        plt.title(f'FOLD {fold + 1}',size=18)\n        \n        plt.legend(loc=3)\n        plt.show()  ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Engine"},{"metadata":{"trusted":true},"cell_type":"code","source":"if USE_FULL_DATA:\n    df = pd.read_csv('../input/cassava-train-folds/train_folds.csv')\n    df['source'] = 2020\n    df_2019 = pd.read_csv('../input/cassava-2019-data-folds/folds_2019.csv')\n    \n    df = df.append(df_2019)\n    print(df)\n    \nelse:\n    df = pd.read_csv('../input/cassava-train-folds/train_folds.csv')\n    print(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run(fold):\n    \n    if USE_FULL_DATA:\n        df_train = df[df.kfold != fold].reset_index(drop=True)\n        df_valid = df[(df['kfold'] == fold)&(df['source'] == 2020)].reset_index(drop=True)\n    \n    else:\n        df_train = df[df.kfold != fold].reset_index(drop=True)\n        df_valid = df[df['kfold'] == fold].reset_index(drop=True)\n    \n    # Defining DataSet\n    train_dataset = CassavaDataset(\n        image_ids=df_train['image_id'].values,\n        labels=df_train['label'].values,\n        augmentations=get_train_transforms(),\n        dimension = DIM\n    )\n        \n    valid_dataset = CassavaDataset(\n        image_ids=df_valid['image_id'].values,\n        labels=df_valid['label'].values,\n        augmentations=get_valid_transforms(),\n        dimension = DIM\n    )\n        \n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=TRAIN_BATCH_SIZE,\n        pin_memory=True,\n        drop_last=True,\n        num_workers=NUM_WORKERS\n    )\n    \n    valid_loader = torch.utils.data.DataLoader(\n        valid_dataset,\n        batch_size=VALID_BATCH_SIZE,\n        num_workers=NUM_WORKERS,\n        shuffle=False,\n        pin_memory=True,\n        drop_last=False,\n    )\n    \n    # Defining Device\n    device = torch.device(\"cuda\")\n    \n    # Defining Model for specific fold\n    model = CassavaModel(out_features=5)\n    model.to(device)\n    \n    #DEfining criterion\n    criterion = fetch_loss()\n    criterion.to(device)\n        \n    # Defining Optimizer with weight decay to params other than bias and layer norms\n    param_optimizer = list(model.named_parameters())\n    no_decay = [\"bias\", \"LayerNorm.bias\", \"LayerNorm.weight\"]\n    optimizer_parameters = [\n        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0},\n            ]  \n    \n    optimizer = torch.optim.Adam(optimizer_parameters, lr=LR)\n    \n    #Defining LR SCheduler\n    scheduler = fetch_scheduler(optimizer)\n    \n    # History dictionary to store everything\n    history = {\n            'train_history_loss': [],\n            'train_history_accuracy': [],\n            'val_history_loss': [],\n            'val_history_accuracy': [],\n        }\n        \n    # THE ENGINE LOOP\n    best_loss = 10000\n    fold_val_accuracy = []\n    for epoch in range(EPOCHS):\n        train_loss,train_accuracy = train_fn(train_loader, model,criterion, optimizer, device,scheduler=scheduler,epoch=epoch)\n        \n        valid_loss,valid_accuracy = eval_fn(valid_loader, model, criterion,device)\n        \n        history['train_history_loss'].append(train_loss.avg)\n        history['train_history_accuracy'].append(train_accuracy.avg)\n        history['val_history_loss'].append(valid_loss.avg)\n        history['val_history_accuracy'].append(valid_accuracy.avg)\n        \n        fold_val_accuracy.append(valid_accuracy.avg)\n        \n        if valid_loss.avg < best_loss:\n            best_loss = valid_loss.avg\n            torch.save(model.state_dict(),f'model_best_loss_fold_{fold}.bin')\n            \n    print('FOLD_CV : {}'.format(np.mean(fold_val_accuracy)))        \n    print_history(fold,history,num_epochs=epoch+1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"run(fold=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"run(fold=1)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}