{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Install and Set-Up"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"dev = 'GPU'\nif dev == 'TPU':\n    !curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py  > /dev/null\n    !python pytorch-xla-env-setup.py --version nightly  > /dev/null\n!pip install timm  > /dev/null\n!wget https://raw.githubusercontent.com/davda54/sam/main/sam.py\n!pip install torch_optimizer","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"##################################### Standard Imports #####################################\nimport os\nimport json\nimport gc\nimport pickle\nimport struct\nfrom time import time\nfrom tqdm import tqdm\nfrom uuid import uuid4\nfrom typing import List, Optional\n\n###################################### Data Handlers #######################################\nimport numpy as np\nimport pandas as pd\n\n##################################### Image Handlers #######################################\nimport cv2\nimport PIL\nfrom PIL import Image\nimport albumentations as A\n\n####################################### Optimizers #########################################\nfrom sam import SAM\nimport torch_optimizer as optim\n\n##################################### Plotting Tools #######################################\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport plotly.figure_factory as ff\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\n\n################################## Data Processing Fxns ####################################\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom sklearn import metrics\n\n##################################### Torch Imports ########################################\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch import Tensor\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\n################################### Torch XLA Imports ######################################\nif dev == 'TPU':\n    import torch_xla\n    import torch_xla.core.xla_model as xm\n    import torch_xla.distributed.parallel_loader as pl\n    import torch_xla.distributed.xla_multiprocessing as xmp\n    import torch_xla.utils.serialization as xser\n    import torch_xla.debug.metrics as met\n    import torch_xla.distributed.data_parallel as dp\n    import torch_xla.utils.utils as xu\n    import torch_xla.test.test_utils as test_utils\n\n##################################### Other Imports #######################################\nimport timm\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nif dev == 'TPU':\n    os.environ['XLA_USE_BF16']=\"1\"\n    os.environ['XLA_TENSOR_ALLOCATOR_MAXSIZE'] = '100000000'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helpers"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TwoWayDict:\n    def __init__(self, input_dict):\n        self.d1 = input_dict\n        try:\n            self.d2 = {v: k for k, v in input_dict.items()}\n        except Exception:\n            raise ('Duplicate Key')\n\n    def get(self, key):\n        if key in self.d1:\n            return self.d1[key]\n        if key in self.d2:\n            return self.d2[key]\n        return None\n\n    def length(self):\n        return len(self.d1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/cassava-leaf-disease-merged/merged.csv')\ndf = df.sample(frac=1).reset_index(drop=True)\nwith open('../input/cassava-leaf-disease-classification/label_num_to_disease_map.json', 'r') as fp:\n    label_map = json.load(fp)\nclass_dict = TwoWayDict(label_map)\n\ny_cols = df.label.unique().tolist()\ny_cols.sort()\ndisplay(df.head())\ndisplay(label_map)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"odf = pd.concat([df, pd.get_dummies(df.label)], 1)\nsplitter = StratifiedShuffleSplit(n_splits=1, train_size=0.95, test_size=0.05, random_state=42)\ntrain_indices, valid_indices = next(splitter.split(odf.label.values, odf[y_cols].values))\n\ntrain_df = odf.iloc[train_indices]\nvalid_df = odf.iloc[valid_indices]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LeafDataset(Dataset):\n    def __init__(self, df, aug=None, transform=None, encoding=\"Label\", process_first=False, path_prefix=\"\"):\n        self.y_cols = y_cols\n        self.image_paths = df.image_id.values\n        if encoding == \"Label\":\n            self.enc = LabelEncoder()\n            self.enc.fit(df.label)\n            self.targets = self.enc.transform(df.label)\n        elif encoding == \"OneHot\":\n            self.targets = pd.get_dummies(df.label).values\n        else:\n            raise \"Unsupprted Target Encoding\"\n        self.aug = aug\n        self.transform = transform\n        self.path_prefix = path_prefix\n        self.process_first = process_first\n        self.process_dict = {}\n        \n        if self.process_first:\n            if not os.path.exists('./temp_data'):\n                os.makedirs('./temp_data')\n            for path in tqdm(self.image_paths):\n                save_name = './temp_data/' + uuid4().hex + '.tnsr'\n                self.process_dict[path] = save_name\n                image = cv2.imread(self.path_prefix  + path)\n                image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n                if self.aug is not None:\n                    augmented = self.aug(image=image)\n                    image = augmented[\"image\"]\n                image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n                image = torch.tensor(image, dtype=torch.float32)\n                if self.transform is not None:\n                    image = self.transform(image)\n                torch.save(image, save_name)\n    def __len__(self):\n        return len(self.image_paths)\n\n    def __getitem__(self, index):\n        if self.process_first:\n            image = torch.load(self.process_dict[self.image_paths[index]])\n        else:\n            image = cv2.imread(self.path_prefix + self.image_paths[index])\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            if self.aug is not None:\n                augmented = self.aug(image=image)\n                image = augmented[\"image\"]\n            image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n            image = torch.tensor(image, dtype=torch.float32)\n            if self.transform is not None:\n                image = self.transform(image)\n        targets = self.targets[index]\n        return {\n            \"image\": image,\n            \"targets\": torch.tensor(targets, dtype=torch.long),\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_SIZE = 224\ntrain_transforms = transforms.Compose([\n#     transforms.RandomApply([\n#         transforms.CenterCrop((IMG_SIZE, IMG_SIZE))\n#     ], p=0.5),\n    transforms.Resize((IMG_SIZE, IMG_SIZE), interpolation=3),\n    transforms.RandomVerticalFlip(p=0.5),\n    transforms.RandomPerspective(0.3, p=0.5),\n    transforms.RandomHorizontalFlip(p=0.5),\n#     transforms.RandomApply([\n#         transforms.ColorJitter(0.2, 0.1, 0.1, 0.05),\n#         transforms.RandomRotation(45),\n#     ], p=0.5),\n    transforms.Normalize(mean=[0.42984136, 0.49624753, 0.3129598], std=[0.21417203, 0.21910103, 0.19542212]),\n])\n\nvalid_transforms = transforms.Compose([\n    transforms.Resize((IMG_SIZE, IMG_SIZE), interpolation=3),\n    transforms.Normalize(mean=[0.42984136, 0.49624753, 0.3129598], std=[0.21417203, 0.21910103, 0.19542212]),\n])\n\n\ntrain_aug = A.Compose(\n        [\n            A.Resize(IMG_SIZE, IMG_SIZE, interpolation=4, always_apply=True),\n            A.Transpose(p=0.5),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.ShiftScaleRotate(p=0.5),\n            A.HueSaturationValue(\n                hue_shift_limit=0.2, \n                sat_shift_limit=0.2, \n                val_shift_limit=0.2,\n                p=0.5\n            ),\n            A.RandomBrightnessContrast(\n                brightness_limit=(-0.1,0.1), \n                contrast_limit=(-0.1, 0.1), \n                p=0.5\n            ),\n            A.CoarseDropout(p=0.5),\n            A.Cutout(p=0.5)\n        ]\n    )\n\nvalid_aug = A.Compose(\n        [\n            A.Resize(IMG_SIZE, IMG_SIZE, interpolation=4, always_apply=True),\n        ]\n)\n\ntrain_dataset = LeafDataset(train_df, encoding=\"Label\", transform=train_transforms, aug=None, path_prefix=\"../input/cassava-leaf-disease-merged/train/\") \nvalid_dataset = LeafDataset(valid_df, encoding=\"Label\", transform=valid_transforms, aug=None, path_prefix=\"../input/cassava-leaf-disease-merged/train/\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# timm.create_model(f\"vit_base_patch32_384\", pretrained=False)\n# timm.list_models()\n# timm.create_model(f\"tf_efficientnet_b1_ns\", pretrained=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    effnet = timm.create_model(f\"tf_efficientnet_b1_ns\", pretrained=True)\n\n    effnet.classifier = nn.Sequential(\n#         nn.Linear(1536, 128),  # B3\n        nn.Linear(1280, 384),  # B4\n        nn.Dropout(0.1),\n        nn.Linear(384, 5),\n        nn.Softmax(1)\n    )\n\n    for param in effnet.parameters():\n        param.requires_grad = True\n\n    def set_bn_eval(m):\n        classname = m.__class__.__name__\n        if classname.find('BatchNorm') != -1:\n            m.eval()\n\n#     effnet = effnet.apply(set_bn_eval)\n    return effnet","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Helpers "},{"metadata":{},"cell_type":"markdown","source":"### Helpers"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"def log_t(u, t):\n    \"\"\"Compute log_t for `u'.\"\"\"\n    if t==1.0:\n        return u.log()\n    else:\n        return (u.pow(1.0 - t) - 1.0) / (1.0 - t)\n\ndef exp_t(u, t):\n    \"\"\"Compute exp_t for `u'.\"\"\"\n    if t==1:\n        return u.exp()\n    else:\n        return (1.0 + (1.0-t)*u).relu().pow(1.0 / (1.0 - t))\n\ndef compute_normalization_fixed_point(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t > 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same shape as activation with the last dimension being 1.\n    \"\"\"\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations_step_0 = activations - mu\n\n    normalized_activations = normalized_activations_step_0\n\n    for _ in range(num_iters):\n        logt_partition = torch.sum(\n                exp_t(normalized_activations, t), -1, keepdim=True)\n        normalized_activations = normalized_activations_step_0 * \\\n                logt_partition.pow(1.0-t)\n\n    logt_partition = torch.sum(\n            exp_t(normalized_activations, t), -1, keepdim=True)\n    normalization_constants = - log_t(1.0 / logt_partition, t) + mu\n\n    return normalization_constants\n\ndef compute_normalization_binary_search(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t < 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (< 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations = activations - mu\n\n    effective_dim = \\\n        torch.sum(\n                (normalized_activations > -1.0 / (1.0-t)).to(torch.int32),\n            dim=-1, keepdim=True).to(activations.dtype)\n\n    shape_partition = activations.shape[:-1] + (1,)\n    lower = torch.zeros(shape_partition, dtype=activations.dtype, device=activations.device)\n    upper = -log_t(1.0/effective_dim, t) * torch.ones_like(lower)\n\n    for _ in range(num_iters):\n        logt_partition = (upper + lower)/2.0\n        sum_probs = torch.sum(\n                exp_t(normalized_activations - logt_partition, t),\n                dim=-1, keepdim=True)\n        update = (sum_probs < 1.0).to(activations.dtype)\n        lower = torch.reshape(\n                lower * update + (1.0-update) * logt_partition,\n                shape_partition)\n        upper = torch.reshape(\n                upper * (1.0 - update) + update * logt_partition,\n                shape_partition)\n\n    logt_partition = (upper + lower)/2.0\n    return logt_partition + mu\n\nclass ComputeNormalization(torch.autograd.Function):\n    \"\"\"\n    Class implementing custom backward pass for compute_normalization. See compute_normalization.\n    \"\"\"\n    @staticmethod\n    def forward(ctx, activations, t, num_iters):\n        if t < 1.0:\n            normalization_constants = compute_normalization_binary_search(activations, t, num_iters)\n        else:\n            normalization_constants = compute_normalization_fixed_point(activations, t, num_iters)\n\n        ctx.save_for_backward(activations, normalization_constants)\n        ctx.t=t\n        return normalization_constants\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        activations, normalization_constants = ctx.saved_tensors\n        t = ctx.t\n        normalized_activations = activations - normalization_constants \n        probabilities = exp_t(normalized_activations, t)\n        escorts = probabilities.pow(t)\n        escorts = escorts / escorts.sum(dim=-1, keepdim=True)\n        grad_input = escorts * grad_output\n        \n        return grad_input, None, None\n\ndef compute_normalization(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example. \n    Backward pass is implemented.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    return ComputeNormalization.apply(activations, t, num_iters)\n\ndef tempered_sigmoid(activations, t, num_iters = 5):\n    \"\"\"Tempered sigmoid function.\n    Args:\n      activations: Activations for the positive class for binary classification.\n      t: Temperature tensor > 0.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_probabilities = tempered_softmax(internal_activations, t, num_iters)\n    return internal_probabilities[..., 0]\n\n\ndef tempered_softmax(activations, t, num_iters=5):\n    \"\"\"Tempered softmax function.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature > 1.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    if t == 1.0:\n        return activations.softmax(dim=-1)\n\n    normalization_constants = compute_normalization(activations, t, num_iters)\n    return exp_t(activations - normalization_constants, t)\n\ndef bi_tempered_binary_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing = 0.0,\n        num_iters=5,\n        reduction='mean'):\n\n    \"\"\"Bi-Tempered binary logistic loss.\n    Args:\n      activations: A tensor containing activations for class 1.\n      labels: A tensor with shape as activations, containing probabilities for class 1\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A loss tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_labels = torch.stack([labels.to(activations.dtype),\n        1.0 - labels.to(activations.dtype)],\n        dim=-1)\n    return bi_tempered_logistic_loss(internal_activations, \n            internal_labels,\n            t1,\n            t2,\n            label_smoothing = label_smoothing,\n            num_iters = num_iters,\n            reduction = reduction)\n\ndef bi_tempered_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing=0.0,\n        num_iters=5,\n        reduction = 'mean'):\n\n    \"\"\"Bi-Tempered Logistic Loss.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      labels: A tensor with shape and dtype as activations (onehot), \n        or a long tensor of one dimension less than activations (pytorch standard)\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing parameter between [0, 1). Default 0.0.\n      num_iters: Number of iterations to run the method. Default 5.\n      reduction: ``'none'`` | ``'mean'`` | ``'sum'``. Default ``'mean'``.\n        ``'none'``: No reduction is applied, return shape is shape of\n        activations without the last dimension.\n        ``'mean'``: Loss is averaged over minibatch. Return shape (1,)\n        ``'sum'``: Loss is summed over minibatch. Return shape (1,)\n    Returns:\n      A loss tensor.\n    \"\"\"\n\n    if len(labels.shape)<len(activations.shape): #not one-hot\n        labels_onehot = torch.zeros_like(activations)\n        labels_onehot.scatter_(1, labels[..., None], 1)\n    else:\n        labels_onehot = labels\n\n    if label_smoothing > 0:\n        num_classes = labels_onehot.shape[-1]\n        labels_onehot = ( 1 - label_smoothing * num_classes / (num_classes - 1) ) \\\n                * labels_onehot + \\\n                label_smoothing / (num_classes - 1)\n\n    probabilities = tempered_softmax(activations, t2, num_iters)\n\n    loss_values = labels_onehot * log_t(labels_onehot + 1e-10, t1) \\\n            - labels_onehot * log_t(probabilities, t1) \\\n            - labels_onehot.pow(2.0 - t1) / (2.0 - t1) \\\n            + probabilities.pow(2.0 - t1) / (2.0 - t1)\n    loss_values = loss_values.sum(dim = -1) #sum over classes\n\n    if reduction == 'none':\n        return loss_values\n    if reduction == 'sum':\n        return loss_values.sum()\n    if reduction == 'mean':\n        return loss_values.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalLoss(nn.Module):\n    \"\"\" Focal Loss, as described in https://arxiv.org/abs/1708.02002.\n    It is essentially an enhancement to cross entropy loss and is\n    useful for classification tasks when there is a large class imbalance.\n    x is expected to contain raw, unnormalized scores for each class.\n    y is expected to contain class labels.\n    Shape:\n        - x: (batch_size, C) or (batch_size, C, d1, d2, ..., dK), K > 0.\n        - y: (batch_size,) or (batch_size, d1, d2, ..., dK), K > 0.\n    \"\"\"\n\n    def __init__(self,\n                 alpha: Optional[Tensor] = None,\n                 gamma: float = 0.,\n                 reduction: str = 'mean',\n                 ignore_index: int = -100):\n        \"\"\"Constructor.\n        Args:\n            alpha (Tensor, optional): Weights for each class. Defaults to None.\n            gamma (float, optional): A constant, as described in the paper.\n                Defaults to 0.\n            reduction (str, optional): 'mean', 'sum' or 'none'.\n                Defaults to 'mean'.\n            ignore_index (int, optional): class label to ignore.\n                Defaults to -100.\n        \"\"\"\n        if reduction not in ('mean', 'sum', 'none'):\n            raise ValueError(\n                'Reduction must be one of: \"mean\", \"sum\", \"none\".')\n\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.ignore_index = ignore_index\n        self.reduction = reduction\n\n        self.nll_loss = nn.NLLLoss(\n            weight=alpha, reduction='none', ignore_index=ignore_index)\n\n    def __repr__(self):\n        arg_keys = ['alpha', 'gamma', 'ignore_index', 'reduction']\n        arg_vals = [self.__dict__[k] for k in arg_keys]\n        arg_strs = [f'{k}={v}' for k, v in zip(arg_keys, arg_vals)]\n        arg_str = ', '.join(arg_strs)\n        return f'{type(self).__name__}({arg_str})'\n\n    def forward(self, x: Tensor, y: Tensor) -> Tensor:\n        if x.ndim > 2:\n            # (N, C, d1, d2, ..., dK) --> (N * d1 * ... * dK, C)\n            c = x.shape[1]\n            x = x.permute(0, *range(2, x.ndim), 1).reshape(-1, c)\n            # (N, d1, d2, ..., dK) --> (N * d1 * ... * dK,)\n            y = y.view(-1)\n\n        unignored_mask = y != self.ignore_index\n        y = y[unignored_mask]\n        if len(y) == 0:\n            return 0.\n        x = x[unignored_mask]\n\n        # compute weighted cross entropy term: -alpha * log(pt)\n        # (alpha is already part of self.nll_loss)\n        log_p = F.log_softmax(x, dim=-1)\n        ce = self.nll_loss(log_p, y)\n\n        # get true class column from each row\n        all_rows = torch.arange(len(x))\n        log_pt = log_p[all_rows, y]\n\n        # compute focal term: (1 - pt)^gamma\n        pt = log_pt.exp()\n        focal_term = (1 - pt)**self.gamma\n\n        # the full loss: -alpha * ((1 - pt)^gamma) * log(pt)\n        loss = focal_term * ce\n\n        if self.reduction == 'mean':\n            loss = loss.mean()\n        elif self.reduction == 'sum':\n            loss = loss.sum()\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LabelSmoothingLoss(nn.Module): \n    def __init__(self, classes=5, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n#         if CFG.criterion_name == 'LabelSmoothingLoss':\n#             pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))\n\nclass TaylorSoftmax(nn.Module):\n    '''\n    This is the autograd version\n    '''\n    def __init__(self, dim=1, n=2):\n        super(TaylorSoftmax, self).__init__()\n        assert n % 2 == 0\n        self.dim = dim\n        self.n = n\n\n    def forward(self, x):\n        '''\n        usage similar to nn.Softmax:\n            >>> mod = TaylorSoftmax(dim=1, n=4)\n            >>> inten = torch.randn(1, 32, 64, 64)\n            >>> out = mod(inten)\n        '''\n        fn = torch.ones_like(x)\n        denor = 1.\n        for i in range(1, self.n+1):\n            denor *= i\n            fn = fn + x.pow(i) / denor\n        out = fn / fn.sum(dim=self.dim, keepdims=True)\n        return out\n    \nclass TaylorCrossEntropyLoss(nn.Module):\n    '''\n    This is the autograd version\n    '''\n    def __init__(self, n=2, ignore_index=-1, reduction='mean'):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n\n    def forward(self, logits, labels):\n        '''\n        usage similar to nn.CrossEntropyLoss:\n            >>> crit = TaylorCrossEntropyLoss(n=4)\n            >>> inten = torch.randn(1, 10, 64, 64)\n            >>> label = torch.randint(0, 10, (1, 64, 64))\n            >>> out = crit(inten, label)\n        '''\n        log_probs = self.taylor_softmax(logits).log()\n        loss = F.nll_loss(log_probs, labels, reduction=self.reduction,\n                ignore_index=self.ignore_index)\n        return loss\n    \nclass TaylorSmoothedLoss(nn.Module):\n\n    def __init__(self, n=2, classes=5, ignore_index=-1, reduction='mean', smoothing=0.2):\n        super(TaylorSmoothedLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n        self.lab_smooth = LabelSmoothingLoss(classes, smoothing=smoothing)\n\n    def forward(self, logits, labels):\n\n        log_probs = self.taylor_softmax(logits).log()\n        #loss = F.nll_loss(log_probs, labels, reduction=self.reduction,\n        #        ignore_index=self.ignore_index)\n        loss = self.lab_smooth(log_probs, labels)\n        return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"class SAM_XLA(torch.optim.Optimizer):\n    def __init__(self, params, base_optimizer, rho=0.05, **kwargs):\n        assert rho >= 0.0, f\"Invalid rho, should be non-negative: {rho}\"\n\n        defaults = dict(rho=rho, **kwargs)\n        super(SAM_XLA, self).__init__(params, defaults)\n\n        self.base_optimizer = base_optimizer(self.param_groups, **kwargs)\n        self.param_groups = self.base_optimizer.param_groups\n\n    @torch.no_grad()\n    def first_step(self, zero_grad=False):\n        grad_norm = self._grad_norm()\n        for group in self.param_groups:\n            scale = group[\"rho\"] / (grad_norm + 1e-12)\n            \n            def update_state(p):\n                if p.grad is not None:\n                    e_w = p.grad * scale.to(p)\n                    p.add_(e_w)  # climb to the local maximum \"w + e(w)\"\n                    self.state[p][\"e_w\"] = e_w\n            \n            map(update_state, group[\"params\"])\n            \n#             for p in group[\"params\"]:\n#                 if p.grad is None: continue\n#                 e_w = p.grad * scale.to(p)\n#                 p.add_(e_w)  # climb to the local maximum \"w + e(w)\"\n#                 self.state[p][\"e_w\"] = e_w\n        if zero_grad: self.zero_grad()\n    \n    @torch.no_grad()\n    def second_step(self, zero_grad=False):\n        for group in self.param_groups:\n            \n            def update_p(p):\n                if p.grad is not None:\n                    p.sub_(self.state[p][\"e_w\"])\n            \n            map(update_p, group[\"params\"])\n            \n#             for p in group[\"params\"]:\n#                 if p.grad is None: continue\n#                 p.sub_(self.state[p][\"e_w\"])  # get back to \"w\" from \"w + e(w)\"\n\n        self.base_optimizer.step()  # do the actual \"sharpness-aware\" update\n\n        if zero_grad: self.zero_grad()\n\n    @torch.no_grad()\n    def step(self, closure=None):\n        assert closure is not None, \"Sharpness Aware Minimization requires closure, but it was not provided\"\n        closure = torch.enable_grad()(closure)  # the closure should do a full forward-backward pass\n\n        self.first_step(zero_grad=True)\n        closure()\n        self.second_step()\n\n    def _grad_norm(self):\n        shared_device = self.param_groups[0][\"params\"][0].device  # put everything on the same device, in case of model parallelism\n        stack = [torch.tensor(list(map(lambda p: p.grad.norm(p=2).to(shared_device), list(filter(lambda p: p.grad is not None, group[\"params\"]))))) for group in self.param_groups]  # torch.tensor(list(map(lambda p: p.grad.norm(p=2).to(shared_device) if p.grad is not None else p, group[\"params\"])))\n        norm = torch.norm(torch.stack(stack), p=2)\n        return norm\n    \n#     def _grad_norm(self):\n#         shared_device = self.param_groups[0][\"params\"][0].device  # put everything on the same device, in case of model parallelism\n#         print(shared_device)\n#         norm = torch.norm(\n#                     torch.stack([\n#                         p.grad.norm(p=2).to(shared_device)\n#                         for group in self.param_groups for p in group[\"params\"]\n#                         if p.grad is not None\n#                     ]),\n#                     p=2\n#                )\n#         return norm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Wrappers"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_optimizer(model):\n    base_optimizer = optim.RAdam\n    optimizer = SAM_XLA(model.parameters(), base_optimizer, lr=0.0002)\n#     optimizer = optim.RAdam(model.parameters(), lr=0.0002)\n    return optimizer\ndef get_scheduler(optimizer):\n    return ReduceLROnPlateau(optimizer, mode='max', patience=3, cooldown=2)\ndef save_checkpoint(model, optimizer, path):\n    if not os.path.exists(os.path.dirname(path)):\n        print(\"Creating directories on path: `{}`\".format(path))\n        os.makedirs(os.path.dirname(path))\n    torch.save({\n        \"model_state_dict\": model.state_dict(),\n        \"optimizer_state_dict\": optimizer.state_dict(),\n    }, path)\ndef save_model(model, path):\n    if not os.path.exists(os.path.dirname(path)):\n        print(\"Creating directories on path: `{}`\".format(path))\n        os.makedirs(os.path.dirname(path))\n    torch.save({\n        \"model_state_dict\": model.state_dict(),\n    }, path)\ndef load_checkpoint(model, path):\n    checkpoint = torch.load(path)\n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    optimizer = get_optimizer(model)\n    optimizer.load_state_dict(checkpoint[\"optimizer_state_dict\"])\n    return model, optimizer\ndef load_model(model, path):\n    restore_dict = torch.load(path)\n    model.load_state_dict(restore_dict[\"model_state_dict\"])\n    model.eval()\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train & Eval Fxns"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(data_loader, model, optimizer, scheduler, criterion, device, epoch=0, update_after=3):\n    model.train()\n    final_targets = []\n    final_outputs = []\n    losses = AverageMeter()\n    for step, data in enumerate(data_loader):\n        inputs = data[\"image\"]\n        targets = data[\"targets\"]\n        inputs = inputs.to(device)\n        targets = targets.to(device)\n        def closure():\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n            losses.update(loss.item(), config['batch_size'])\n            loss.backward()\n            return loss\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        losses.update(loss.item(), config['batch_size'])\n        loss.backward\n        if config['device_name'] == 'TPU':\n            xm.optimizer_step(optimizer, optimizer_args={'closure': closure})\n            optimizer.zero_grad()\n        elif config['device_name'] == 'GPU':\n            optimizer.step(closure)\n            optimizer.zero_grad()\n        del inputs\n        targets = targets.detach().cpu().numpy().tolist()\n        outputs = outputs.detach().cpu().numpy().tolist()\n        \n        final_targets.extend(targets)\n        final_outputs.extend(outputs)\n        if (step+ 1) % update_after == 0 or (step + 1) == len(data_loader):\n            if config['device_name'] == 'TPU':\n                # since the loss is on all 8 cores, reduce the loss values and print the average\n                loss_reduced = xm.mesh_reduce('loss_reduce', losses.avg, lambda x: sum(x) / len(x))\n                final_targets_tensor = torch.tensor(final_targets, device=device)\n                final_outputs_tensor = torch.tensor(final_outputs, device=device)\n                targets_reduced = xm.all_gather(final_targets_tensor, dim=0)\n                outputs_reduced = xm.all_gather(final_outputs_tensor, dim=1)\n                # master_print will only print once (not from all 8 cores)\n                lr = optimizer.param_groups[0]['lr']\n                accuracy = metrics.accuracy_score(np.array(targets_reduced.cpu()), np.array(outputs_reduced.cpu()).argmax(axis=1))\n                cohen_kappa = metrics.cohen_kappa_score(np.array(targets_reduced.cpu()), np.array(outputs_reduced.cpu()).argmax(axis=1))\n                try:\n                    pass\n                    roc_auc = metrics.roc_auc_score(np.array(targets_reduced.cpu()), np.array(outputs_reduced.cpu()), multi_class='ovr', labels=config['labels'])\n                except Exception:\n                    roc_auc = 0\n                xm.master_print(f\"Epoch {epoch + 1}/{config['epochs']}, Step {step + 1}/{len(data_loader)} :: Train Loss={loss_reduced}, LR={lr}, Accuracy={accuracy}, Cohen Kappa Score={cohen_kappa}, MultiClass ROC AUC={roc_auc}\")\n                # xm.master_print(f\"loss_reduced: {loss_reduced}\")\n            elif config['device_name'] == 'GPU':\n                lr = optimizer.param_groups[0]['lr']\n                accuracy = round(metrics.accuracy_score(np.array(final_targets), np.array(final_outputs).argmax(axis=1)), 5)\n                cohen_kappa = round(metrics.cohen_kappa_score(np.array(final_targets), np.array(final_outputs).argmax(axis=1)), 5)\n                try:\n                    roc_auc = round(metrics.roc_auc_score(np.array(final_targets), np.array(final_outputs), multi_class='ovr', labels=config['labels']), 5)\n                except Exception:\n                    roc_auc = 0\n                print(f\"Epoch {epoch + 1}/{config['epochs']}, Step {step + 1}/{len(data_loader)} :: Train Loss={losses.avg}, LR={lr}, Accuracy={accuracy}, Cohen Kappa Score={cohen_kappa}, MultiClass ROC AUC={roc_auc}\", end='\\r')\n        \n        del targets, outputs\n    gc.collect() # delete for memory conservation\n\n    scheduler.step(loss)\n    if config['device_name'] == 'GPU':\n        torch.cuda.empty_cache()\n        print(f\"Epoch {epoch + 1}/{config['epochs']}, Step {step + 1}/{len(data_loader)} :: Train Loss={losses.avg}, LR={lr}, Accuracy={accuracy}, Cohen Kappa Score={cohen_kappa}, MultiClass ROC AUC={roc_auc}\")\n    return losses.avg, accuracy, cohen_kappa, roc_auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate_model(data_loader, model, criterion, device, epoch=0, update_after=3):\n    model.eval()\n    final_targets = []\n    final_outputs = []\n    losses = AverageMeter()\n    for step, data in enumerate(data_loader):      \n        inputs = data[\"image\"]\n        targets = data[\"targets\"]\n        inputs = inputs.to(device)\n        targets = targets.to(device)\n\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        losses.update(loss.item(), config['batch_size'])\n        del inputs\n        targets = targets.detach().cpu().numpy().tolist()\n        outputs = outputs.detach().cpu().numpy().tolist()\n        \n        final_targets.extend(targets)\n        final_outputs.extend(outputs)\n        if (step+ 1) % update_after == 0 or (step + 1) == len(data_loader):\n            if config['device_name'] == 'TPU':\n                # since the loss is on all 8 cores, reduce the loss values and print the average\n                loss_reduced = xm.mesh_reduce('loss_reduce', losses.avg, lambda x: sum(x) / len(x)) \n                targets_reduced = xm.all_gather(final_targets, dim=0)\n                outputs_reduced = xm.all_gather(final_outputs, dim=1)\n                # master_print will only print once (not from all 8 cores)\n                accuracy = metrics.accuracy_score(np.array(targets_reduced), np.array(outputs_reduced).argmax(axis=1))\n                cohen_kappa = metrics.cohen_kappa_score(np.array(targets_reduced), np.array(outputs_reduced).argmax(axis=1))\n                try:\n                    roc_auc = metrics.roc_auc_score(np.array(targets_reduced), np.array(outputs_reduced), multi_class='ovr', labels=config['labels'])\n                except Exception:\n                    roc_auc = 0\n                xm.master_print(f\"Epoch {epoch + 1}/{config['epochs']}, Step {step + 1}/{len(data_loader)} :: Train Loss={loss_reduced}, Accuracy={accuracy}, Cohen Kappa Score={cohen_kappa}, MultiClass ROC AUC={roc_auc}\")\n\n            elif config['device_name'] == 'GPU':\n                accuracy = round(metrics.accuracy_score(np.array(final_targets), np.array(final_outputs).argmax(axis=1)), 5)\n                cohen_kappa = round(metrics.cohen_kappa_score(np.array(final_targets), np.array(final_outputs).argmax(axis=1)), 5)\n                try:\n                    roc_auc = round(metrics.roc_auc_score(np.array(final_targets), np.array(final_outputs), multi_class='ovr', labels=config['labels']), 5)\n                except Exception:\n                    roc_auc = 0\n                print(f\"Epoch {epoch + 1}/{config['epochs']}, Step {step + 1}/{len(data_loader)} :: Valid Loss={losses.avg}, Accuracy={accuracy}, Cohen Kappa Score={cohen_kappa}, MultiClass ROC AUC={roc_auc}\", end='\\r')\n\n    del targets, outputs\n    gc.collect() # delete for memory conservation\n    if config['device_name'] == 'GPU':\n        torch.cuda.empty_cache()\n        print(f\"Epoch {epoch + 1}/{config['epochs']}, Step {step + 1}/{len(data_loader)} :: Valid Loss={losses.avg}, Accuracy={accuracy}, Cohen Kappa Score={cohen_kappa}, MultiClass ROC AUC={roc_auc}\")\n    return losses.avg, accuracy, cohen_kappa, roc_auc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model_with_data(rank, model, train_dataset, valid_dataset):\n    \n    ######################################################################\n    ######################## TRAINING DEPENDENCIES #######################\n    ######################################################################\n    \n    device = config['device'](config['device_name'])\n    if config['device_name'] == 'TPU':\n        MX = xmp.MpModelWrapper(model)\n        model = MX.to(device)\n    elif config['device_name'] == 'GPU':\n        model.to(device)\n    \n    optimizer = config['optimizer'](model)\n    scheduler = config['scheduler'](optimizer)\n    criterion = config['loss_fn']\n    \n    gc.collect()  # delete for memory conservation\n    \n    ######################################################################\n    ############################ LOAD DATA ###############################\n    ######################################################################\n    \n    if config['device_name'] == 'TPU':\n        # special sampler needed for distributed/multi-core (divides dataset among the replicas/cores/devices)\n        train_sampler = torch.utils.data.distributed.DistributedSampler(\n            train_dataset,\n            num_replicas=xm.xrt_world_size(), #divide dataset among this many replicas\n            rank=xm.get_ordinal(), #which replica/device/core\n            shuffle=True)\n\n        # define DataLoader with the defined sampler\n        train_loader = torch.utils.data.DataLoader(\n            train_dataset,\n            batch_size=config['batch_size'],\n            sampler=train_sampler,\n            num_workers=config['num_workers'],\n            drop_last=True,\n            prefetch_factor=8,\n            persistent_workers=True\n        )\n\n        # same as train but with valid data\n        valid_sampler = torch.utils.data.distributed.DistributedSampler(\n            valid_dat48aset,\n            num_replicas=xm.xrt_world_size(),\n            rank=xm.get_ordinal(),\n            shuffle=False)\n\n        valid_loader = torch.utils.data.DataLoader(\n            valid_dataset,\n            batch_size=config['batch_size'],\n            sampler=valid_sampler,\n            num_workers=config['num_workers'],\n            drop_last=False,\n            prefetch_factor=8,\n            persistent_workers=True\n        )\n\n        train_loader = pl.MpDeviceLoader(train_loader, device) # puts the train data onto the current TPU core\n        valid_loader = pl.MpDeviceLoader(valid_loader, device) # puts the valid data onto the current TPU core\n    else:  # GPU and CPU\n        train_loader = DataLoader(dataset=train_dataset, batch_size=config['batch_size'], num_workers=config['num_workers'], pin_memory=False, shuffle=False, prefetch_factor=8, persistent_workers=True)\n        valid_loader = DataLoader(dataset=valid_dataset, batch_size=config['batch_size'], num_workers=config['num_workers'], pin_memory=False, shuffle=False, prefetch_factor=8, persistent_workers=True)\n    \n    ######################################################################\n    ############################## TRAIN LOOP ############################\n    ######################################################################\n    \n    train_history = {\n        \"train_loss\": [],\n        \"valid_loss\": [],\n        \"train_accuracy\": [],\n        \"valid_accuracy\": [],\n        \"train_cohen_kappa\": [],\n        \"valid_cohen_kappa\": [],\n        \"train_roc_auc\": [],\n        \"valid_roc_auc\": []\n    }\n    \n    for epoch in range(config['epochs']):\n        \n        train_loss, train_accuracy, train_cohen_kappa, train_roc_auc = train_model(train_loader, model, optimizer, scheduler, criterion, device, epoch, update_after=1)\n        valid_loss, valid_accuracy, valid_cohen_kappa, valid_roc_auc = evaluate_model(valid_loader, model, criterion, device, epoch)\n        \n        train_history[\"train_loss\"].append(train_loss)\n        train_history[\"valid_loss\"].append(valid_loss)\n        train_history[\"train_accuracy\"].append(train_accuracy)\n        train_history[\"valid_accuracy\"].append(valid_accuracy)\n        train_history[\"train_cohen_kappa\"].append(train_cohen_kappa)\n        train_history[\"valid_cohen_kappa\"].append(valid_cohen_kappa)\n        train_history[\"train_roc_auc\"].append(train_roc_auc)\n        train_history[\"valid_roc_auc\"].append(valid_roc_auc)\n        \n        save_checkpoint(model, optimizer, f'./effnet_{epoch}.pth')\n    \n    return train_history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = {       # should actually call it dependencies\n    'optimizer': lambda _model: get_optimizer(_model),\n    'scheduler': lambda optimizer: get_scheduler(optimizer),\n#     'loss_fn': lambda outputs, targets: bi_tempered_logistic_loss(outputs, targets, 0.7, 1.3, label_smoothing=0.2),\n#     'loss_fn': lambda outputs, targets: FocalLoss().to(config['device'](config['device_name']))(outputs, targets),\n    'loss_fn': lambda outputs, targets: TaylorSmoothedLoss(classes=5, smoothing=0.2).to(config['device'](config['device_name']))(outputs, targets),\n    'device': lambda d: xm.xla_device() if d == 'TPU' else torch.device('cuda' if torch.cuda.is_available() and d == 'GPU' else 'cpu'),\n    'device_name': dev,\n    'epochs': 25,\n    'num_workers': 4,\n    'batch_size': 48,\n    'labels': [0.0, 1.0, 2.0, 3.0, 4.0]\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if config['device_name'] == 'GPU':\n    torch.cuda.empty_cache()\n#     model, _ = load_checkpoint(get_model(), '../input/cassavapytorcheffnettrain/effnet_9.pth')\n    model = get_model()\n    history = train_model_with_data(0, model, train_dataset, valid_dataset)\nelif config['device_name'] == 'TPU':\n#     xmp.spawn(train_model_with_data, args=(get_model(), train_dataset, valid_dataset), nprocs=1, start_method='fork')\n    xmp.spawn(train_model_with_data, args=(get_model(), train_dataset, valid_dataset), nprocs=config['num_workers'], start_method='fork')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_fold(history, title):\n\n    fig = make_subplots(specs=[[{\"secondary_y\": True}]])\n\n    comx = np.arange(len(history['train_loss']))\n    trace = (        \n          px.line(history, x=comx, y='train_loss') \n         .add_trace(px.line(history, x=comx, y='valid_loss').data[0]) \n         .add_trace(px.line(history, x=comx, y='train_accuracy').data[0])\n         .add_trace(px.line(history, x=comx, y='valid_accuracy').data[0]) \n         \n        ).data\n\n    \n    fig.add_trace(trace[0], secondary_y=False) \n    fig.add_trace(trace[1], secondary_y=False) \n    fig.add_trace(trace[2], secondary_y=True)\n    fig.add_trace(trace[3], secondary_y=True)\n\n    \n    fig.data[0].line.dash='dash';fig.data[0].mode ='markers+lines';fig.data[0].line.color='#2ca02c';fig.data[0].line.width=3;fig.data[0].hovertemplate=None;fig.data[0].name='train loss' \n    fig.data[1].line.dash='dash';fig.data[1].mode ='markers+lines';fig.data[1].line.color='#d62728';fig.data[1].line.width=3;fig.data[1].hovertemplate=None;fig.data[1].name='valid loss'\n    fig.data[2].line.dash='dashdot';fig.data[2].mode ='markers+lines';fig.data[2].line.color='#ff7f0e';fig.data[2].line.width=3;fig.data[2].hovertemplate=None;fig.data[2].name='train accuracy'\n    fig.data[3].line.dash='dashdot';fig.data[3].mode ='markers+lines';fig.data[3].line.color='#1f77b4';fig.data[3].line.width=3;fig.data[3].hovertemplate=None;fig.data[3].name='valid accuracy'\n    \n    \n    # Set x-axis title\n    fig.update_xaxes(title_text=\"Epoch\")\n\n    # Set y-axes titles\n    fig.update_yaxes(title_text=\"Loss\", secondary_y=False)    \n    fig.update_yaxes(title_text=\"Accuracy\", secondary_y=True)\n    fig.update_layout(height=450, margin=dict(r=5, t=50, b=50, l=5), title_text='<b>'+title+'</b>', title_font_size=12, legend=dict(orientation='h',yanchor='top',y=1.03,xanchor='left',x=0.15))\n    fig.update_layout(font_size=12)\n    fig.for_each_annotation(lambda a: a.update(font=dict(size=14)))\n    fig.update_layout(hovermode=\"x unified\")\n    fig.update_traces(showlegend=True)\n    \n    fig.show()\n    \n\n    \ndef plot_confusion_matrix(label, pred):\n    c_matrix = metrics.confusion_matrix(label, pred, labels=range(len(LABELS)), normalize='true')\n    df = pd.DataFrame(c_matrix, index=LABELS, columns=LABELS)\n    df_text = np.around(df.values, decimals=2)\n\n    fig = ff.create_annotated_heatmap(df.values, annotation_text=df_text, x=LABELS, y=LABELS, colorscale='PuBu' )\n    fig.update_layout(font_size=9, height=450, margin=dict(r=5, t=50, b=50, l=5)) \n    \n    fig.show()       ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_fold(history, \"Training\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}