{"cells":[{"metadata":{},"cell_type":"markdown","source":"## References:"},{"metadata":{"trusted":false},"cell_type":"code","source":"https://www.kaggle.com/haqishen/baseline-modified-from-previous-competition\n\nhttps://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-train-amp-aug\n    \nhttps://www.kaggle.com/piantic/train-cassava-starter-using-various-loss-funcs\n\nAdamP Optimizer: Slowing Down the Slowdown for Momentum Optimizers on Scale-invariant Weights\n    https://arxiv.org/abs/2006.08217\n    https://www.kaggle.com/seriousran/adam-vs-adamp-iclr-2021\n    https://github.com/clovaai/AdamP\n\nSAM Optimizer: Sharpness-Aware Minimization for Efficiently Improving Generalization\n    https://arxiv.org/pdf/2010.01412v2.pdf\n    https://github.com/davda54/sam","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"!pip install git+https://github.com/ildoonet/pytorch-gradual-warmup-lr.git","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"\"\"\"\n    import external libraries\n\"\"\"\nfrom warmup_scheduler import GradualWarmupScheduler    \n    \npackage_paths = [\n    '../input/timm-models/pytorch-image-models-master',\n]\n\nimport sys; \n\nfor pth in package_paths:\n    sys.path.append(pth)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom glob import glob\nimport os, time, gc, random, warnings, joblib\nfrom sklearn.model_selection import StratifiedKFold\nimport sklearn\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\n\nimport cv2\nfrom skimage import io\nfrom scipy.ndimage.interpolation import zoom\nimport timm\nimport albumentations\nfrom albumentations.pytorch import ToTensorV2\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\n\nimport torchvision\nfrom torchvision import transforms\nfrom tqdm import tqdm\nimport argparse","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"Config = {\n    'seed': 42,\n    'data_image_root': '../input/cassava-leaf-disease-classification/train_images/',\n    #'data_image_root': '../input/cassava-leaf-disease-merged/train/', # if you want to consider 2019 data for early stage of model training \n    'train_set': '../input/train-folds/train_folds.csv',\n    #'train_set2': '../input/train-folds/train_2019_folds.csv', # if you want to consider 2019 data for early stage of model training \n    'image_size': 512, \n    'model_arch': 'tf_efficientnet_b4_ns', # 'gluon_seresnext101_32x4d', 'seresnext50_32x4d'\n    'epochs': 10,\n    'device': 'cuda:0',\n    'kernel_type': 'training_stage',\n    'cutmix_alpha': 1,\n    'output_label': True,\n    'one_hot_label': False,\n    'do_cutmix': False,\n    'do_mixup': False,\n    'num_workers': 4,\n    'batch_size': 12,\n    'pin_memory': True,\n    'init_lr': 1e-5,\n    'warmup_factor': 10,\n    'warmup_epoch': 1,\n    'weight_decay': 1e-6,\n    'rho': 0.05,\n    'num_classes': 5,\n    'beta': 0.0,\n    'filter_bias_and_bn': True,\n    'resume': False\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utils"},{"metadata":{"trusted":false},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    return im_rgb\n\ndef rand_bbox(size, lam):\n    W = size[0]\n    H = size[1]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n    \n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n    \n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":false},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, df, data_image_root, transforms=None, output_label=True, one_hot_label=False, \n                 do_cutmix=False, cutmix_params={'alpha': 1.}, \n                 do_mixup=False):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_image_root = data_image_root\n        self.do_cutmix = do_cutmix\n        self.cutmix_params = cutmix_params\n        self.do_mixup = do_mixup\n        \n        self.output_label = output_label\n        self.one_hot_label = one_hot_label\n        \n        if output_label == True: # set to true for train set\n            self.labels = self.df['label'].values\n            \n            if one_hot_label is True:\n                self.labels = np.eye(self.df['label'].max()+1)[self.labels]\n                \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        # get labels\n        if self.output_label: # set to true for train set\n            target = self.labels[index]\n        \n        # retrieve image from assigned directory\n        img = get_img(\"{}/{}\".format(self.data_image_root, self.df.loc[index]['image_id']))\n        \n        # do augmentation/s\n        if self.transforms:\n            img = self.transforms(image=img)['image'] \n                \n        if self.do_cutmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            with torch.no_grad():\n                cutmix_ix = np.random.choice(self.df.index, size=1)[0]\n                cutmix_img = get_img(\"{}/{}\".format(self.data_image_root, self.df.iloc[cutmix_ix]['image_id']))\n                \n                if self.transforms:\n                    cutmix_img = self.transforms(image=cutmix_img)['image']\n                    \n                lam = np.clip(np.random.beta(self.cutmix_params['alpha'], self.cutmix_params['alpha']), 0.3, 0.7)\n                bbx1, bby1, bbx2, bby2 = rand_bbox((Config['image_size'], Config['image_size']), lam)\n                img[:, bbx1:bbx2, bby1:bby2] = cutmix_img[:, bbx1:bbx2, bby1:bby2]\n                rate = 1 - ((bbx2-bbx1) * (bby2-bby1) / (Config['image_size'] * Config['image_size']))\n                target = rate*target + (1.-rate)*self.labels[cutmix_ix]\n                \n        if self.do_mixup and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            with torch.no_grad():\n                mixup_idx = np.random.choice(self.df.index, size=1)[0]\n                mixup_img = get_img(\"{}/{}\".format(self.data_image_root, self.df.iloc[mixup_idx, :]['image_id']))\n                \n                if self.transforms:\n                    mixup_img = self.transforms(image=mixup_img)['image']\n                    \n                lam = np.random.beta(1.0, 1.0)\n                lam = max(lam, 1-lam)\n                \n                img = lam * img + (1 - lam) * mixup_img\n                target = lam*target + (1.-lam)*self.labels[mixup_idx]\n                \n        if self.output_label==True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Augmentations"},{"metadata":{"trusted":false},"cell_type":"code","source":"\"\"\"\nNote: use these augmentations with combination of cutmix by setting it to True during early stage of model training\n\"\"\"\n#def get_train_transforms():\n#    return albumentations.Compose([\n#        albumentations.RandomResizedCrop(Config['image_size'], Config['image_size']),\n#        albumentations.Transpose(p=0.5),\n#        albumentations.HorizontalFlip(p=0.5),\n#        albumentations.VerticalFlip(p=0.5),\n#        albumentations.ShiftScaleRotate(p=0.5),\n#        albumentations.Cutout(max_h_size=int(Config['image_size'] * 0.4), max_w_size=int(Config['image_size'] * 0.4), num_holes=1, p=0.5),\n#        albumentations.Normalize(mean=[0.485,0.456,0.406], std=[0.229,0.224,0.225], max_pixel_value=255,p=1.0),\n#        ToTensorV2(p=1.0),\n#    ], p=1.0)\n\n\"\"\"\nNote: for fine-tuning, use these augmentations, disable cutmix\n\"\"\"\ndef get_train_transforms():\n    return albumentations.Compose([\n        albumentations.RandomCrop(Config['image_size'], Config['image_size']),\n        albumentations.Resize(Config['image_size'], Config['image_size']),\n        albumentations.HorizontalFlip(p=0.5),\n        albumentations.VerticalFlip(0.5),\n        albumentations.Transpose(p=0.5),\n        albumentations.Rotate(limit=(-90, 90), p=0.5),\n        albumentations.OneOf([\n            albumentations.ShiftScaleRotate(),\n            albumentations.ElasticTransform(alpha=3)\n        ], p=0.5),\n        albumentations.OneOf([\n            albumentations.OpticalDistortion(distort_limit=1.0),\n            albumentations.GridDistortion(num_steps=5, distort_limit=1.0)\n        ], p=0.5),\n        albumentations.OneOf([\n            albumentations.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2),\n            albumentations.RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1)),\n            albumentations.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.2),\n            albumentations.FancyPCA(),\n            albumentations.CLAHE(clip_limit=4.0)\n        ], p=0.5),\n        albumentations.OneOf([\n            albumentations.IAAAffine(),\n            albumentations.IAAPerspective(),\n            albumentations.IAAPiecewiseAffine(),\n            albumentations.IAASuperpixels()\n        ], p=0.5),\n        albumentations.Cutout(max_h_size=int(Config['image_size'] * 0.375), max_w_size=int(Config['image_size'] * 0.375), num_holes=1, p=0.5),\n        albumentations.Normalize(mean=[0.485,0.456,0.406], std=[0.229,0.224,0.225], max_pixel_value=255, p=1.0),\n        ToTensorV2(p=1.0)\n    ])\n\ndef get_valid_transforms():\n    return albumentations.Compose([\n        albumentations.CenterCrop(Config['image_size'], Config['image_size'], p=1.0),\n        albumentations.Resize(Config['image_size'], Config['image_size']),\n        albumentations.Normalize(mean=[0.485,0.456,0.406], std=[0.229,0.224,0.225], max_pixel_value=255, p=1.0),\n        ToTensorV2(p=1.0),\n    ], p=1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def prepare_dataloader(df, df2, train_idx, valid_idx, data_image_root):\n    \n    train_ = df.loc[train_idx, :].reset_index(drop=True)\n    \n    #train_ = pd.concat([\n    #    df.loc[train_idx, :].reset_index(drop=True),\n    #    df2\n    #], ignore_index=True) # enable this if you want to consider 2019 data\n    \n    valid_ = df.loc[valid_idx, :].reset_index(drop=True)\n    \n    train_ds = CassavaDataset(train_, \n                              data_image_root, \n                              transforms=get_train_transforms(), \n                              output_label=Config['output_label'],\n                              one_hot_label=Config['one_hot_label'], \n                              do_cutmix=Config['do_cutmix'], \n                              do_mixup=Config['do_mixup'])\n    \n    valid_ds = CassavaDataset(valid_, \n                              data_image_root, \n                              transforms=get_valid_transforms(), \n                              output_label=Config['output_label'])\n    \n    train_dataloader = DataLoader(train_ds, \n                                  batch_size=Config['batch_size'], \n                                  pin_memory=Config['pin_memory'], \n                                  shuffle=True, \n                                  drop_last=True,\n                                  num_workers=Config['num_workers'])\n    \n    valid_dataloader = DataLoader(valid_ds, \n                                  batch_size=Config['batch_size'], \n                                  pin_memory=Config['pin_memory'], \n                                  shuffle=False, drop_last=False,\n                                  num_workers=Config['num_workers'])\n    \n    return train_dataloader, valid_dataloader","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":false},"cell_type":"code","source":"class CassavaImageClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        \n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        \n        ## SE-ResNext\n        #n_features = self.model.fc.in_features\n        #self.model.fc = nn.Linear(n_features, n_class)          \n        \n        ## EfficientNet\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train & Validation Step Function"},{"metadata":{"trusted":false},"cell_type":"code","source":"def train_one_epoch(model, epoch, criterion, optimizer, train_loader, device):\n    \n    model.train()\n    \n    t = time.time()\n    train_loss = []\n    image_preds_all = []\n    image_targets_all = []\n    \n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    num_iters = len(train_loader)\n    \n    for step, (imgs, image_labels) in pbar:\n        \n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()  \n                \n        # # first forward-backward step\n        image_preds = model(imgs)\n        loss = criterion(image_preds, image_labels)\n        \n        loss.mean().backward()\n        optimizer.first_step(zero_grad=True)\n        \n        # second forward-backward step\n        criterion(model(imgs), image_labels).mean().backward()\n        optimizer.second_step(zero_grad=True)\n        \n        image_pred = image_preds.softmax(1).argmax(1).detach()\n        image_preds_all.append(image_pred)\n        image_targets_all.append(image_labels)\n        \n        loss_np = loss.detach().cpu().numpy()\n        train_loss.append(loss_np)\n        smooth_loss = sum(train_loss[-100:]) / min(len(train_loss), 100)\n        pbar.set_description('loss: %.5f, smooth: %.5f' % (loss_np, smooth_loss))\n        \n    image_preds_all = torch.cat(image_preds_all).cpu().numpy()\n    image_targets_all = torch.cat(image_targets_all).cpu().numpy()\n    acc = (image_preds_all == image_targets_all).mean()\n        \n    return train_loss, acc\n\ndef valid_one_epoch(model, criterion, valid_loader, device):\n        \n    model.eval()\n    \n    t = time.time()\n    valid_loss = []\n    image_logits_all = []\n    image_preds_all = []\n    image_targets_all = []\n    \n    with torch.no_grad():\n        pbar = tqdm(valid_loader)\n        for (imgs, image_labels) in pbar:\n            imgs = imgs.to(device).float()\n            image_labels = image_labels.to(device).long()\n        \n            image_logits = model(imgs)\n            loss = criterion(image_logits, image_labels)\n\n            image_pred = image_logits.softmax(1).argmax(1).detach()\n            image_logits_all.append(image_logits)\n            image_preds_all.append(image_pred)\n            image_targets_all.append(image_labels)\n        \n            valid_loss.append(loss.detach().cpu().numpy())\n        valid_loss = np.mean(valid_loss)\n    \n    image_logits_all = torch.cat(image_logits_all).cpu().numpy()\n    image_preds_all = torch.cat(image_preds_all).cpu().numpy()\n    image_targets_all = torch.cat(image_targets_all).cpu().numpy()\n    acc = (image_preds_all == image_targets_all).mean()\n    \n    return valid_loss, acc ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Learning Rate Scheduler"},{"metadata":{"trusted":false},"cell_type":"code","source":"class GradualWarmupSchedulerV2(GradualWarmupScheduler):\n    def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None):\n        super(GradualWarmupSchedulerV2, self).__init__(optimizer, multiplier, total_epoch, after_scheduler)\n    def get_lr(self):\n        if self.last_epoch > self.total_epoch:\n            if self.after_scheduler:\n                if not self.finished:\n                    self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs]\n                    self.finished = True\n                return self.after_scheduler.get_lr()\n            return [base_lr * self.multiplier for base_lr in self.base_lrs]\n        if self.multiplier == 1.0:\n            return [base_lr * (float(self.last_epoch) / self.total_epoch) for base_lr in self.base_lrs]\n        else:\n            return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in self.base_lrs]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss Functions / Criterions"},{"metadata":{"trusted":false},"cell_type":"code","source":"class LabelSmoothingLoss(nn.Module): \n    def __init__(self, classes=5, smoothing=0.06, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))\n    \nclass FocalCosineLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, xent=.1): #change from gamma value from 2 to 1.934; alpha from 1 to 0.3868\n        super(FocalCosineLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n        self.xent = xent\n\n        self.y = torch.Tensor([1]).cuda()\n\n    def forward(self, input, target, reduction=\"mean\"):\n        cosine_loss = F.cosine_embedding_loss(input, F.one_hot(target, num_classes=input.size(-1)), self.y, reduction=reduction)\n\n        cent_loss = F.cross_entropy(F.normalize(input), target, reduce=False)\n        pt = torch.exp(-cent_loss)\n        focal_loss = self.alpha * (1-pt)**self.gamma * cent_loss\n\n        if reduction == \"mean\":\n            focal_loss = torch.mean(focal_loss)\n\n        return cosine_loss + self.xent * focal_loss\n    \ndef log_t(u, t):\n    \"\"\"Compute log_t for `u'.\"\"\"\n    if t==1.0:\n        return u.log()\n    else:\n        return (u.pow(1.0 - t) - 1.0) / (1.0 - t)\n\ndef exp_t(u, t):\n    \"\"\"Compute exp_t for `u'.\"\"\"\n    if t==1:\n        return u.exp()\n    else:\n        return (1.0 + (1.0-t)*u).relu().pow(1.0 / (1.0 - t))\n\ndef compute_normalization_fixed_point(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t > 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same shape as activation with the last dimension being 1.\n    \"\"\"\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations_step_0 = activations - mu\n\n    normalized_activations = normalized_activations_step_0\n\n    for _ in range(num_iters):\n        logt_partition = torch.sum(\n                exp_t(normalized_activations, t), -1, keepdim=True)\n        normalized_activations = normalized_activations_step_0 * \\\n                logt_partition.pow(1.0-t)\n\n    logt_partition = torch.sum(\n            exp_t(normalized_activations, t), -1, keepdim=True)\n    normalization_constants = - log_t(1.0 / logt_partition, t) + mu\n\n    return normalization_constants\n\ndef compute_normalization_binary_search(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t < 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (< 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations = activations - mu\n\n    effective_dim = \\\n        torch.sum(\n                (normalized_activations > -1.0 / (1.0-t)).to(torch.int32),\n            dim=-1, keepdim=True).to(activations.dtype)\n\n    shape_partition = activations.shape[:-1] + (1,)\n    lower = torch.zeros(shape_partition, dtype=activations.dtype, device=activations.device)\n    upper = -log_t(1.0/effective_dim, t) * torch.ones_like(lower)\n\n    for _ in range(num_iters):\n        logt_partition = (upper + lower)/2.0\n        sum_probs = torch.sum(\n                exp_t(normalized_activations - logt_partition, t),\n                dim=-1, keepdim=True)\n        update = (sum_probs < 1.0).to(activations.dtype)\n        lower = torch.reshape(\n                lower * update + (1.0-update) * logt_partition,\n                shape_partition)\n        upper = torch.reshape(\n                upper * (1.0 - update) + update * logt_partition,\n                shape_partition)\n\n    logt_partition = (upper + lower)/2.0\n    return logt_partition + mu\n\nclass ComputeNormalization(torch.autograd.Function):\n    \"\"\"\n    Class implementing custom backward pass for compute_normalization. See compute_normalization.\n    \"\"\"\n    @staticmethod\n    def forward(ctx, activations, t, num_iters):\n        if t < 1.0:\n            normalization_constants = compute_normalization_binary_search(activations, t, num_iters)\n        else:\n            normalization_constants = compute_normalization_fixed_point(activations, t, num_iters)\n\n        ctx.save_for_backward(activations, normalization_constants)\n        ctx.t=t\n        return normalization_constants\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        activations, normalization_constants = ctx.saved_tensors\n        t = ctx.t\n        normalized_activations = activations - normalization_constants \n        probabilities = exp_t(normalized_activations, t)\n        escorts = probabilities.pow(t)\n        escorts = escorts / escorts.sum(dim=-1, keepdim=True)\n        grad_input = escorts * grad_output\n        \n        return grad_input, None, None\n\ndef compute_normalization(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example. \n    Backward pass is implemented.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    return ComputeNormalization.apply(activations, t, num_iters)\n\ndef tempered_sigmoid(activations, t, num_iters = 5):\n    \"\"\"Tempered sigmoid function.\n    Args:\n      activations: Activations for the positive class for binary classification.\n      t: Temperature tensor > 0.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_probabilities = tempered_softmax(internal_activations, t, num_iters)\n    return internal_probabilities[..., 0]\n\n\ndef tempered_softmax(activations, t, num_iters=5):\n    \"\"\"Tempered softmax function.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature > 1.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    if t == 1.0:\n        return activations.softmax(dim=-1)\n\n    normalization_constants = compute_normalization(activations, t, num_iters)\n    return exp_t(activations - normalization_constants, t)\n\ndef bi_tempered_binary_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing = 0.0,\n        num_iters=5,\n        reduction='mean'):\n\n    \"\"\"Bi-Tempered binary logistic loss.\n    Args:\n      activations: A tensor containing activations for class 1.\n      labels: A tensor with shape as activations, containing probabilities for class 1\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A loss tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_labels = torch.stack([labels.to(activations.dtype),\n        1.0 - labels.to(activations.dtype)],\n        dim=-1)\n    return bi_tempered_logistic_loss(internal_activations, \n            internal_labels,\n            t1,\n            t2,\n            label_smoothing = label_smoothing,\n            num_iters = num_iters,\n            reduction = reduction)\n\ndef bi_tempered_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing=0.0,\n        num_iters=5,\n        reduction = 'mean'):\n\n    \"\"\"Bi-Tempered Logistic Loss.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      labels: A tensor with shape and dtype as activations (onehot), \n        or a long tensor of one dimension less than activations (pytorch standard)\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing parameter between [0, 1). Default 0.0.\n      num_iters: Number of iterations to run the method. Default 5.\n      reduction: ``'none'`` | ``'mean'`` | ``'sum'``. Default ``'mean'``.\n        ``'none'``: No reduction is applied, return shape is shape of\n        activations without the last dimension.\n        ``'mean'``: Loss is averaged over minibatch. Return shape (1,)\n        ``'sum'``: Loss is summed over minibatch. Return shape (1,)\n    Returns:\n      A loss tensor.\n    \"\"\"\n\n    if len(labels.shape)<len(activations.shape): #not one-hot\n        labels_onehot = torch.zeros_like(activations)\n        labels_onehot.scatter_(1, labels[..., None], 1)\n    else:\n        labels_onehot = labels\n\n    if label_smoothing > 0:\n        num_classes = labels_onehot.shape[-1]\n        labels_onehot = ( 1 - label_smoothing * num_classes / (num_classes - 1) ) \\\n                * labels_onehot + \\\n                label_smoothing / (num_classes - 1)\n\n    probabilities = tempered_softmax(activations, t2, num_iters)\n\n    loss_values = labels_onehot * log_t(labels_onehot + 1e-10, t1) \\\n            - labels_onehot * log_t(probabilities, t1) \\\n            - labels_onehot.pow(2.0 - t1) / (2.0 - t1) \\\n            + probabilities.pow(2.0 - t1) / (2.0 - t1)\n    loss_values = loss_values.sum(dim = -1) #sum over classes\n\n    if reduction == 'none':\n        return loss_values\n    if reduction == 'sum':\n        return loss_values.sum()\n    if reduction == 'mean':\n        return loss_values.mean()\n\nclass BiTemperedLogisticLoss(nn.Module): \n    def __init__(self, t1, t2, smoothing=0.0): \n        super(BiTemperedLogisticLoss, self).__init__() \n        self.t1 = t1\n        self.t2 = t2\n        self.smoothing = smoothing\n    def forward(self, logit_label, truth_label):\n        loss_label = bi_tempered_logistic_loss(\n            logit_label, truth_label,\n            t1=self.t1, t2=self.t2,\n            label_smoothing=self.smoothing,\n            reduction='none'\n        )\n        \n        loss_label = loss_label.mean()\n        return loss_label\n    \nclass TaylorSoftmax(nn.Module):\n    '''\n    This is the autograd version\n    '''\n    def __init__(self, dim=1, n=2):\n        super(TaylorSoftmax, self).__init__()\n        assert n % 2 == 0\n        self.dim = dim\n        self.n = n\n\n    def forward(self, x):\n        '''\n        usage similar to nn.Softmax:\n            >>> mod = TaylorSoftmax(dim=1, n=4)\n            >>> inten = torch.randn(1, 32, 64, 64)\n            >>> out = mod(inten)\n        '''\n        fn = torch.ones_like(x)\n        denor = 1.\n        for i in range(1, self.n+1):\n            denor *= i\n            fn = fn + x.pow(i) / denor\n        out = fn / fn.sum(dim=self.dim, keepdims=True)\n        return out\n\n\n##\n# version 1: use torch.autograd\nclass BaseTaylorCrossEntropyLoss(nn.Module):\n    '''\n    This is the autograd version\n    '''\n    def __init__(self, n=2, ignore_index=-1, reduction='mean'):\n        super(BaseTaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n\n    def forward(self, logits, labels):\n        '''\n        usage similar to nn.CrossEntropyLoss:\n            >>> crit = TaylorCrossEntropyLoss(n=4)\n            >>> inten = torch.randn(1, 10, 64, 64)\n            >>> label = torch.randint(0, 10, (1, 64, 64))\n            >>> out = crit(inten, label)\n        '''\n        log_probs = self.taylor_softmax(logits).log()\n        loss = F.nll_loss(log_probs, labels, reduction=self.reduction,\n                ignore_index=self.ignore_index)\n        return loss\n\nclass TaylorCrossEntropyLoss(nn.Module):\n\n    def __init__(self, n=2, ignore_index=-1, reduction='mean', smoothing=0.2):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n        self.lab_smooth = LabelSmoothingLoss(Config['num_classes'], smoothing=smoothing)\n\n    def forward(self, logits, labels):\n\n        log_probs = self.taylor_softmax(logits).log()\n        loss = self.lab_smooth(log_probs, labels)\n        return loss\n    \n    \n## reference: https://github.com/HanxunH/Active-Passive-Losses    \nclass NormalizedCrossEntropy(nn.Module):\n    def __init__(self, num_classes, scale=1.0):\n        super(NormalizedCrossEntropy, self).__init__()\n        #self.device = device\n        self.num_classes = num_classes\n        self.scale = scale\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=2)\n\n    def forward(self, pred, labels):\n        pred = F.log_softmax(pred, dim=1)\n        #pred = self.taylor_softmax(pred).log()\n        label_one_hot = torch.nn.functional.one_hot(labels, self.num_classes).float()\n        nce = -1 * torch.sum(label_one_hot * pred, dim=1) / (- pred.sum(dim=1))\n        return self.scale * nce.mean()\n    \nclass ReverseCrossEntropy(nn.Module):\n    def __init__(self, num_classes, scale=1.0):\n        super(ReverseCrossEntropy, self).__init__()\n        #self.device = device\n        self.num_classes = num_classes\n        self.scale = scale\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=2)\n\n    def forward(self, pred, labels):\n        pred = F.softmax(pred, dim=1)\n        #pred = self.taylor_softmax(pred).log()\n        pred = torch.clamp(pred, min=1e-7, max=1.0)\n        label_one_hot = torch.nn.functional.one_hot(labels, self.num_classes).float()\n        label_one_hot = torch.clamp(label_one_hot, min=1e-4, max=1.0)\n        rce = (-1*torch.sum(pred * torch.log(label_one_hot), dim=1))\n        return self.scale * rce.mean()\n    \n\nclass NCEandRCE(nn.Module):\n    def __init__(self, alpha, beta, num_classes):\n        super(NCEandRCE, self).__init__()\n        self.num_classes = num_classes\n        self.nce = NormalizedCrossEntropy(scale=alpha, num_classes=num_classes)\n        self.rce = ReverseCrossEntropy(scale=beta, num_classes=num_classes)\n\n    def forward(self, pred, labels):\n        return self.nce(pred, labels) + self.rce(pred, labels)\n    \nclass ComboLoss(nn.Module):\n    def __init__(self):\n        super(ComboLoss, self).__init__()\n        \n        self.loss_1 = BiTemperedLogisticLoss(t1=0.32, t2=1.0, smoothing=0.0)\n        self.loss_2 = TaylorCrossEntropyLoss(smoothing=0.06)\n        self.loss_3 = NCEandRCE(alpha=0.17, beta=0.34, num_classes=5)\n        #self.loss_4 = FocalCosineLoss(alpha=0.3868, gamma=1.934, xent=0.1)\n        \n    def forward(self, preds, labels):\n        return self.loss_1(preds, labels) + self.loss_2(preds, labels) + self.loss_3(preds, labels) #+ self.loss_4(preds, labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Optimizers"},{"metadata":{"trusted":false},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim.optimizer import Optimizer, required\nimport math\n\nclass AdamP(Optimizer):\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,\n                 weight_decay=0, delta=0.1, wd_ratio=0.1, nesterov=False):\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay,\n                        delta=delta, wd_ratio=wd_ratio, nesterov=nesterov)\n        super(AdamP, self).__init__(params, defaults)\n\n    def _channel_view(self, x):\n        return x.view(x.size(0), -1)\n\n    def _layer_view(self, x):\n        return x.view(1, -1)\n\n    def _cosine_similarity(self, x, y, eps, view_func):\n        x = view_func(x)\n        y = view_func(y)\n\n        return F.cosine_similarity(x, y, dim=1, eps=eps).abs_()\n\n    def _projection(self, p, grad, perturb, delta, wd_ratio, eps):\n        wd = 1\n        expand_size = [-1] + [1] * (len(p.shape) - 1)\n        for view_func in [self._channel_view, self._layer_view]:\n\n            cosine_sim = self._cosine_similarity(grad, p.data, eps, view_func)\n\n            if cosine_sim.max() < delta / math.sqrt(view_func(p.data).size(1)):\n                p_n = p.data / view_func(p.data).norm(dim=1).view(expand_size).add_(eps)\n                perturb -= p_n * view_func(p_n * perturb).sum(dim=1).view(expand_size)\n                wd = wd_ratio\n\n                return perturb, wd\n\n        return perturb, wd\n\n    def step(self, closure=None):\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n\n                grad = p.grad.data\n                beta1, beta2 = group['betas']\n                nesterov = group['nesterov']\n\n                state = self.state[p]\n\n                # State initialization\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p.data)\n                    state['exp_avg_sq'] = torch.zeros_like(p.data)\n\n                # Adam\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n\n                state['step'] += 1\n                bias_correction1 = 1 - beta1 ** state['step']\n                bias_correction2 = 1 - beta2 ** state['step']\n\n                exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)\n                exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)\n\n                denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(group['eps'])\n                step_size = group['lr'] / bias_correction1\n\n                if nesterov:\n                    perturb = (beta1 * exp_avg + (1 - beta1) * grad) / denom\n                else:\n                    perturb = exp_avg / denom\n\n                # Projection\n                wd_ratio = 1\n                if len(p.shape) > 1:\n                    perturb, wd_ratio = self._projection(p, grad, perturb, group['delta'], group['wd_ratio'], group['eps'])\n\n                # Weight decay\n                if group['weight_decay'] > 0:\n                    p.data.mul_(1 - group['lr'] * group['weight_decay'] * wd_ratio)\n\n                # Step\n                p.data.add_(perturb, alpha=-step_size)\n\n        return loss\n\nclass SGDP(Optimizer):\n    def __init__(self, params, lr=required, momentum=0, dampening=0,\n                 weight_decay=0, nesterov=False, eps=1e-8, delta=0.1, wd_ratio=0.1):\n        defaults = dict(lr=lr, momentum=momentum, dampening=dampening, weight_decay=weight_decay,\n                        nesterov=nesterov, eps=eps, delta=delta, wd_ratio=wd_ratio)\n        super(SGDP, self).__init__(params, defaults)\n\n    def _channel_view(self, x):\n        return x.view(x.size(0), -1)\n\n    def _layer_view(self, x):\n        return x.view(1, -1)\n\n    def _cosine_similarity(self, x, y, eps, view_func):\n        x = view_func(x)\n        y = view_func(y)\n\n        return F.cosine_similarity(x, y, dim=1, eps=eps).abs_()\n\n    def _projection(self, p, grad, perturb, delta, wd_ratio, eps):\n        wd = 1\n        expand_size = [-1] + [1] * (len(p.shape) - 1)\n        for view_func in [self._channel_view, self._layer_view]:\n\n            cosine_sim = self._cosine_similarity(grad, p.data, eps, view_func)\n\n            if cosine_sim.max() < delta / math.sqrt(view_func(p.data).size(1)):\n                p_n = p.data / view_func(p.data).norm(dim=1).view(expand_size).add_(eps)\n                perturb -= p_n * view_func(p_n * perturb).sum(dim=1).view(expand_size)\n                wd = wd_ratio\n\n                return perturb, wd\n\n        return perturb, wd\n\n    def step(self, closure=None):\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n            momentum = group['momentum']\n            dampening = group['dampening']\n            nesterov = group['nesterov']\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data\n                state = self.state[p]\n\n                # State initialization\n                if len(state) == 0:\n                    state['momentum'] = torch.zeros_like(p.data)\n\n                # SGD\n                buf = state['momentum']\n                buf.mul_(momentum).add_(grad, alpha=1 - dampening)\n                if nesterov:\n                    d_p = grad + momentum * buf\n                else:\n                    d_p = buf\n\n                # Projection\n                wd_ratio = 1\n                if len(p.shape) > 1:\n                    d_p, wd_ratio = self._projection(p, grad, d_p, group['delta'], group['wd_ratio'], group['eps'])\n\n                # Weight decay\n                if group['weight_decay'] > 0:\n                    p.data.mul_(1 - group['lr'] * group['weight_decay'] * wd_ratio / (1-momentum))\n\n                # Step\n                p.data.add_(d_p, alpha=-group['lr'])\n\n        return loss\n\nclass SAM(torch.optim.Optimizer):\n    def __init__(self, params, base_optimizer, rho=0.05, **kwargs):\n        assert rho >= 0.0, f\"Invalid rho, should be non-negative: {rho}\"\n\n        defaults = dict(rho=rho, **kwargs)\n        super(SAM, self).__init__(params, defaults)\n\n        self.base_optimizer = base_optimizer(self.param_groups, **kwargs)\n        self.param_groups = self.base_optimizer.param_groups\n\n    @torch.no_grad()\n    def first_step(self, zero_grad=False):\n        grad_norm = self._grad_norm()\n        for group in self.param_groups:\n            scale = group[\"rho\"] / (grad_norm + 1e-12)\n\n            for p in group[\"params\"]:\n                if p.grad is None: continue\n                e_w = p.grad * scale.to(p)\n                p.add_(e_w)  # climb to the local maximum \"w + e(w)\"\n                self.state[p][\"e_w\"] = e_w\n\n        if zero_grad: self.zero_grad()\n\n    @torch.no_grad()\n    def second_step(self, zero_grad=False):\n        for group in self.param_groups:\n            for p in group[\"params\"]:\n                if p.grad is None: continue\n                p.sub_(self.state[p][\"e_w\"])  # get back to \"w\" from \"w + e(w)\"\n\n        self.base_optimizer.step()  # do the actual \"sharpness-aware\" update\n\n        if zero_grad: self.zero_grad()\n\n    def step(self, closure=None):\n        raise NotImplementedError(\"SAM doesn't work like the other optimizers, you should first call `first_step` and the `second_step`; see the documentation for more info.\")\n\n    def _grad_norm(self):\n        shared_device = self.param_groups[0][\"params\"][0].device  # put everything on the same device, in case of model parallelism\n        norm = torch.norm(\n                    torch.stack([\n                        p.grad.norm(p=2).to(shared_device)\n                        for group in self.param_groups for p in group[\"params\"]\n                        if p.grad is not None\n                    ]),\n                    p=2\n               )\n        return norm","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"## reference: https://github.com/rwightman/pytorch-image-models/blob/master/timm/optim/optim_factory.py\ndef add_weight_decay(model, weight_decay=1e-5, skip_list=()):\n    decay = []\n    no_decay = []\n    for name, param in model.named_parameters():\n        if not param.requires_grad:\n            continue  # frozen weights\n        if len(param.shape) == 1 or name.endswith(\".bias\") or name in skip_list:\n            no_decay.append(param)\n        else:\n            decay.append(param)\n    return [\n        {'params': no_decay, 'weight_decay': 0.},\n        {'params': decay, 'weight_decay': weight_decay}]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def main(fold):\n   \n    seed_everything(Config['seed'])\n    device = torch.device(Config['device'])\n    \n    train = pd.read_csv(Config['train_set'])\n    train2 = pd.read_csv(Config['train_set2'])\n    \n    train_idx = np.where((train['fold'] != fold))[0]\n    valid_idx = np.where((train['fold'] == fold))[0]\n        \n    num_classes = train.label.nunique()\n\n    print('Training with {} started'.format(fold))\n    \n    print(\"Training set size :\", len(train_idx), \"Validation set size :\", len(valid_idx))\n    train_loader, valid_loader = prepare_dataloader(train, train2, train_idx, valid_idx, data_image_root=Config['data_image_root'])\n        \n    model = CassavaImageClassifier(Config['model_arch'], num_classes, pretrained=True).to(device)\n    \n    # load trained model\n    if Config['resume']:\n        #model.load_state_dict(torch.load('model_path'))\n    \n    # reference: https://github.com/rwightman/pytorch-image-models/blob/master/timm/optim/optim_factory.py\n    weight_decay = Config['weight_decay']\n    if weight_decay and Config['filter_bias_and_bn']:\n        skip = {}\n        if hasattr(model, 'no_weight_decay'):\n            skip = model.no_weight_decay()\n        model_parameters = add_weight_decay(model, weight_decay, skip)\n        weight_decay = 0.\n    else:\n        model_parameters = model.parameters()\n    \n    # set optimizer\n    base_optimizer = AdamP\n    optimizer = SAM(model_parameters,\n                    base_optimizer,\n                    rho=Config['rho'],\n                    lr=Config['init_lr'],\n                    weight_decay=weight_decay,\n                    nesterov=True)\n\n    # set learning rate scheduler\n    scheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, Config['epochs']-Config['warmup_epoch'])\n    scheduler = GradualWarmupScheduler(optimizer, multiplier=Config['warmup_factor'], total_epoch=Config['warmup_epoch'], after_scheduler=scheduler_cosine)\n    \n    # set loss function\n    criterion = ComboLoss().to(device)\n\n    best_accuracy = 0\n        \n    for epoch in range(1, Config['epochs']+1):\n        print(time.ctime(), 'Epoch :', epoch)\n        scheduler.step(epoch-1)\n\n        train_loss, train_accuracy = train_one_epoch(model, epoch, criterion, optimizer, train_loader, device)\n        valid_loss, valid_accuracy = valid_one_epoch(model, criterion, valid_loader, device)\n        \n        content = time.ctime() + ' ' + f'Epoch {epoch}, lr: {optimizer.param_groups[0][\"lr\"]:.7f}, train loss: {np.mean(train_loss):.5f}, train accuracy: {(train_accuracy):.5f}, valid loss: {np.mean(valid_loss):.5f}, validation accuracy: {(valid_accuracy):.5f}'\n        print(content)\n        with open('log_{}_{}_fold_{}.txt'.format(Config['kernel_type'], Config['model_arch'], fold), 'a') as appender:\n            appender.write(content + '\\n')\n                \n        ## check maximum accuracy per validation and save best model\n        if valid_accuracy > best_accuracy:\n            print('valid accuracy ({:.6f} --> {:.6f}).  Saving model ...'.format(best_accuracy, valid_accuracy))\n            torch.save(model.state_dict(), '{}_{}_best_fold_{}.pth'.format(Config['kernel_type'], Config['model_arch'], fold))\n            best_accuracy = valid_accuracy\n       \n    del model, optimizer, train_loader, valid_loader, scheduler_cosine, scheduler\n    torch.cuda.empty_cache()\n    gc.collect()   \n    \n    return best_accuracy","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#main(fold=1)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}