{"cells":[{"metadata":{},"cell_type":"markdown","source":"# CONFIG"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"CFG = {\n #general setting\n 'fold_num': 5,\n 'only_one_fold':True,\n 'epochs': 7,\n 'seed': 719,\n 'train_bs': 16,\n 'valid_bs': 32,\n 'num_workers': 4,\n 'accum_iter': 1, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n 'verbose_step': 1,\n 'device': 'tpu',#device can be either cuda:0 or tpu\n #dataset setting(change between 2020 dataset and 2019+2020 dataset)\n 'train_imgs_path':'../input/cassava-leaf-disease-classification/train_images',\n #data cleaning setting\n 'train_csv_path':'../input/cassava-leaf-disease-classification/train.csv',\n #model archetecture setting\n 'model_arch': 'tf_efficientnet_b4_ns',\n #augmentation\n 'img_size': 512,\n #loss function setting\n 'loss_function':'CrossEntropyLoss',\n #(CrossEntropyLoss,LabelSmoothingLoss,FocalLoss,FocalCosineLoss,SymmetricCrossEntropy,BiTemperedLogisticLoss,TaylorCrossEntropyLoss)\n 'LabelSmoothingLoss_smoothing':0.1,\n 'FocalLoss_alpha':1, \n 'FocalLoss_gamma':2,\n 'FocalCosineLoss_alpha':1,\n 'FocalCosineLoss_gamma':2, \n 'FocalCosineLoss_xent':0.1,\n 'SymmetricCrossEntropy_alpha':0.1, \n 'SymmetricCrossEntropy_beta':1.0,\n 'BiTemperedLogisticLoss_t1':0.3, \n 'BiTemperedLogisticLoss_t2':1.0, \n 'BiTemperedLogisticLoss_smoothing':0.0,\n 'TaylorCrossEntropyLoss_smoothing':0.05,\n 'TaylorCrossEntropyLoss_n':2,\n #optimizer setting\n 'optimizer':'Adam',#(Adam,SGD)\n 'Adam_lr': 1e-4,\n 'Adam_weight_decay':1e-6,#regularization,add l2 loss\n 'SGD_lr':1e-4,\n 'SGD_momentum':0.9,\n #schedular setting\n 'lr_schedular':'CosineAnnealingLR',\n #(StepLR,ExponentialLR,CosineAnnealingLR,ReduceLROnPlateau,CosineAnnealingWarmRestarts)\n 'StepLR_step_size':2,\n 'StepLR_gamma':0.5,\n 'ExponentialLR_gamma':0.9,\n 'CosineAnnealingLR_T_max':10,\n 'CosineAnnealingLR_eta_min':0,\n 'ReduceLROnPlateau_factor':0.5,\n 'ReduceLROnPlateau_patience':1,\n 'ReduceLROnPlateau_threshold':0.0001,\n 'ReduceLROnPlateau_min_lr':0,\n 'CosineAnnealingWarmRestarts_T_0':10,\n 'CosineAnnealingWarmRestarts_min_lr':1e-6,\n \n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"package_paths = ['../input/pytorch-image-models/pytorch-image-models-master']\nimport sys; \n\nfor pth in package_paths:\n sys.path.append(pth)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TPU package installation"},{"metadata":{"trusted":true},"cell_type":"code","source":"if CFG['device']!='cuda:0':\n !curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py > /dev/null\n !python pytorch-xla-env-setup.py --version nightly > /dev/null\n \n import torch_xla.core.xla_model as xm\n import torch_xla.distributed.parallel_loader as pl\n import torch_xla.distributed.xla_multiprocessing as xmp\n import torch_xla.utils.serialization as xser\n import gc\n import os\n\n os.environ['XLA_USE_BF16']=\"1\"\n os.environ['XLA_TENSOR_ALLOCATOR_MAXSIZE'] = '100000000'\n\n import warnings\n warnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2,gc\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\n\nimport timm\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\n#from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def seed_everything(seed):\n random.seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = True\n \ndef get_img(path):\n im_bgr = cv2.imread(path)\n im_rgb = im_bgr[:, :, ::-1]\n #print(im_rgb)\n return im_rgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def rand_bbox(size, lam):\n W = size[0]\n H = size[1]\n cut_rat = np.sqrt(1. - lam)\n cut_w = np.int(W * cut_rat)\n cut_h = np.int(H * cut_rat)\n\n # uniform\n cx = np.random.randint(W)\n cy = np.random.randint(H)\n\n bbx1 = np.clip(cx - cut_w // 2, 0, W)\n bby1 = np.clip(cy - cut_h // 2, 0, H)\n bbx2 = np.clip(cx + cut_w // 2, 0, W)\n bby2 = np.clip(cy + cut_h // 2, 0, H)\n return bbx1, bby1, bbx2, bby2\n\n\nclass CassavaDataset(Dataset):\n def __init__(self, df, data_root, \n transforms=None, \n output_label=True, \n one_hot_label=False,\n do_fmix=False, \n fmix_params={\n 'alpha': 1., \n 'decay_power': 3., \n 'shape': (CFG['img_size'], CFG['img_size']),\n 'max_soft': True, \n 'reformulate': False\n },\n do_cutmix=False,\n cutmix_params={\n 'alpha': 1,\n }\n ):\n \n super().__init__()\n self.df = df.reset_index(drop=True).copy()\n self.transforms = transforms\n self.data_root = data_root\n self.do_fmix = do_fmix\n self.fmix_params = fmix_params\n self.do_cutmix = do_cutmix\n self.cutmix_params = cutmix_params\n \n self.output_label = output_label\n self.one_hot_label = one_hot_label\n \n if output_label == True:\n self.labels = self.df['label'].values\n #print(self.labels)\n \n if one_hot_label is True:\n self.labels = np.eye(self.df['label'].max()+1)[self.labels]\n #print(self.labels)\n \n def __len__(self):\n return self.df.shape[0]\n \n def __getitem__(self, index: int):\n \n # get labels\n if self.output_label:\n target = self.labels[index]\n \n img = get_img(\"{}/{}\".format(self.data_root, self.df.loc[index]['image_id']))\n\n if self.transforms:\n img = self.transforms(image=img)['image']\n \n if self.do_fmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n with torch.no_grad():\n #lam, mask = sample_mask(**self.fmix_params)\n \n lam = np.clip(np.random.beta(self.fmix_params['alpha'], self.fmix_params['alpha']),0.6,0.7)\n \n # Make mask, get mean / std\n mask = make_low_freq_image(self.fmix_params['decay_power'], self.fmix_params['shape'])\n mask = binarise_mask(mask, lam, self.fmix_params['shape'], self.fmix_params['max_soft'])\n \n fmix_ix = np.random.choice(self.df.index, size=1)[0]\n fmix_img = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[fmix_ix]['image_id']))\n\n if self.transforms:\n fmix_img = self.transforms(image=fmix_img)['image']\n\n mask_torch = torch.from_numpy(mask)\n \n # mix image\n img = mask_torch*img+(1.-mask_torch)*fmix_img\n\n #print(mask.shape)\n\n #assert self.output_label==True and self.one_hot_label==True\n\n # mix target\n rate = mask.sum()/CFG['img_size']/CFG['img_size']\n target = rate*target + (1.-rate)*self.labels[fmix_ix]\n #print(target, mask, img)\n #assert False\n \n if self.do_cutmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n #print(img.sum(), img.shape)\n with torch.no_grad():\n cmix_ix = np.random.choice(self.df.index, size=1)[0]\n cmix_img = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[cmix_ix]['image_id']))\n if self.transforms:\n cmix_img = self.transforms(image=cmix_img)['image']\n \n lam = np.clip(np.random.beta(self.cutmix_params['alpha'], self.cutmix_params['alpha']),0.3,0.4)\n bbx1, bby1, bbx2, bby2 = rand_bbox((CFG['img_size'], CFG['img_size']), lam)\n\n img[:, bbx1:bbx2, bby1:bby2] = cmix_img[:, bbx1:bbx2, bby1:bby2]\n\n rate = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (CFG['img_size'] * CFG['img_size']))\n target = rate*target + (1.-rate)*self.labels[cmix_ix]\n \n #print('-', img.sum())\n #print(target)\n #assert False\n \n # do label smoothing\n #print(type(img), type(target))\n if self.output_label == True:\n return img, target\n else:\n return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"from albumentations import (\n HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n return Compose([\n RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n HorizontalFlip(p=0.5),\n VerticalFlip(p=0.5),\n ShiftScaleRotate(p=0.5),\n RandomBrightnessContrast(brightness_limit=(-0.5,0.5), contrast_limit=(-0.5, 0.5), p=0.5),\n Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n CoarseDropout(p=0.5),\n Cutout(p=0.5),\n ToTensorV2(p=1.0),\n ], p=1.)\n \n \ndef get_valid_transforms():\n return Compose([\n Resize(CFG['img_size'], CFG['img_size']),\n Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n ToTensorV2(p=1.0),\n ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n def __init__(self, model_arch, n_class, pretrained=False):\n super().__init__()\n self.model = timm.create_model(model_arch, pretrained=pretrained,num_classes=n_class)\n\n def forward(self, x):\n x = self.model(x)\n return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LOSS FUNCTION"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# ====================================================\n# Label Smoothing\n# ====================================================\nclass LabelSmoothingLoss(nn.Module): \n def __init__(self, classes=5, smoothing=0.1, dim=-1): \n super(LabelSmoothingLoss, self).__init__() \n self.confidence = 1.0 - smoothing \n self.smoothing = smoothing \n self.cls = classes \n self.dim = dim \n def forward(self, pred, target): \n pred = pred.log_softmax(dim=self.dim) \n with torch.no_grad():\n true_dist = torch.zeros_like(pred) \n true_dist.fill_(self.smoothing / (self.cls - 1)) \n true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"class FocalLoss(nn.Module):\n def __init__(self, alpha=1, gamma=2, reduce=True):\n super(FocalLoss, self).__init__()\n self.alpha = alpha\n self.gamma = gamma\n self.reduce = reduce\n\n def forward(self, inputs, targets):\n BCE_loss = nn.CrossEntropyLoss()(inputs, targets)\n\n pt = torch.exp(-BCE_loss)\n F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n\n if self.reduce:\n return torch.mean(F_loss)\n else:\n return F_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"class FocalCosineLoss(nn.Module):#few shot and inbalanced dataset\n def __init__(self, alpha=1, gamma=2, xent=0.1):\n super(FocalCosineLoss, self).__init__()\n self.alpha = alpha\n self.gamma = gamma\n self.xent = xent\n\n self.y = torch.Tensor([1]).cuda()\n\n def forward(self, input, target, reduction=\"mean\"):\n cosine_loss = F.cosine_embedding_loss(input, F.one_hot(target, num_classes=input.size(-1)), self.y, reduction=reduction)\n\n cent_loss = F.cross_entropy(F.normalize(input), target, reduce=False)\n pt = torch.exp(-cent_loss)\n focal_loss = self.alpha * (1-pt)**self.gamma * cent_loss\n\n if reduction == \"mean\":\n focal_loss = torch.mean(focal_loss)\n\n return cosine_loss + self.xent * focal_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"class SymmetricCrossEntropy(nn.Module):#noisy data\n\n def __init__(self, alpha=0.1, beta=1.0, num_classes=5):\n super(SymmetricCrossEntropy, self).__init__()\n self.alpha = alpha\n self.beta = beta\n self.num_classes = num_classes\n\n def forward(self, logits, targets, reduction='mean'):\n onehot_targets = torch.eye(self.num_classes)[targets].cuda()\n ce_loss = F.cross_entropy(logits, targets, reduction=reduction)\n rce_loss = (-onehot_targets*logits.softmax(1).clamp(1e-7, 1.0).log()).sum(1)\n if reduction == 'mean':\n rce_loss = rce_loss.mean()\n elif reduction == 'sum':\n rce_loss = rce_loss.sum()\n return self.alpha * ce_loss + self.beta * rce_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"#Bi-Tempered-Loss(noisy data)\ndef log_t(u, t):\n \"\"\"Compute log_t for `u'.\"\"\"\n if t==1.0:\n return u.log()\n else:\n return (u.pow(1.0 - t) - 1.0) / (1.0 - t)\n\ndef exp_t(u, t):\n \"\"\"Compute exp_t for `u'.\"\"\"\n if t==1:\n return u.exp()\n else:\n return (1.0 + (1.0-t)*u).relu().pow(1.0 / (1.0 - t))\n\ndef compute_normalization_fixed_point(activations, t, num_iters):\n\n \"\"\"Returns the normalization value for each example (t > 1.0).\n Args:\n activations: A multi-dimensional tensor with last dimension `num_classes`.\n t: Temperature 2 (> 1.0 for tail heaviness).\n num_iters: Number of iterations to run the method.\n Return: A tensor of same shape as activation with the last dimension being 1.\n \"\"\"\n mu, _ = torch.max(activations, -1, keepdim=True)\n normalized_activations_step_0 = activations - mu\n\n normalized_activations = normalized_activations_step_0\n\n for _ in range(num_iters):\n logt_partition = torch.sum(\n exp_t(normalized_activations, t), -1, keepdim=True)\n normalized_activations = normalized_activations_step_0 * \\\n logt_partition.pow(1.0-t)\n\n logt_partition = torch.sum(\n exp_t(normalized_activations, t), -1, keepdim=True)\n normalization_constants = - log_t(1.0 / logt_partition, t) + mu\n\n return normalization_constants\n\ndef compute_normalization_binary_search(activations, t, num_iters):\n\n \"\"\"Returns the normalization value for each example (t < 1.0).\n Args:\n activations: A multi-dimensional tensor with last dimension `num_classes`.\n t: Temperature 2 (< 1.0 for finite support).\n num_iters: Number of iterations to run the method.\n Return: A tensor of same rank as activation with the last dimension being 1.\n \"\"\"\n\n mu, _ = torch.max(activations, -1, keepdim=True)\n normalized_activations = activations - mu\n\n effective_dim = \\\n torch.sum(\n (normalized_activations > -1.0 / (1.0-t)).to(torch.int32),\n dim=-1, keepdim=True).to(activations.dtype)\n\n shape_partition = activations.shape[:-1] + (1,)\n lower = torch.zeros(shape_partition, dtype=activations.dtype, device=activations.device)\n upper = -log_t(1.0/effective_dim, t) * torch.ones_like(lower)\n\n for _ in range(num_iters):\n logt_partition = (upper + lower)/2.0\n sum_probs = torch.sum(\n exp_t(normalized_activations - logt_partition, t),\n dim=-1, keepdim=True)\n update = (sum_probs < 1.0).to(activations.dtype)\n lower = torch.reshape(\n lower * update + (1.0-update) * logt_partition,\n shape_partition)\n upper = torch.reshape(\n upper * (1.0 - update) + update * logt_partition,\n shape_partition)\n\n logt_partition = (upper + lower)/2.0\n return logt_partition + mu\n\nclass ComputeNormalization(torch.autograd.Function):\n \"\"\"\n Class implementing custom backward pass for compute_normalization. See compute_normalization.\n \"\"\"\n @staticmethod\n def forward(ctx, activations, t, num_iters):\n if t < 1.0:\n normalization_constants = compute_normalization_binary_search(activations, t, num_iters)\n else:\n normalization_constants = compute_normalization_fixed_point(activations, t, num_iters)\n\n ctx.save_for_backward(activations, normalization_constants)\n ctx.t=t\n return normalization_constants\n\n @staticmethod\n def backward(ctx, grad_output):\n activations, normalization_constants = ctx.saved_tensors\n t = ctx.t\n normalized_activations = activations - normalization_constants \n probabilities = exp_t(normalized_activations, t)\n escorts = probabilities.pow(t)\n escorts = escorts / escorts.sum(dim=-1, keepdim=True)\n grad_input = escorts * grad_output\n \n return grad_input, None, None\n\ndef compute_normalization(activations, t, num_iters=5):\n \"\"\"Returns the normalization value for each example. \n Backward pass is implemented.\n Args:\n activations: A multi-dimensional tensor with last dimension `num_classes`.\n t: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n num_iters: Number of iterations to run the method.\n Return: A tensor of same rank as activation with the last dimension being 1.\n \"\"\"\n return ComputeNormalization.apply(activations, t, num_iters)\n\ndef tempered_sigmoid(activations, t, num_iters = 5):\n \"\"\"Tempered sigmoid function.\n Args:\n activations: Activations for the positive class for binary classification.\n t: Temperature tensor > 0.0.\n num_iters: Number of iterations to run the method.\n Returns:\n A probabilities tensor.\n \"\"\"\n internal_activations = torch.stack([activations,\n torch.zeros_like(activations)],\n dim=-1)\n internal_probabilities = tempered_softmax(internal_activations, t, num_iters)\n return internal_probabilities[..., 0]\n\n\ndef tempered_softmax(activations, t, num_iters=5):\n \"\"\"Tempered softmax function.\n Args:\n activations: A multi-dimensional tensor with last dimension `num_classes`.\n t: Temperature > 1.0.\n num_iters: Number of iterations to run the method.\n Returns:\n A probabilities tensor.\n \"\"\"\n if t == 1.0:\n return activations.softmax(dim=-1)\n\n normalization_constants = compute_normalization(activations, t, num_iters)\n return exp_t(activations - normalization_constants, t)\n\ndef bi_tempered_binary_logistic_loss(activations,\n labels,\n t1,\n t2,\n label_smoothing = 0.0,\n num_iters=5,\n reduction='mean'):\n\n \"\"\"Bi-Tempered binary logistic loss.\n Args:\n activations: A tensor containing activations for class 1.\n labels: A tensor with shape as activations, containing probabilities for class 1\n t1: Temperature 1 (< 1.0 for boundedness).\n t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n label_smoothing: Label smoothing\n num_iters: Number of iterations to run the method.\n Returns:\n A loss tensor.\n \"\"\"\n internal_activations = torch.stack([activations,\n torch.zeros_like(activations)],\n dim=-1)\n internal_labels = torch.stack([labels.to(activations.dtype),\n 1.0 - labels.to(activations.dtype)],\n dim=-1)\n return bi_tempered_logistic_loss(internal_activations, \n internal_labels,\n t1,\n t2,\n label_smoothing = label_smoothing,\n num_iters = num_iters,\n reduction = reduction)\n\ndef bi_tempered_logistic_loss(activations,\n labels,\n t1,\n t2,\n label_smoothing=0.0,\n num_iters=5,\n reduction = 'mean'):\n\n \"\"\"Bi-Tempered Logistic Loss.\n Args:\n activations: A multi-dimensional tensor with last dimension `num_classes`.\n labels: A tensor with shape and dtype as activations (onehot), \n or a long tensor of one dimension less than activations (pytorch standard)\n t1: Temperature 1 (< 1.0 for boundedness).\n t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n label_smoothing: Label smoothing parameter between [0, 1). Default 0.0.\n num_iters: Number of iterations to run the method. Default 5.\n reduction: ``'none'`` | ``'mean'`` | ``'sum'``. Default ``'mean'``.\n ``'none'``: No reduction is applied, return shape is shape of\n activations without the last dimension.\n ``'mean'``: Loss is averaged over minibatch. Return shape (1,)\n ``'sum'``: Loss is summed over minibatch. Return shape (1,)\n Returns:\n A loss tensor.\n \"\"\"\n\n if len(labels.shape) 0:\n num_classes = labels_onehot.shape[-1]\n labels_onehot = ( 1 - label_smoothing * num_classes / (num_classes - 1) ) \\\n * labels_onehot + \\\n label_smoothing / (num_classes - 1)\n\n probabilities = tempered_softmax(activations, t2, num_iters)\n\n loss_values = labels_onehot * log_t(labels_onehot + 1e-10, t1) \\\n - labels_onehot * log_t(probabilities, t1) \\\n - labels_onehot.pow(2.0 - t1) / (2.0 - t1) \\\n + probabilities.pow(2.0 - t1) / (2.0 - t1)\n loss_values = loss_values.sum(dim = -1) #sum over classes\n\n if reduction == 'none':\n return loss_values\n if reduction == 'sum':\n return loss_values.sum()\n if reduction == 'mean':\n return loss_values.mean()\n \nclass BiTemperedLogisticLoss(nn.Module): \n def __init__(self, t1=0.3, t2=1.0, smoothing=0.0): \n super(BiTemperedLogisticLoss, self).__init__() \n self.t1 = t1\n self.t2 = t2\n self.smoothing = smoothing\n def forward(self, logit_label, truth_label):\n loss_label = bi_tempered_logistic_loss(\n logit_label, truth_label,\n t1=self.t1, t2=self.t2,\n label_smoothing=self.smoothing,\n reduction='none'\n )\n \n loss_label = loss_label.mean()\n return loss_label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"class TaylorSoftmax(nn.Module):\n '''\n This is the autograd version\n '''\n def __init__(self, dim=1, n=2):\n super(TaylorSoftmax, self).__init__()\n assert n % 2 == 0\n self.dim = dim\n self.n = n\n\n def forward(self, x):\n '''\n usage similar to nn.Softmax:\n >>> mod = TaylorSoftmax(dim=1, n=4)\n >>> inten = torch.randn(1, 32, 64, 64)\n >>> out = mod(inten)\n '''\n fn = torch.ones_like(x)\n denor = 1.\n for i in range(1, self.n+1):\n denor *= i\n fn = fn + x.pow(i) / denor\n out = fn / fn.sum(dim=self.dim, keepdims=True)\n return out\n\n \nclass TaylorCrossEntropyLoss(nn.Module):\n def __init__(self, n=2, ignore_index=-1, reduction='mean', smoothing=0.05):\n super(TaylorCrossEntropyLoss, self).__init__()\n assert n % 2 == 0\n self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n self.reduction = reduction\n self.ignore_index = ignore_index\n self.lab_smooth = LabelSmoothingLoss(5, smoothing=smoothing)\n\n def forward(self, logits, labels):\n log_probs = self.taylor_softmax(logits).log()\n #loss = F.nll_loss(log_probs, labels, reduction=self.reduction,\n # ignore_index=self.ignore_index)\n loss = self.lab_smooth(log_probs, labels)\n return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def define_loss_function():\n if CFG['loss_function']=='CrossEntropyLoss':\n return nn.CrossEntropyLoss()\n if CFG['loss_function']=='LabelSmoothingLoss':\n return LabelSmoothingLoss(smoothing=CFG['LabelSmoothingLoss_smoothing'])\n if CFG['loss_function']=='FocalLoss':\n return FocalLoss(alpha=CFG['FocalLoss_alpha'],gamma=CFG['FocalLoss_gamma'])\n if CFG['loss_function']=='FocalCosineLoss':\n return FocalCosineLoss(alpha=CFG['FocalCosineLoss_alpha'],gamma=CFG['FocalCosineLoss_gamma'],xent=CFG['FocalCosineLoss_xent'])\n if CFG['loss_function']=='SymmetricCrossEntropy':\n return SymmetricCrossEntropy(alpha=CFG['SymmetricCrossEntropy_alpha'],beta=CFG['SymmetricCrossEntropy_beta'])\n if CFG['loss_function']=='BiTemperedLogisticLoss':\n return BiTemperedLogisticLoss(t1=CFG['BiTemperedLogisticLoss_t1'],t2=CFG['BiTemperedLogisticLoss_t2'],smoothing=CFG['BiTemperedLogisticLoss_smoothing'])\n if CFG['loss_function']=='TaylorCrossEntropyLoss':\n return TaylorCrossEntropyLoss(n=CFG['TaylorCrossEntropyLoss_n'],smoothing=CFG['TaylorCrossEntropyLoss_smoothing'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def prepare_dataloader(df, trn_idx, val_idx,device,data_root=CFG['train_imgs_path']):\n \n train_ = df.loc[trn_idx,:].reset_index(drop=True)\n valid_ = df.loc[val_idx,:].reset_index(drop=True)\n \n train_ds = CassavaDataset(train_, data_root, transforms=get_train_transforms(), output_label=True, one_hot_label=False, do_fmix=False, do_cutmix=False)\n valid_ds = CassavaDataset(valid_, data_root, transforms=get_valid_transforms(), output_label=True)\n \n if CFG['device'] == 'cuda:0':\n train_loader = torch.utils.data.DataLoader(\n train_ds,\n batch_size=CFG['train_bs'],\n pin_memory=False,\n drop_last=False,\n shuffle=True, \n num_workers=CFG['num_workers'],\n )\n valid_loader = torch.utils.data.DataLoader(\n valid_ds, \n batch_size=CFG['valid_bs'],\n num_workers=CFG['num_workers'],\n shuffle=False,\n pin_memory=False,\n )\n \n else: \n train_sampler = torch.utils.data.distributed.DistributedSampler(\n train_ds,\n num_replicas=xm.xrt_world_size(), #divide dataset among this many replicas\n rank=xm.get_ordinal(), #which replica/device/core\n shuffle=True)\n\n # define DataLoader with the defined sampler\n train_loader = torch.utils.data.DataLoader(\n train_ds,\n batch_size=CFG['train_bs'],\n sampler=train_sampler,\n num_workers=CFG['num_workers'],\n drop_last=True)\n\n # same as train but with valid data\n valid_sampler = torch.utils.data.distributed.DistributedSampler(\n valid_ds,\n num_replicas=xm.xrt_world_size(),\n rank=xm.get_ordinal(),\n shuffle=False)\n\n valid_loader = torch.utils.data.DataLoader(\n valid_ds,\n batch_size=CFG['valid_bs'],\n sampler=valid_sampler,\n num_workers=CFG['num_workers'],\n drop_last=False)\n\n train_loader = pl.MpDeviceLoader(train_loader, device) # puts the train data onto the current TPU core\n valid_loader = pl.MpDeviceLoader(valid_loader, device) # puts the valid data onto the current TPU core\n \n return train_loader, valid_loader\n\ndef train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None, schd_batch_update=False):\n \n model.train()\n t = time.time()\n running_loss = None\n \n if CFG['device']=='cuda:0' :\n pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n else:\n pbar = enumerate(train_loader)\n xm.master_print('start_training')\n\n \n for step, (imgs, image_labels) in pbar:\n imgs = imgs.to(device).float()\n image_labels = image_labels.to(device).long()\n \n image_preds = model(imgs) #output = model(input)\n loss = loss_fn(image_preds, image_labels)\n loss.backward()\n\n if ((step + 1) % CFG['accum_iter'] == 0) or ((step + 1) == len(train_loader)):\n # may unscale_ here if desired (e.g., to allow clipping unscaled gradients)\n\n if CFG['device']=='cuda:0':\n optimizer.step()\n else:\n xm.optimizer_step(optimizer) \n optimizer.zero_grad() \n\n if CFG['device']=='cuda:0':\n if running_loss is None:\n running_loss = loss.item()\n else:\n running_loss = running_loss * .99 + loss.item() * .01\n\n if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(train_loader)):\n description = f'epoch {epoch} loss: {running_loss:.4f}'\n pbar.set_description(description)\n else:\n loss_reduced = xm.mesh_reduce('loss_reduce',loss, lambda x: sum(x) / len(x)) \n xm.master_print(f'training step : {step} loss_reduced : {loss_reduced}')\n \n if scheduler is not None and not schd_batch_update:\n scheduler.step()\n \ndef valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):\n \n model.eval()\n\n t = time.time()\n loss_sum = 0\n sample_num = 0\n image_preds_all = []\n image_targets_all = []\n \n if CFG['device']=='cuda:0' :\n pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n else:\n pbar = enumerate(val_loader)\n xm.master_print('start validation')\n \n for step, (imgs, image_labels) in pbar:\n imgs = imgs.to(device).float()\n image_labels = image_labels.to(device).long()\n \n image_preds = model(imgs) #output = model(input)\n #print(image_preds.shape, exam_pred.shape)\n image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n image_targets_all += [image_labels.detach().cpu().numpy()]\n \n loss = loss_fn(image_preds, image_labels)\n sample_num += image_labels.shape[0] \n \n if CFG['device']=='cuda:0':\n loss_sum += loss.item()*image_labels.shape[0]\n if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(val_loader)):\n description = f'epoch {epoch} loss: {loss_sum/sample_num:.4f}'\n pbar.set_description(description)\n elif ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(train_loader)):\n xm.master_print(f'validation step : {step}')\n \n image_preds_all = np.concatenate(image_preds_all)\n image_targets_all = np.concatenate(image_targets_all)\n \n if CFG['device']=='cuda:0' or xm.is_master_ordinal():\n print('validation multi-class accuracy = {:.4f}'.format((image_preds_all==image_targets_all).mean()))\n \n if scheduler is not None:\n if schd_loss_update:\n scheduler.step(loss_sum/sample_num)\n else:\n scheduler.step()\n return (image_preds_all==image_targets_all).mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# OPTIMIZER"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def define_optimizer(model):\n if CFG['optimizer']=='Adam':\n return torch.optim.Adam(model.parameters(), lr=CFG['Adam_lr'], weight_decay=CFG['Adam_weight_decay'])\n if CFG['optimizer']=='SGD':\n return torch.optim.SGD(model.parameters(), lr=CFG['SGD_lr'], momentum=CFG['SGD_momentum'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LR_SCHEDULAR"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def define_lr_schedular(optimizer):\n if CFG['lr_schedular']=='StepLR':\n return torch.optim.lr_scheduler.StepLR(optimizer, step_size=CFG['StepLR_step_size'], gamma=CFG['StepLR_gamma'])\n if CFG['lr_schedular']=='ExponentialLR':\n return torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=CFG['ExponentialLR_gamma'])\n if CFG['lr_schedular']=='CosineAnnealingLR':\n return torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=CFG['CosineAnnealingLR_T_max'], \n eta_min=CFG['CosineAnnealingLR_eta_min'])\n if CFG['lr_schedular']=='ReduceLROnPlateau':\n return torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=CFG['ReduceLROnPlateau_factor'],\n patience=CFG['ReduceLROnPlateau_patience'], \n threshold=CFG['ReduceLROnPlateau_threshold'], threshold_mode='rel', \n cooldown=0, min_lr=CFG['ReduceLROnPlateau_min_lr'], eps=1e-08)\n if CFG['lr_schedular']=='CosineAnnealingWarmRestarts':\n return torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG['CosineAnnealingWarmRestarts_T_0'], \n eta_min=CFG['CosineAnnealingWarmRestarts'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# print hyperparameter"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def print_hyperparameter():\n print(f'overall epochs:{CFG[\"epochs\"]}')\n print(f'archetecture:{CFG[\"model_arch\"]}')\n print(f'train images path:{CFG[\"train_imgs_path\"]}')\n print(f'train csv path:{CFG[\"train_csv_path\"]}')\n \n print(f'loss function:{CFG[\"loss_function\"]}')\n if CFG[\"loss_function\"]=='LabelSmoothingLoss':\n print(f'LabelSmoothingLoss_smoothing:{CFG[\"LabelSmoothingLoss_smoothing\"]}')\n elif CFG[\"loss_function\"]=='FocalLoss':\n print(f'FocalLoss_alpha:{CFG[\"FocalLoss_alpha\"]} FocalLoss_gamma:{CFG[\"FocalLoss_gamma\"]}')\n elif CFG[\"loss_function\"]=='FocalCosineLoss':\n print(f'FocalCosineLoss_alpha:{CFG[\"FocalCosineLoss_alpha\"]} \\\n FocalCosineLoss_gamma:{CFG[\"FocalCosineLoss_gamma\"]}\\\n FocalCosineLoss_xent:{CFG[\"FocalCosineLoss_xent\"]}')\n elif CFG[\"loss_function\"]=='SymmetricCrossEntropy':\n print(f'SymmetricCrossEntropy_alpha:{CFG[\"SymmetricCrossEntropy_alpha\"]} \\\n SymmetricCrossEntropy_beta:{CFG[\"SymmetricCrossEntropy_beta\"]}')\n elif CFG[\"loss_function\"]=='BiTemperedLogisticLoss':\n print(f'BiTemperedLogisticLoss_t1:{CFG[\"BiTemperedLogisticLoss_t1\"]} \\\n BiTemperedLogisticLoss_t2:{CFG[\"BiTemperedLogisticLoss_t2\"]}\\\n BiTemperedLogisticLoss_smoothing:{CFG[\"BiTemperedLogisticLoss_smoothing\"]}')\n elif CFG[\"loss_function\"]=='TaylorCrossEntropyLoss':\n print(f'TaylorCrossEntropyLoss_smoothing:{CFG[\"TaylorCrossEntropyLoss_smoothing\"]} \\\n TaylorCrossEntropyLoss_n {CFG[\"TaylorCrossEntropyLoss_n\"]}')\n \n print(f'optimizer:{CFG[\"optimizer\"]}')\n if CFG[\"optimizer\"]=='Adam':\n print(f'Adam_lr:{CFG[\"Adam_lr\"]} Adam_weight_decay:{CFG[\"Adam_weight_decay\"]}')\n elif CFG[\"optimizer\"]=='SGD':\n print(f'SGD_lr:{CFG[\"SGD_lr\"]} SGD_momentum:{CFG[\"SGD_momentum\"]}')\n \n print(f'lr_schedular:{CFG[\"lr_schedular\"]}')\n if CFG[\"lr_schedular\"]=='StepLR':\n print(f'StepLR_step_size:{CFG[\"StepLR_step_size\"]} StepLR_gamma:{CFG[\"StepLR_gamma\"]}')\n elif CFG[\"lr_schedular\"]=='ExponentialLR':\n print(f'ExponentialLR_gamma:{CFG[\"ExponentialLR_gamma\"]}')\n elif CFG[\"lr_schedular\"]=='CosineAnnealingLR':\n print(f'CosineAnnealingLR_T_max:{CFG[\"CosineAnnealingLR_T_max\"]}\\\n CosineAnnealingLR_eta_min:{CFG[\"CosineAnnealingLR_eta_min\"]}')\n elif CFG[\"lr_schedular\"]=='ReduceLROnPlateau':\n print(f'ReduceLROnPlateau_factor:{CFG[\"ReduceLROnPlateau_factor\"]}\\\n ReduceLROnPlateau_patience:{CFG[\"ReduceLROnPlateau_patience\"]}\\\n ReduceLROnPlateau_threshold:{CFG[\"ReduceLROnPlateau_threshold\"]}\\\n ReduceLROnPlateau_min_lr:{CFG[\"ReduceLROnPlateau_min_lr\"]}')\n elif CFG[\"lr_schedular\"]=='CosineAnnealingWarmRestarts':\n print(f'CosineAnnealingWarmRestarts_T_0:{CFG[\"CosineAnnealingWarmRestarts_T_0\"]}\\\n CosineAnnealingWarmRestarts_min_lr:{CFG[\"CosineAnnealingWarmRestarts_min_lr\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MAIN"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _mp_fn(rank):\n \n global MX\n \n if CFG['device']=='cuda:0' or (CFG['device']!='cuda:0' and xm.is_master_ordinal()):\n print_hyperparameter()\n \n torch.set_default_tensor_type('torch.FloatTensor')\n seed_everything(CFG['seed'])\n device=torch.device('cuda:0') if CFG['device']=='cuda:0' else xm.xla_device()\n \n train = pd.read_csv(CFG['train_csv_path'])\n folds = StratifiedKFold(n_splits=CFG['fold_num'], shuffle=True, random_state=CFG['seed']).split(np.arange(train.shape[0]), train.label.values)\n \n fold_accuracy=[]\n fold_epoch=[]\n for fold, (trn_idx, val_idx) in enumerate(folds):\n # we'll train fold 0 first\n if fold > 0 and CFG['only_one_fold']==True:\n break \n if CFG['device'] =='cuda:0' or (CFG['device']!='cuda:0' and xm.is_master_ordinal()):\n print('Training with {} started ,{} train ,{} test'.format(fold,len(trn_idx),len(val_idx)))\n\n train_loader, val_loader = prepare_dataloader(train, trn_idx, val_idx, device=device,data_root=CFG['train_imgs_path'])\n \n if CFG['device']=='cuda:0':\n model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique(), pretrained=True).to(device)\n else:\n model = MX.to(device)\n optimizer = define_optimizer(model)\n scheduler = define_lr_schedular(optimizer)\n \n loss_tr = define_loss_function().to(device)\n loss_fn = define_loss_function().to(device)\n \n epoch_highest_acc=0 #define highest accuracy recorder\n epoch_highest_record=0 #record highest accuracy occur in which epoch\n \n for epoch in range(CFG['epochs']):\n train_one_epoch(epoch, model, loss_tr, optimizer, train_loader, device, scheduler=scheduler, schd_batch_update=False)\n\n with torch.no_grad():\n accuracy = valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False)\n if accuracy > epoch_highest_acc:\n epoch_highest_acc = accuracy\n epoch_highest_record = epoch\n \n if CFG['device']=='cuda:0':\n torch.save(model.state_dict(),'{}_fold_{}_{}_{:.4f}'.format(CFG['model_arch'], fold, epoch,accuracy))\n else:\n xm.rendezvous('save_model')\n xm.master_print('save model')\n xm.save(model.state_dict(),'{}_fold_{}_{}_{:.4f}'.format(CFG['model_arch'], fold, epoch,accuracy))\n \n fold_accuracy.append(epoch_highest_acc)\n fold_epoch.append(epoch_highest_record)\n #create best link\n if CFG['device'] =='cuda:0' or (CFG['device']!='cuda:0' and xm.is_master_ordinal()):\n print(f'fold {fold} finish,highest accuracy:{epoch_highest_acc},highest epoch:{epoch_highest_record}')\n\n del model, optimizer, train_loader, val_loader, scaler, scheduler\n torch.cuda.empty_cache()\n if CFG['device'] =='cuda:0' or (CFG['device']!='cuda:0' and xm.is_master_ordinal()):\n print_hyperparameter()\n print('----------ALL FOLDS FINISHED----------')\n if CFG['only_one_fold']:\n print(f'best epochs fold0:{fold_epoch[0]}')\n else:\n print(f'best epochs fold0:{fold_epoch[0]},fold1:{fold_epoch[1]},fold2:{fold_epoch[2]},fold3:{fold_epoch[3]},fold4:{fold_epoch[4]}')\n print('final accuracy = {:.4f}'.format(np.mean(fold_accuracy)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"if __name__ == '__main__':\n if CFG['device']=='cuda:0':\n run(0)\n else:\n MX=xmp.MpModelWrapper(CassvaImgClassifier(CFG['model_arch'], 5, pretrained=True))\n xmp.spawn(_mp_fn, args=(), nprocs=8, start_method='fork')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}