{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# 需要手动添加\n# ../input/cassava-leaf-disease-classification\n# ../input/image-fmix\n# ../input/pytorch-image-models\n# ../input/efficientnet-pytorch-07\n\npackage_paths = [\n    '../input/pytorch-image-models/pytorch-image-models-master', #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\n    '../input/image-fmix/FMix-master'\n]\nimport sys; \n\nfor pth in package_paths:\n    sys.path.append(pth)\n    \nfrom fmix import sample_mask, make_low_freq_image, binarise_mask\n\nfrom glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\n\nimport timm\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\n#from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom\nfrom torch.optim.optimizer import Optimizer, required\nimport math\n\nclass AdamP(Optimizer):\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,\n                 weight_decay=0, delta=0.1, wd_ratio=0.1, nesterov=False):\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay,\n                        delta=delta, wd_ratio=wd_ratio, nesterov=nesterov)\n        super(AdamP, self).__init__(params, defaults)\n\n    def _channel_view(self, x):\n        return x.view(x.size(0), -1)\n\n    def _layer_view(self, x):\n        return x.view(1, -1)\n\n    def _cosine_similarity(self, x, y, eps, view_func):\n        x = view_func(x)\n        y = view_func(y)\n\n        return F.cosine_similarity(x, y, dim=1, eps=eps).abs_()\n\n    def _projection(self, p, grad, perturb, delta, wd_ratio, eps):\n        wd = 1\n        expand_size = [-1] + [1] * (len(p.shape) - 1)\n        for view_func in [self._channel_view, self._layer_view]:\n\n            cosine_sim = self._cosine_similarity(grad, p.data, eps, view_func)\n\n            if cosine_sim.max() < delta / math.sqrt(view_func(p.data).size(1)):\n                p_n = p.data / view_func(p.data).norm(dim=1).view(expand_size).add_(eps)\n                perturb -= p_n * view_func(p_n * perturb).sum(dim=1).view(expand_size)\n                wd = wd_ratio\n\n                return perturb, wd\n\n        return perturb, wd\n\n    def step(self, closure=None):\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n\n                grad = p.grad.data\n                beta1, beta2 = group['betas']\n                nesterov = group['nesterov']\n\n                state = self.state[p]\n\n                # State initialization\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p.data)\n                    state['exp_avg_sq'] = torch.zeros_like(p.data)\n\n                # Adam\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n\n                state['step'] += 1\n                bias_correction1 = 1 - beta1 ** state['step']\n                bias_correction2 = 1 - beta2 ** state['step']\n\n                exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)\n                exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)\n\n                denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(group['eps'])\n                step_size = group['lr'] / bias_correction1\n\n                if nesterov:\n                    perturb = (beta1 * exp_avg + (1 - beta1) * grad) / denom\n                else:\n                    perturb = exp_avg / denom\n\n                # Projection\n                wd_ratio = 1\n                if len(p.shape) > 1:\n                    perturb, wd_ratio = self._projection(p, grad, perturb, group['delta'], group['wd_ratio'], group['eps'])\n\n                # Weight decay\n                if group['weight_decay'] > 0:\n                    p.data.mul_(1 - group['lr'] * group['weight_decay'] * wd_ratio)\n\n                # Step\n                p.data.add_(perturb, alpha=-step_size)\n\n        return loss\nCFG = {\n    'fold_num': 5,\n    'seed': 719,\n    'model_arch': 'ssl_resnext101_32x4d',\n    'img_size': 512,\n    'epochs': 20,\n    'train_bs': 16,\n    'valid_bs': 32,\n    'T_0': 10,\n    'lr': 1e-4,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_workers': 4,\n    'accum_iter': 2, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'num_classes':5\n}\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nimport numpy as np\nimport scipy.stats as stats2\nimport sys\nfrom numpy import float32\ntry:\n    from kornia.losses import FocalLoss as focal_loss\nexcept:\n    pass\n\ndef get_gauss_label(label, n_classes, amplifier, noise=0):\n    n = n_classes*amplifier\n    half_int = amplifier/2\n    label_noise = np.random.uniform(low=-noise, high=noise)\n    if label == 0:\n        label_noise = np.abs(label_noise)\n    if label == 4:\n        label_noise = -np.abs(label_noise)\n    label += label_noise\n    label_new = half_int + label*amplifier\n    gauss_label = stats2.norm.pdf(np.arange(n), label_new, half_int/2)\n    gauss_label/=np.sum(gauss_label)\n    return gauss_label\n\ndef get_gaussian_label_distribution(n_classes, std=0.5):\n    cls = []\n    for n in range(n_classes):\n        cls.append(stats2.norm.pdf(range(n_classes), 0, std))\n    dists = np.stack(cls, axis=0)\n    return dists\n    # if n_classes == 3:\n    #     CL_0 = stats2.norm.pdf([0, 1, 2], 0, std)\n    #     CL_1 = stats2.norm.pdf([0, 1, 2], 1, std)\n    #     CL_2 = stats2.norm.pdf([0, 1, 2], 2, std)\n    #     dists = np.stack([CL_0, CL_1, CL_2], axis=0)\n    #     return dists\n    # if n_classes == 5:\n    #     CL_0 = stats2.norm.pdf([0, 1, 2, 3, 4], 0, std)\n    #     CL_1 = stats2.norm.pdf([0, 1, 2, 3, 4], 1, std)\n    #     CL_2 = stats2.norm.pdf([0, 1, 2, 3, 4], 2, std)\n    #     CL_3 = stats2.norm.pdf([0, 1, 2, 3, 4], 3, std)\n    #     CL_4 = stats2.norm.pdf([0, 1, 2, 3, 4], 4, std)\n    #     dists = np.stack([CL_0, CL_1, CL_2, CL_3, CL_4], axis=0)\n    #     return dists\n    # if n_classes == 6:\n    #     CL_0 = stats2.norm.pdf([0, 1, 2, 3, 4, 5], 0, std)\n    #     CL_1 = stats2.norm.pdf([0, 1, 2, 3, 4, 5], 1, std)\n    #     CL_2 = stats2.norm.pdf([0, 1, 2, 3, 4, 5], 2, std)\n    #     CL_3 = stats2.norm.pdf([0, 1, 2, 3, 4, 5], 3, std)\n    #     CL_4 = stats2.norm.pdf([0, 1, 2, 3, 4, 5], 4, std)\n    #     CL_5 = stats2.norm.pdf([0, 1, 2, 3, 4, 5], 5, std)\n    #     dists = np.stack([CL_0, CL_1, CL_2, CL_3, CL_4, CL_5], axis=0)\n    #     return dists\n    # else:\n    #     raise NotImplementedError\n\ndef cross_entropy_loss_one_hot(logits, target, reduction='mean'):\n    logp = F.log_softmax(logits, dim=1)\n    loss = torch.sum(-logp * target, dim=1)\n    if reduction == 'none':\n        return loss\n    elif reduction == 'mean':\n        return loss.mean()\n    elif reduction == 'sum':\n        return loss.sum()\n    else:\n        raise ValueError(\n            '`reduction` must be one of \\'none\\', \\'mean\\', or \\'sum\\'.')\n\ndef one_hot_encoding(label, n_classes):\n    return torch.zeros(label.size(0), n_classes).to(label.device).scatter_(\n        1, label.view(-1, 1), 1)\n\ndef label_smoothing_criterion(alpha=0.1, distribution='uniform', std=0.5, reduction='mean'):\n    def _label_smoothing_criterion(logits, labels):\n        n_classes = logits.size(1)\n        device = logits.device\n        # manipulate labels\n        one_hot = one_hot_encoding(labels, n_classes).float().to(device)\n        if distribution == 'uniform':\n            uniform = torch.ones_like(one_hot).to(device)/n_classes\n            soft_labels = (1 - alpha)*one_hot + alpha*uniform\n        elif distribution == 'gaussian':\n            dist = get_gaussian_label_distribution(n_classes, std=std)\n            soft_labels = torch.from_numpy(dist[labels.cpu().numpy()]).to(device)\n        else:\n            raise NotImplementedError\n\n        loss = cross_entropy_loss_one_hot(logits, soft_labels.float(), reduction)\n\n        return loss\n\n    return _label_smoothing_criterion\n\ndef cost_sensitive_loss(input, target, M):\n    if input.size(0) != target.size(0):\n        raise ValueError('Expected input batch_size ({}) to match target batch_size ({}).'\n                         .format(input.size(0), target.size(0)))\n    device = input.device\n    M = M.to(device)\n    return (M[target, :]*input.float()).sum(axis=-1)\n    # return torch.diag(torch.matmul(input, M[:, target]))\n\nclass CostSensitiveLoss(nn.Module):\n    def __init__(self,  n_classes, exp=1, normalization='softmax', reduction='mean'):\n        super(CostSensitiveLoss, self).__init__()\n        if normalization == 'softmax':\n            self.normalization = nn.Softmax(dim=1)\n        elif normalization == 'sigmoid':\n            self.normalization = nn.Sigmoid()\n        else:\n            self.normalization = None\n        self.reduction = reduction\n        x = np.abs(np.arange(n_classes, dtype=np.float32))\n        #M = np.abs((x[:, np.newaxis] - x[np.newaxis, :])) ** exp\n        M = np. array([[0., 1., 1., 1., 1.],\n                       [1., 0., 1., 1., 1.],\n                       [1., 1., 0., 1., 1.],\n                       [1., 1., 1., 0., 1.],\n                       [1., 1., 1., 1., 0.]],dtype=float32)\n        M /= M.max()\n        self.M = torch.from_numpy(M)\n\n    def forward(self, logits, target):\n        preds = self.normalization(logits)\n        loss = cost_sensitive_loss(preds, target, self.M)\n        if self.reduction == 'none':\n            return loss\n        elif self.reduction == 'mean':\n            return loss.mean()\n        elif self.reduction == 'sum':\n            return loss.sum()\n        else:\n            raise ValueError('`reduction` must be one of \\'none\\', \\'mean\\', or \\'sum\\'.')\n\nclass CostSensitiveRegularizedLoss(nn.Module):\n    def __init__(self,  n_classes=5, exp=2, normalization='softmax', reduction='mean', base_loss='ls', lambd=10):\n        super(CostSensitiveRegularizedLoss, self).__init__()\n        if normalization == 'softmax':\n            self.normalization = nn.Softmax(dim=1)\n        elif normalization == 'sigmoid':\n            self.normalization = nn.Sigmoid()\n        else:\n            self.normalization = None\n        self.reduction = reduction\n        x = np.abs(np.arange(n_classes, dtype=np.float32))\n#         M = np.abs((x[:, np.newaxis] - x[np.newaxis, :])) ** exp\n        M = np. array([[0., 1., 1., 1., 1.],\n                       [1., 0., 1., 1., 1.],\n                       [1., 1., 0., 1., 1.],\n                       [1., 1., 1., 0., 1.],\n                       [1., 1., 1., 1., 0.]],dtype=float32)\n        # M_oph = M_oph.T\n        # # Normalize M_oph to obtain M_difficulty:\n        # M_difficulty = 1-np.divide(M_oph, np.sum(M_oph, axis=1)[:, None])\n        # # OPTION 1: average M and M_difficulty:\n        # M = 0.5 * M + 0.5 * M_difficulty\n        # ################\n        # # OPTION 2: replace uninformative entries in M_difficulty by entries of M:\n        # # M_difficulty[M_oph == 0] = M[M_oph == 0]\n        # # M = M_difficulty\n        M /= M.max()\n        self.M = torch.from_numpy(M)\n        self.lambd = lambd\n        self.base_loss = base_loss\n\n\n        if self.base_loss == 'ce':\n            self.base_loss = torch.nn.CrossEntropyLoss(reduction=reduction)\n        elif self.base_loss == 'ls':\n            self.base_loss = label_smoothing_criterion(distribution='uniform', reduction=reduction)\n        elif self.base_loss == 'gls':\n            self.base_loss = label_smoothing_criterion(distribution='gaussian', reduction=reduction)\n        elif self.base_loss == 'focal_loss':\n            kwargs = {\"alpha\": 0.5, \"gamma\": 2.0, \"reduction\": reduction}\n            self.base_loss = focal_loss(**kwargs)\n        else:\n            sys.exit('not a supported base_loss')\n\n    def forward(self, logits, target):\n        base_l = self.base_loss(logits, target)\n        if self.lambd == 0:\n            return self.base_loss(logits, target)\n        else:\n            preds = self.normalization(logits)\n            loss = cost_sensitive_loss(preds, target, self.M)\n            if self.reduction == 'none':\n                return base_l + self.lambd*loss\n            elif self.reduction == 'mean':\n                return base_l + self.lambd*loss.mean()\n            elif self.reduction == 'sum':\n                return base_l + self.lambd*loss.sum()\n            else:\n                raise ValueError('`reduction` must be one of \\'none\\', \\'mean\\', or \\'sum\\'.')\n\ndef get_cost_sensitive_criterion(n_classes=5, exp=2):\n    train_criterion = CostSensitiveLoss(n_classes, exp=exp, normalization='softmax')\n    val_criterion = CostSensitiveLoss(n_classes, exp=exp, normalization='softmax')\n    return train_criterion, val_criterion\n\ndef get_cost_sensitive_regularized_criterion(base_loss='ce', n_classes=5, lambd=1, exp=2):\n    train_criterion = CostSensitiveRegularizedLoss(n_classes, exp=exp, normalization='softmax', base_loss=base_loss, lambd=lambd)\n    val_criterion = CostSensitiveRegularizedLoss(n_classes, exp=exp, normalization='softmax', base_loss=base_loss, lambd=lambd)\n\n    return train_criterion, val_criterion\n\ntrain = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()\n\ntrain.label.value_counts()\n\nsubmission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nplt.imshow(img)\nplt.show()\n\n\ndef rand_bbox(size, lam):\n    W = size[0]\n    H = size[1]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\n\nclass CassavaDataset(Dataset):\n    def __init__(self, df, data_root, \n                 transforms=None, \n                 output_label=True, \n                 one_hot_label=False,\n                 do_fmix=False, \n                 fmix_params={\n                     'alpha': 1., \n                     'decay_power': 3., \n                     'shape': (CFG['img_size'], CFG['img_size']),\n                     'max_soft': True, \n                     'reformulate': False\n                 },\n                 do_cutmix=False,\n                 cutmix_params={\n                     'alpha': 1,\n                 }\n                ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.do_fmix = do_fmix\n        self.fmix_params = fmix_params\n        self.do_cutmix = do_cutmix\n        self.cutmix_params = cutmix_params\n        \n        self.output_label = output_label\n        self.one_hot_label = one_hot_label\n        \n        if output_label == True:\n            self.labels = self.df['label'].values\n            #print(self.labels)\n            \n            if one_hot_label is True:\n                self.labels = np.eye(self.df['label'].max()+1)[self.labels]\n                #print(self.labels)\n            \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.labels[index]\n          \n        img  = get_img(\"{}/{}\".format(self.data_root, self.df.loc[index]['image_id']))\n\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n        if self.do_fmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            with torch.no_grad():\n                #lam, mask = sample_mask(**self.fmix_params)\n                \n                lam = np.clip(np.random.beta(self.fmix_params['alpha'], self.fmix_params['alpha']),0.6,0.7)\n                \n                # Make mask, get mean / std\n                mask = make_low_freq_image(self.fmix_params['decay_power'], self.fmix_params['shape'])\n                mask = binarise_mask(mask, lam, self.fmix_params['shape'], self.fmix_params['max_soft'])\n    \n                fmix_ix = np.random.choice(self.df.index, size=1)[0]\n                fmix_img  = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[fmix_ix]['image_id']))\n\n                if self.transforms:\n                    fmix_img = self.transforms(image=fmix_img)['image']\n\n                mask_torch = torch.from_numpy(mask)\n                \n                # mix image\n                img = mask_torch*img+(1.-mask_torch)*fmix_img\n\n                #print(mask.shape)\n\n                #assert self.output_label==True and self.one_hot_label==True\n\n                # mix target\n                rate = mask.sum()/CFG['img_size']/CFG['img_size']\n                target = rate*target + (1.-rate)*self.labels[fmix_ix]\n                #print(target, mask, img)\n                #assert False\n        \n        if self.do_cutmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            #print(img.sum(), img.shape)\n            with torch.no_grad():\n                cmix_ix = np.random.choice(self.df.index, size=1)[0]\n                cmix_img  = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[cmix_ix]['image_id']))\n                if self.transforms:\n                    cmix_img = self.transforms(image=cmix_img)['image']\n                    \n                lam = np.clip(np.random.beta(self.cutmix_params['alpha'], self.cutmix_params['alpha']),0.3,0.4)\n                bbx1, bby1, bbx2, bby2 = rand_bbox((CFG['img_size'], CFG['img_size']), lam)\n\n                img[:, bbx1:bbx2, bby1:bby2] = cmix_img[:, bbx1:bbx2, bby1:bby2]\n\n                rate = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (CFG['img_size'] * CFG['img_size']))\n                target = rate*target + (1.-rate)*self.labels[cmix_ix]\n                \n            #print('-', img.sum())\n            #print(target)\n            #assert False\n                            \n        # do label smoothing\n        #print(type(img), type(target))\n        if self.output_label == True:\n            return img, target\n        else:\n            return img\n        \n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\n\n\n\nclass CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(2048, n_class)\n        self.model.classifier = nn.Linear(n_features, n_class)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \n    \nclass TaylorSoftmax(nn.Module):\n\n    def __init__(self, dim=1, n=2):\n        super(TaylorSoftmax, self).__init__()\n        assert n % 2 == 0\n        self.dim = dim\n        self.n = n\n\n    def forward(self, x):\n        \n        fn = torch.ones_like(x)\n        denor = 1.\n        for i in range(1, self.n+1):\n            denor *= i\n            fn = fn + x.pow(i) / denor\n        out = fn / fn.sum(dim=self.dim, keepdims=True)\n        return out\n\nclass LabelSmoothingLoss(nn.Module):\n\n    def __init__(self, classes, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        \"\"\"Taylor Softmax and log are already applied on the logits\"\"\"\n        #pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad(): \n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))\n    \n\nclass TaylorCrossEntropyLoss(nn.Module):\n\n    def __init__(self, n=2, ignore_index=-1, reduction='mean', smoothing=0.2):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n        self.lab_smooth = LabelSmoothingLoss(CFG['num_classes'], smoothing=smoothing)\n\n    def forward(self, logits, labels):\n\n        log_probs = self.taylor_softmax(logits).log()\n        #loss = F.nll_loss(log_probs, labels, reduction=self.reduction,\n        #        ignore_index=self.ignore_index)\n        loss = self.lab_smooth(log_probs, labels)\n        return loss\n    \ndef prepare_dataloader(df, trn_idx, val_idx, data_root='../input/cassava-leaf-disease-classification/train_images/'):\n    \n    from catalyst.data.sampler import BalanceClassSampler\n    \n    train_ = df.loc[trn_idx,:].reset_index(drop=True)\n    valid_ = df.loc[val_idx,:].reset_index(drop=True)\n        \n    train_ds = CassavaDataset(train_, data_root, transforms=get_train_transforms(), output_label=True, one_hot_label=False, do_fmix=False, do_cutmix=False)\n    valid_ds = CassavaDataset(valid_, data_root, transforms=get_valid_transforms(), output_label=True)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n        #sampler=BalanceClassSampler(labels=train_['label'].values, mode=\"downsampling\")\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n    return train_loader, val_loader\n\ndef train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None, schd_batch_update=False):\n    model.train()\n\n    t = time.time()\n    running_loss = None\n\n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n\n        #print(image_labels.shape, exam_label.shape)\n        with autocast():\n            image_preds = model(imgs)   #output = model(input)\n            #print(image_preds.shape, exam_pred.shape)\n\n            loss = loss_fn(image_preds, image_labels)\n            \n            scaler.scale(loss).backward()\n\n            if running_loss is None:\n                running_loss = loss.item()\n            else:\n                running_loss = running_loss * .99 + loss.item() * .01\n\n            if ((step + 1) %  CFG['accum_iter'] == 0) or ((step + 1) == len(train_loader)):\n                # may unscale_ here if desired (e.g., to allow clipping unscaled gradients)\n\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad() \n                \n                if scheduler is not None and schd_batch_update:\n                    scheduler.step()\n\n            if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(train_loader)):\n                description = f'epoch {epoch} loss: {running_loss:.4f}'\n                \n                pbar.set_description(description)\n                \n    if scheduler is not None and not schd_batch_update:\n        scheduler.step()\n        \ndef valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    sample_num = 0\n    image_preds_all = []\n    image_targets_all = []\n    \n    pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n        \n        image_preds = model(imgs)   #output = model(input)\n        #print(image_preds.shape, exam_pred.shape)\n        image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n        image_targets_all += [image_labels.detach().cpu().numpy()]\n        \n        loss = loss_fn(image_preds, image_labels)\n        \n        loss_sum += loss.item()*image_labels.shape[0]\n        sample_num += image_labels.shape[0]  \n\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(val_loader)):\n            description = f'epoch {epoch} loss: {loss_sum/sample_num:.4f}'\n            pbar.set_description(description)\n    \n    image_preds_all = np.concatenate(image_preds_all)\n    image_targets_all = np.concatenate(image_targets_all)\n    print('validation multi-class accuracy = {:.4f}'.format((image_preds_all==image_targets_all).mean()))\n    \n    if scheduler is not None:\n        if schd_loss_update:\n            scheduler.step(loss_sum/sample_num)\n        else:\n            scheduler.step()\n\n            \n    \nif __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num'], shuffle=True, random_state=CFG['seed']).split(np.arange(train.shape[0]), train.label.values)\n    device = torch.device(CFG['device'])\n        \n    model =torch.load('../input/zyhresnet222/ssl_resnext101_32x4d_fold_0_19.pt', map_location=torch.device(CFG['device']))\n    \n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        # we'll train fold 0 first\n        if fold > 0:\n            break \n\n        print('Training with {} started'.format(fold))\n\n        print(len(trn_idx), len(val_idx))\n        train_loader, val_loader = prepare_dataloader(train, trn_idx, val_idx, data_root='../input/cassava-leaf-disease-classification/train_images/')\n\n        \n        \n        scaler = GradScaler()   \n        optimizer = AdamP(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'])\n        #scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=CFG['epochs']-1)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)\n        #scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=25, \n        #                                                max_lr=CFG['lr'], epochs=CFG['epochs'], steps_per_epoch=len(train_loader))\n        \n        loss_tr = CostSensitiveRegularizedLoss().to(device) #MyCrossEntropyLoss().to(device)\n        loss_fn = CostSensitiveRegularizedLoss().to(device)\n        \n        for epoch in range(CFG['epochs']):\n            train_one_epoch(epoch, model, loss_tr, optimizer, train_loader, device, scheduler=scheduler, schd_batch_update=False)\n\n            with torch.no_grad():\n                valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False)\n\n            torch.save(model, f\"{CFG['model_arch']}_fold_{fold}_{epoch}.pt\")\n            \n        #torch.save(model.cnn_model.state_dict(),'{}/cnn_model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag']))\n        del model, optimizer, train_loader, val_loader, scaler, scheduler\n        torch.cuda.empty_cache()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}