{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/pytimm/pytorch-image-models-master/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.utils.data as D\nimport cv2\nimport math\nfrom scipy.stats import beta","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import random\nimport os\nimport numpy as np\nimport torch\nfrom collections import defaultdict\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\nimport logging\nfrom tqdm import tqdm\nimport numpy as np\nimport timm\nimport torch.nn as nn\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\ndef get_cv(df, n_fold = 5):\n    tmp = []\n    fold_rate = 1 / n_fold\n    for i in range(n_fold - 1):\n        df, test_df = train_test_split(df, test_size = fold_rate / (1 - fold_rate * i), random_state = 1001)\n        tmp.append(test_df)\n    tmp.append(df)\n    res = defaultdict(list)\n    for i in range(n_fold):\n        res[i].append(tmp[i])\n        tmp2 = pd.DataFrame()\n        for j in range(n_fold):\n            if i != j: tmp2 = tmp2.append(tmp[j])\n        res[i].append(tmp2)\n    return res\n\nclass label_smoothing2(nn.Module):\n    def __init__(self, classes = 5, smoothing=0.0, dim=-1):\n        super(label_smoothing2, self).__init__()\n        self.confidence = 1.0 - smoothing\n        self.smoothing = smoothing\n        self.cls = classes\n        self.dim = dim\n    def forward(self, pred, target):\n        pred = pred.log_softmax(dim=self.dim)\n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred)\n            true_dist.fill_(self.smoothing / (self.cls - 1))\n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)\n        return torch.mean(torch.sum(-true_dist * pred, dim = self.dim))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout,\n    ShiftScaleRotate, CenterCrop, Resize\n)\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms(img_size):\n    return Compose([\n        RandomResizedCrop(img_size, img_size),\n        Transpose(p=0.5),\n        HorizontalFlip(p=0.5),\n        VerticalFlip(p=0.5),\n        ShiftScaleRotate(p=0.5),\n        HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n        RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        CoarseDropout(p=0.5),\n        Cutout(p=0.5),\n        ToTensorV2(p=1.0),\n    ], p=1.)\n\ndef get_valid_transforms(img_size):\n    return Compose([\n        CenterCrop(img_size, img_size, p=1.),\n        Resize(img_size, img_size),\n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        ToTensorV2(p=1.0),\n    ], p=1.)\n\ndef get_inference_transforms(img_size):\n    return Compose([\n            RandomResizedCrop(img_size, img_size),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyDataset(D.Dataset):\n    def __init__(self, df, trans, img_path):\n        self.df = df.to_records(index = False)\n        self.trans = trans\n        self.img_path = img_path\n    def __len__(self):\n        return len(self.df)\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_path, self.df.image_id[idx])\n        img = cv2.imread(img_path)\n        data = self.trans(image = img)['image']\n        target = self.df.label[idx]\n        return data, target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataloader(train_df, valid_df, batch_size, img_size, img_path):\n    train_trans = get_train_transforms(img_size)\n    valid_trans = get_valid_transforms(img_size)\n    train_loader = D.DataLoader(MyDataset(train_df, train_trans, img_path), batch_size=batch_size, shuffle=True,\n                                num_workers=8, pin_memory=True)\n    valid_loader = D.DataLoader(MyDataset(valid_df, valid_trans, img_path), batch_size=batch_size * 4, num_workers=8,\n                                pin_memory=True)\n    return train_loader, valid_loader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fftfreqnd(h, w=None, z=None):\n    \"\"\" Get bin values for discrete fourier transform of size (h, w, z)\n\n    :param h: Required, first dimension size\n    :param w: Optional, second dimension size\n    :param z: Optional, third dimension size\n    \"\"\"\n    fz = fx = 0\n    fy = np.fft.fftfreq(h)\n\n    if w is not None:\n        fy = np.expand_dims(fy, -1)\n\n        if w % 2 == 1:\n            fx = np.fft.fftfreq(w)[: w // 2 + 2]\n        else:\n            fx = np.fft.fftfreq(w)[: w // 2 + 1]\n\n    if z is not None:\n        fy = np.expand_dims(fy, -1)\n        if z % 2 == 1:\n            fz = np.fft.fftfreq(z)[:, None]\n        else:\n            fz = np.fft.fftfreq(z)[:, None]\n\n    return np.sqrt(fx * fx + fy * fy + fz * fz)\n\n\ndef get_spectrum(freqs, decay_power, ch, h, w=0, z=0):\n    \"\"\" Samples a fourier image with given size and frequencies decayed by decay power\n\n    :param freqs: Bin values for the discrete fourier transform\n    :param decay_power: Decay power for frequency decay prop 1/f**d\n    :param ch: Number of channels for the resulting mask\n    :param h: Required, first dimension size\n    :param w: Optional, second dimension size\n    :param z: Optional, third dimension size\n    \"\"\"\n    scale = np.ones(1) / (np.maximum(freqs, np.array([1. / max(w, h, z)])) ** decay_power)\n\n    param_size = [ch] + list(freqs.shape) + [2]\n    param = np.random.randn(*param_size)\n\n    scale = np.expand_dims(scale, -1)[None, :]\n\n    return scale * param\n\n\ndef make_low_freq_image(decay, shape, ch=1):\n    \"\"\" Sample a low frequency image from fourier space\n\n    :param decay_power: Decay power for frequency decay prop 1/f**d\n    :param shape: Shape of desired mask, list up to 3 dims\n    :param ch: Number of channels for desired mask\n    \"\"\"\n    freqs = fftfreqnd(*shape)\n    spectrum = get_spectrum(freqs, decay, ch, *shape)#.reshape((1, *shape[:-1], -1))\n    spectrum = spectrum[:, 0] + 1j * spectrum[:, 1]\n    mask = np.real(np.fft.irfftn(spectrum, shape))\n\n    if len(shape) == 1:\n        mask = mask[:1, :shape[0]]\n    if len(shape) == 2:\n        mask = mask[:1, :shape[0], :shape[1]]\n    if len(shape) == 3:\n        mask = mask[:1, :shape[0], :shape[1], :shape[2]]\n\n    mask = mask\n    mask = (mask - mask.min())\n    mask = mask / mask.max()\n    return mask\n\n\ndef sample_lam(alpha, reformulate=False):\n    \"\"\" Sample a lambda from symmetric beta distribution with given alpha\n\n    :param alpha: Alpha value for beta distribution\n    :param reformulate: If True, uses the reformulation of [1].\n    \"\"\"\n    if reformulate:\n        lam = beta.rvs(alpha+1, alpha)\n    else:\n        lam = beta.rvs(alpha, alpha)\n\n    return lam\n\n\ndef binarise_mask(mask, lam, in_shape, max_soft=0.0):\n    \"\"\" Binarises a given low frequency image such that it has mean lambda.\n\n    :param mask: Low frequency image, usually the result of `make_low_freq_image`\n    :param lam: Mean value of final mask\n    :param in_shape: Shape of inputs\n    :param max_soft: Softening value between 0 and 0.5 which smooths hard edges in the mask.\n    :return:\n    \"\"\"\n    idx = mask.reshape(-1).argsort()[::-1]\n    mask = mask.reshape(-1)\n    num = math.ceil(lam * mask.size) if random.random() > 0.5 else math.floor(lam * mask.size)\n\n    eff_soft = max_soft\n    if max_soft > lam or max_soft > (1-lam):\n        eff_soft = min(lam, 1-lam)\n\n    soft = int(mask.size * eff_soft)\n    num_low = num - soft\n    num_high = num + soft\n\n    mask[idx[:num_high]] = 1\n    mask[idx[num_low:]] = 0\n    mask[idx[num_low:num_high]] = np.linspace(1, 0, (num_high - num_low))\n\n    mask = mask.reshape((1, *in_shape))\n    return mask\n\n\ndef sample_mask(alpha, decay_power, shape, max_soft=0.0, reformulate=False):\n    \"\"\" Samples a mean lambda from beta distribution parametrised by alpha, creates a low frequency image and binarises\n    it based on this lambda\n\n    :param alpha: Alpha value for beta distribution from which to sample mean of mask\n    :param decay_power: Decay power for frequency decay prop 1/f**d\n    :param shape: Shape of desired mask, list up to 3 dims\n    :param max_soft: Softening value between 0 and 0.5 which smooths hard edges in the mask.\n    :param reformulate: If True, uses the reformulation of [1].\n    \"\"\"\n    if isinstance(shape, int):\n        shape = (shape,)\n\n    # Choose lambda\n    lam = sample_lam(alpha, reformulate)\n\n    # Make mask, get mean / std\n    mask = make_low_freq_image(decay_power, shape)\n    mask = binarise_mask(mask, lam, shape, max_soft)\n\n    return lam, mask\n\n\ndef sample_and_apply(x, alpha, decay_power, shape, max_soft=0.0, reformulate=False):\n    \"\"\"\n\n    :param x: Image batch on which to apply fmix of shape [b, c, shape*]\n    :param alpha: Alpha value for beta distribution from which to sample mean of mask\n    :param decay_power: Decay power for frequency decay prop 1/f**d\n    :param shape: Shape of desired mask, list up to 3 dims\n    :param max_soft: Softening value between 0 and 0.5 which smooths hard edges in the mask.\n    :param reformulate: If True, uses the reformulation of [1].\n    :return: mixed input, permutation indices, lambda value of mix,\n    \"\"\"\n    lam, mask = sample_mask(alpha, decay_power, shape, max_soft, reformulate)\n    index = np.random.permutation(x.shape[0])\n\n    x1, x2 = x * mask, x[index] * (1-mask)\n    return x1+x2, index, lam\n\n\nclass FMixBase:\n    r\"\"\" FMix augmentation\n\n        Args:\n            decay_power (float): Decay power for frequency decay prop 1/f**d\n            alpha (float): Alpha value for beta distribution from which to sample mean of mask\n            size ([int] | [int, int] | [int, int, int]): Shape of desired mask, list up to 3 dims\n            max_soft (float): Softening value between 0 and 0.5 which smooths hard edges in the mask.\n            reformulate (bool): If True, uses the reformulation of [1].\n    \"\"\"\n\n    def __init__(self, decay_power=3, alpha=1, size=(32, 32), max_soft=0.0, reformulate=False):\n        super().__init__()\n        self.decay_power = decay_power\n        self.reformulate = reformulate\n        self.size = size\n        self.alpha = alpha\n        self.max_soft = max_soft\n        self.index = None\n        self.lam = None\n\n    def __call__(self, x):\n        raise NotImplementedError\n\n    def loss(self, *args, **kwargs):\n        raise NotImplementedError\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fmix(data, targets, alpha, decay_power, shape, max_soft=0.0, reformulate=False):\n    lam, mask = sample_mask(alpha, decay_power, shape, max_soft, reformulate)\n    # mask =torch.tensor(mask, device=device).float()\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_targets = targets[indices]\n    x1 = torch.from_numpy(mask).to('cuda') * data\n    x2 = torch.from_numpy(1 - mask).to('cuda') * shuffled_data\n    targets = (targets, shuffled_targets, lam)\n\n    return (x1 + x2), targets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model_name, train_df, valid_df, batch_size, img_size, img_path, logger, cv_idx, if_bn):\n    # 模型加载\n    model = timm.create_model(model_name, True)\n    model.fc = nn.Linear(2048, 5)\n    model = model.to('cuda')\n\n    # 优化器和损失\n    optimizer = optimizer = torch.optim.Adam(model.parameters(), lr=1e-4, weight_decay=1e-6)\n    scheduler = scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1,\n                                                                     eta_min=1e-6, last_epoch=-1)\n    criterion = label_smoothing2()\n    scaler = torch.cuda.amp.GradScaler()\n\n    # 数据加载\n    train_loader, valid_loader = get_dataloader(train_df, valid_df, batch_size, img_size, img_path)\n\n    # 训练\n    early_stop = 0\n    best_valid_acc = 0\n    for epoch in range(10000):\n        model.train()\n        train_loss = 0\n        train_acc = 0\n        for data, target in tqdm(train_loader):\n            data = data.cuda()\n            target = target.cuda()\n#             mix_decision = np.random.rand()\n#             if mix_decision < 0.25:\n#                 data, target = dataset.cutmix(data, target, 1.)\n#             if mix_decision < 0.5:\n            data, target = fmix(data, target, alpha=1., decay_power=5., shape=(512, 512))\n            optimizer.zero_grad()\n            with torch.cuda.amp.autocast():\n                output = model(data.float())\n#                 if mix_decision < 0.50:\n                loss = criterion(output, target[0]) * target[2] + criterion(output, target[1]) * (1. - target[2])\n#                 else:\n#                     loss = criterion(output, target)\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            train_loss += loss.item()\n        scheduler.step()\n        train_loss = train_loss / len(train_loader.dataset)\n\n        model.eval()\n        valid_acc = 0\n        valid_loss = 0\n        with torch.no_grad():\n            for data, target in tqdm(valid_loader):\n                data = data.cuda()\n                target = target.cuda()\n                with torch.cuda.amp.autocast():\n                    output = model(data.float())\n                loss = criterion(output, target)\n                pred = torch.argmax(output, dim=1)\n                correct = pred.eq(target)\n                acc = correct.type(torch.FloatTensor).sum()\n                valid_acc += acc\n                valid_loss += loss.item()\n        #                 print(loss.item(), valid_loss)\n        valid_loss = valid_loss / len(valid_loader.dataset)\n        valid_acc = valid_acc / len(valid_loader.dataset)\n\n        print(f'epoch:{epoch} | train_loss:{train_loss} | valid_loss:{valid_loss}')\n        print(f'train_acc:{train_acc} | valid_acc:{valid_acc}')\n        logger.info(f'epoch:{epoch} | train_loss:{train_loss} | valid_loss:{valid_loss}')\n        logger.info(f'train_acc:{train_acc} | valid_acc:{valid_acc}')\n\n        if valid_acc > best_valid_acc:\n            best_valid_acc = valid_acc\n            early_stop = 0\n            torch.save(model.state_dict(), f'cv{cv_idx}.pkl')\n            logger.info(f'best at epoch {epoch}')\n        else:\n            early_stop += 1\n            if early_stop > 5:\n                logger.info(f'stop at epoch {epoch}')\n                break\n    del model\n    torch.cuda.empty_cache()\n    return best_valid_acc","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# 日志记录\nlogger = logging.getLogger(__name__)\nlogger.setLevel(level=logging.INFO)\nhandler = logging.FileHandler('result.log')\nformatter = logging.Formatter('%(message)s')\nhandler.setFormatter(formatter)\nlogger.addHandler(handler)\n\nlogger.info('')\n\n# 固定随机种子\nseed_everything(1001)\n\n# 加载数据表，并进行五折划分\ndf = pd.read_csv('/kaggle/input/cassava-leaf-disease-classification/train.csv')\ncv = get_cv(df)\n\n# 进行五折交叉验证的训练\nfor i in range(5):\n    print(f'cv: {i} start:')\n    logger.info(f'cv: {i} start:')\n    train_df = cv[i][1]\n    valid_df = cv[i][0]\n    best_acc = train('seresnext50_32x4d', train_df, valid_df, 16,\n                           512, '/kaggle/input/cassava-leaf-disease-classification/train_images', logger, i, if_bn = True)\n    print(f'cv{i} best_acc: {best_acc}')\n    logger.info(f'cv{i} best_acc: {best_acc}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}