{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 0.Prep"},{"metadata":{},"cell_type":"markdown","source":"## 0.1 Import Libs"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport cv2    #OpenCV基于BSD许可(开源)发行的跨平台计算机视觉库\nfrom glob import glob    #glob.golb：匹配路径文件名\n\nimport sklearn\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold    #scikit-learn\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nfrom sklearn.metrics import log_loss\n\nfrom skimage import io    #Scikit-Image基于python脚本语言开发的数字图片处理包；子模块io读取、保存和显示图片或视频\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport torchvision\nfrom torchvision import transforms\nfrom tqdm import tqdm    #进度条\n\nimport matplotlib.pyplot as plt\n\nimport torch\nfrom torch import nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport warnings\nimport joblib\n\nfrom scipy.ndimage.interpolation import zoom","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 0.2 Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {    #configuration\n    'fold_num': 12,\n    'seed': 719,\n    'model_arch': 'tf_efficientnet_b3_ns',\n    'img_size': 384,\n    'epochs': 120,\n    'train_bs': 28,\n    'valid_bs': 32,\n    'lr': 1e-2,\n    'num_workers': 5,\n    'accum_iter': 1,\n    'verbose_step': 2,\n    'device': 'cuda:0',\n    'tta': 10,\n    'used_epochs': [6,7,8,9],\n    'weights': [1,1,1,1]\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 0.3 Helper function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def all_seed(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    return im_rgb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 0.4 Data Profile"},{"metadata":{},"cell_type":"markdown","source":"Image"},{"metadata":{"trusted":true},"cell_type":"code","source":"#显示图像\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"csv"},{"metadata":{"trusted":true},"cell_type":"code","source":"#读csv文件\ntrain = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#csv文件前五项\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#训练集数据总况\ntrain.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Sample of submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"#要提交的csv文件样例\nsample_submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1. Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):    #继承类(基类：torch.utils.data.Dataset)\n    \n    def __init__(self, df, data_root, transforms = None, output_label = True):    #类的初始化方法\n        '''\n        Arguments:\n            self: 类的实例\n            df: pandas数据帧(DataFrame)\n            data_root: 数据根目录\n            transforms: 数据增强\n            output_label: 输出标记\n\n        Returns:\n            None\n        '''\n        super().__init__()\n        self.df = df.reset_index(drop = True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n        \n    def __len__(self):    #返回数据集大小\n        '''\n        Arguments:\n            self: 类的实例\n        \n        Returns:\n            self.df.shape[0]: 数据集大小\n        '''\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):    #按索引取出数据，并预处理\n        '''\n        Arguments:\n            self: 类的定义\n            index: 要取数据的索引\n        \n        Returns:\n            img: 图像\n            target: 当output_label为True，则输出图像相应标记(label)\n        '''\n        #获取图像\n        #pandas.iloc基于整数的索引方式，第index位的图像id\n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        img = get_img(path)\n        \n        #图像预处理\n        if self.transforms:\n            img = self.transforms(image = img)['image']\n            \n        #获取标记(label)\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n        \n        #返回值\n        if self.output_label:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2.Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, Transpose, ShiftScaleRotate, \n    HueSaturationValue,RandomResizedCrop, RandomBrightnessContrast, \n    Compose, Normalize, Cutout, CoarseDropout, CenterCrop, Resize\n)\n#albumentations: 图片数据增强库\n\nfrom albumentations.pytorch import ToTensorV2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transforms():\n    '''\n    Arguments:\n        None\n    \n    Returns:\n        Compose: albumentations.Compose(),串联多个图片变换的操作\n            RandomResizedCrop: 图像随机裁剪为不同大小和宽高比，然后缩放为制定的大小\n            Transpose: 转置\n            HorizontalFlip: Y轴水平翻转\n            VerticalFlip: X轴垂直翻转\n            ShiftScaleRotate: 随机应用仿射变换：平移，缩放和旋转\n            HueSaturationValue: 色调饱和度值\n            RandomBrightnessContrast: 随机亮度对比度\n            Normalize: 归一化\n            CoarseDropout: 在图像上生成矩形区域\n            Cutout: 在图像中生成正方形区域\n            ToTensorV2: 张量转换     \n    '''\n    return Compose([\n        RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n        Transpose(p=0.5),\n        HorizontalFlip(p=0.5),\n        VerticalFlip(p=0.5),\n        ShiftScaleRotate(p=0.5),\n        HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n        RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        CoarseDropout(p=0.5),\n        Cutout(p=0.5),\n        ToTensorV2(p=1.0),\n    ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_valid_transforms():\n    '''\n    Arguments:\n        None\n    \n    Returns:\n        Compose\n            CenterCrop: 随机中心裁剪\n            Resize: 将图像调整为给定的高度和宽度\n            Normalize: 归一化\n            ToTensorV2: 张量转换\n    '''\n    return Compose([\n        CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n        Resize(CFG['img_size'], CFG['img_size']),\n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        ToTensorV2(p=1.0),\n    ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_inference_transforms():\n    '''\n    Arguments:\n        None\n    \n    Returns:\n        Compose: albumentations.Compose(),串联多个图片变换的操作\n            RandomResizedCrop: 图像随机裁剪为不同大小和宽高比，然后缩放为制定的大小\n            Transpose: 转置\n            HorizontalFlip: Y轴水平翻转\n            VerticalFlip: X轴垂直翻转\n            HueSaturationValue: 色调饱和度值\n            RandomBrightnessContrast: 随机亮度对比度\n            Normalize: 归一化\n            ToTensorV2: 张量转换     \n    '''\n    return Compose([\n        RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n        Transpose(p=0.5),\n        HorizontalFlip(p=0.5),\n        VerticalFlip(p=0.5),\n        HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n        RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        ToTensorV2(p=1.0),\n    ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## 3.Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\n\npackage_path = '../input/pytorch-image-models/pytorch-image-models-master'\nsys.path.append(package_path)\n\nimport timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaImgClassifier(nn.Module):    #继承类(torch.nn.Module)\n\n    def __init__(self, model_arch, n_class, pretrained=False):\n        '''\n        Arguments:\n            self:类的实例\n            model_arch:模型名\n            n_class: 类别数\n            pretrained: 是否加载预训练模型\n            \n        Returns:\n            None\n        '''\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        \n    def forward(self, x):    #前向传递\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**CLASS** PyTorch.nn.Linear(*in_features, out_features, bias = True*)\n\nApplies a linear transformation to the incoming data: $y = xA^T+b$\n\n用于设置网络中的全连接层，输入与输出都是二维张量，一般形状为`[batch_size,size]`\n\n`in_features`指的是输入的二维张量的大小，即输入的`[batch_size, size]`中的`size`。\n\n`out_features`指的是输出的二维张量的大小，也代表了该全连接层的神经元个数。"},{"metadata":{},"cell_type":"markdown","source":"**efficientnet**\n\n卷积神经网络(cnn)通常是以固定的资源成本开发，然后在更多资源加入进来时扩大规模，以达到更高精度。传统的模型缩放实践是任意增加 CNN 的深度或宽度，或使用更大的输入图像分辨率进行训练和评估。 虽然这些方法确实提高了准确性，但它们通常需要长时间的手动调优，并且仍然会经常产生次优的性能。\n\n最近的一篇ICML文章提出了一个更有原则性的方法来扩大 CNN 的规模，从而可以获得更好的准确性和效率。该论文提出了一种新的模型缩放方法，它使用一个简单而高效的复合系数来以更结构化的方式放大 CNNs。 不像传统的方法那样任意缩放网络维度，如宽度，深度和分辨率，该论文的方法用一系列固定的尺度缩放系数来统一缩放网络维度。 通过使用这种新颖的缩放方法和AutoML技术，作者将这种模型称为 EfficientNets ，它具有最高达10倍的效率(更小、更快)。\n\n作者系统的研究了网络深度（Depth）、宽度（Width）和分辨率（resolution)对网络性能的影响，然后提出了一个新的缩放方法--使用简单但高效的复合系数均匀地缩放深度/宽度/分辨率的所有尺寸，在MobileNets 和 ResNet上证明了有效性。\n\n同时，使用神经架构搜索来设计新的baseline并进行扩展以获得一系列模型，称EfficientNets，比之前的ConvNets更accuracy和更efficiency。其中EfficientNet-B7实现了ImageNet的state-of-the-art，即84.4% top-1 / 97.1% top-5 accuracy，同时该模型比现存最好的ConvNets的size小8.4倍，速度快6.1倍。\n\n摘自：\n    https://zhuanlan.zhihu.com/p/67508423\n    https://zhuanlan.zhihu.com/p/67834114"},{"metadata":{"trusted":true},"cell_type":"code","source":"package_path = '../input/visiontransformer-pytorch/VisionTransformer-Pytorch-main'\nsys.path.append(package_path)\n\nfrom vision_transformer_pytorch import VisionTransformer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EnsembleClassifier(nn.Module):    #继承类(torch.nn.Module)\n    \n    def __init__(self, model_arch, n_class, pretrained=False):\n        '''\n        Arguments:\n            self:类的实例\n            model_arch:模型名\n            n_class: 类别数\n            pretrained: 是否加载预训练模型\n            \n        Returns:\n            None\n        '''\n        super().__init__()\n        self.model1 = VisionTransformer.from_name('ViT-B_16', num_classes=5)\n        self.model1.load_state_dict(torch.load('../input/vit-model-1/ViT-B_16.pt'))    #预训练模型1\n        self.model2 = CassavaImgClassifier(model_arch, n_class, pretrained)    #预训练模型2\n        \n    def forward(self, x):    #前向传递，模型1权值0.6，模型2权值0.4\n        x1 = self.model1(x)\n        x2 = self.model2(x)\n        return 0.6 * x1 + 0.4 * x2\n    \n    def load(self, state_dict):\n        self.model2.load_state_dict(state_dict)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4.Main Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    '''\n    Arguments:\n        model: \n        data_loader: \n        device: \n    Returns:\n        image_preds_all: \n    '''\n    model.eval()    #注1\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))    #进度条\n        #enumerate(): 将一个可遍历的数据对象(如列表、元组或字符串)组合为一个索引序列，同时列出数据和数据下标\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()    #将数据copy到device指定的GPU上\n        \n        image_preds = model(imgs)   \n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]    #注2\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)    #对array进行拼接\n    return image_preds_all","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**注1：model.train()与model.eval()**\n\n如果模型中有`BN(Batch Normalization)`和`Dropout`，需要在训练时添加`model.train()`，在测试时添加`model.eval()`。其中`model.train()`是保证`BN`用每一批数据的均值和方差，而`model.eval()`是保证`BN`用全部训练数据的均值和方差；而对于`Dropout`，`model.train()`是随机取一部分网络连接来训练更新参数，而`model.eval()`是用到了所有网络连接。\n\n`Batch Normalization`对网络中间的每层进行归一化处理，并且使用变换重构（Batch Normalization Transform）保证每层所提取的特征分布不会被破坏。\n\n`Dropout`能克服过拟合，该层(layer)的神经元在每次迭代训练时有概率p的可能性不参与训练"},{"metadata":{},"cell_type":"markdown","source":"**注2：torch.softmax(image_preds, 1).detach().cpu().numpy()**\n\n**①**`class torch.nn.Softmax(input, dim)`\n\n对n维输入张量运用`Softmax`函数，将张量的每个元素缩放到（0,1）区间且和为1。`Softmax`函数定义如下：\n\n$$f_i(x) = \\frac{e^{(x_i-shift)}}{\\Sigma^je^{(x_j-shift)}},shift = max(x_i)$$\n\n`dim`指明维度，`dim=0`表示按列计算；`dim=1`表示按行计算\n\n**②**`detach()`\n\n再训练网络时可能希望保持一部分参数不变，只对其中一部分参数进行调整；或只训练部分分支网络，希望其梯度不对主网络的梯度造成影响\n\n`detach()`阻断反向传播，返回张量(tensor)\n\n**③**`cpu()`将数据移至CPU中\n\n**④**`numpy()`tensor变量转numpy变量"},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':    #确保只有单独运行该模块时，此表达式才成立，才可以进入此判断语法\n\n    all_seed(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n        #注1\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        \n        # 首先训练 fold 0\n        if fold > 0:\n            break \n\n        print('Inference fold {} started'.format(fold))\n        \n        #数据集的建立(valid_ds, test_ds)\n        valid_ = train.loc[val_idx,:].reset_index(drop=True)    #loc: 通过行标签索引行数据; reset_index重置索引\n                  #CassavaDataset的参数：df, data_root, transforms, output_label\n        valid_ds = CassavaDataset(valid_, '../input/cassava-leaf-disease-classification/train_images/', \n                                  transforms = get_inference_transforms(), output_label=False)\n\n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n            #os.listdir: 返回指定的文件夹包含的文件或文件夹的名字的列表\n        test_ds = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', \n                                 transforms = get_inference_transforms(), output_label=False)\n        \n        #读数据\n        val_loader = torch.utils.data.DataLoader(\n            valid_ds, \n            batch_size = CFG['valid_bs'],\n            num_workers = CFG['num_workers'],\n            shuffle = False,\n            pin_memory = False,\n        )\n        \n        tst_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size = CFG['valid_bs'],\n            num_workers = CFG['num_workers'],\n            shuffle = False,\n            pin_memory = False,\n        )\n\n        device = torch.device(CFG['device'])\n        model = EnsembleClassifier(CFG['model_arch'], train.label.nunique()).to(device)\n        \n        val_preds = []\n        tst_preds = []\n        \n        #for epoch in range(CFG['epochs']-3):\n        for i, epoch in enumerate(CFG['used_epochs']):    \n            model.load(torch.load('../input/fork-pytorch-efficientnet-baseline-train-amp-a/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))\n            \n            with torch.no_grad():\n                for _ in range(CFG['tta']):\n                    val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]\n                    tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]\n\n        val_preds = np.mean(val_preds, axis=0) \n        tst_preds = np.mean(tst_preds, axis=0) \n        \n        print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))\n        print('fold {} validation accuracy = {:.5f}'.format(fold, (valid_.label.values==np.argmax(val_preds, axis=1)).mean()))\n        \n        del model\n        torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"**注1**`StratifiedKFold`\n\n① *k折交叉验证*\n\n- 将全部训练集S分成k个不相交的子集，假设S中的训练样例个数为m，则每个子集有m/k个训练样例，相应的子集为{s1，s2，...，sk}\n- 每次从分好的子集里面，拿出一个作为测试集，其他k-1个作为训练集\n- 在k-1个训练集上训练出模型，再把这个模型放到测试集上，得到分类率的平均值，作为该模型或者假设函数的真实分类率\n- 10折交叉验证是最常用的\n\n② *StratifiedKFold 和 KFold 的比较*\n\n- StratifiedKFold 分层采样交叉切分，确保训练集，测试集中各类别样本的比例与原始数据集中相同\n- KFold不能整除时，每个互斥集的样本数会尽量相近\n\n③ *StratifiedKFold参数*\n- `n_splits`: 折叠次数，默认为3，至少为2\n- `shuffle`: 是否在每次分割之前打乱顺序\n- `random_state`: 随机种子，在shuffle==True时使用，默认使用np.random\n\n④ *.split(X,y)*\n- X:array-like,shape(n_sample,n_features)，训练数据集。\n- y:array-like,shape(n_sample)，标签。\n- 返回值：训练集数据的index与验证集数据的index。"},{"metadata":{"trusted":true},"cell_type":"code","source":"test['label'] = np.argmax(tst_preds, axis=1)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}