{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/git-repos/pretrained-models.pytorch/pretrained-models.pytorch')\nsys.path.append('../input/git-repos/EfficientNet-PyTorch/EfficientNet-PyTorch')\nsys.path.append('../input/git-repos/pytorch-image-models/pytorch-image-models')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nimport time\nimport random\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nfrom torch.utils.data import Dataset,DataLoader\n\nfrom sklearn import metrics\nimport warnings\nimport timm #from efficientnet_pytorch import EfficientNet\nfrom collections import OrderedDict\nimport pretrainedmodels\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG01 = {\n    'weight': 0.902,\n    'model_arch': 'tf_efficientnet_b4_ns', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':512,\n    'width':512,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-exp37',\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'all', # best last\n}\n\nCFG02 = {\n    'weight': 0.900,\n    'model_arch': 'seresnext50_32x4d', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':512,\n    'width':512,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-exp39',\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'all', # best last\n}\n\nCFG04 = {\n    'weight': 0.903,\n    'model_arch': 'vit_base_patch16_384', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':384,\n    'width':384,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-vit-best', # '../input/cassava-vit-cutmix',\n    'clsw': [[0.2, 0.15, 0, 0.1, 0.1], [0.4, 0.3, 0.2, 0.35, 0.25], [0, 0, 0.3, 0.3, 0], [0, 0.15, 0.1, 0.25, 0.4], [0.4, 0.4, 0.4, 0, 0.25]],\n#     'clsw': [[0.40, 0.30, 0.00, 0.20, 0.00], \n#              [0.10, 0.30, 0.40, 0.20, 0.10], \n#              [0.10, 0.10, 0.15, 0.20, 0.30], \n#              [0.00, 0.00, 0.10, 0.20, 0.40], \n#              [0.40, 0.30, 0.15, 0.20, 0.20]],\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'best', # best last\n}\n\nCFG05 = {\n    'weight': 0.900,\n    'model_arch': 'tf_efficientnet_b1_ns', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':512,\n    'width':512,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-exp56-b1',\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'all', # best last\n}\n\nCFG06 = {\n    'weight': 0.899,\n    'model_arch': 'tf_efficientnet_b2_ns', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':512,\n    'width':512,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-exp56-b2',\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'all', # best last\n}\n\nCFG07 = {\n    'weight': 0.900,\n    'model_arch': 'tf_efficientnet_b3_ns', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':512,\n    'width':512,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-exp56-b3',\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'all', # best last\n}\n\nCFG08 = {\n    'weight': 0.900,\n    'model_arch': 'regnety_032', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':512,\n    'width':512,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-regnety032',\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'best', # best last\n}\n\n\nCFG09 = {\n    'weight': 0.902,\n    'model_arch': 'tf_efficientnet_b4_ns', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':512,\n    'width':512,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-exp57-radam',\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'all', # best last\n}\n\nCFG10 = {\n    'weight': 0.902,\n    'model_arch': 'tf_efficientnet_b4_ns', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':512,\n    'width':512,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-fenghan',\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'best', # best last\n}\n\nCFG11 = {\n    'weight': 0.903,\n    'model_arch': 'tf_efficientnet_b4_ns', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':512,\n    'width':512,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-exp67-sgd',\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'best', # best last\n}\n\nCFG12 = {\n    'weight': 0.902,\n    'model_arch': 'tf_efficientnet_b4_ns', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':512,\n    'width':512,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-exp68-ts',\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'best', # best last\n}\n\nCFG13 = {\n    'weight': 0.901,\n    'model_arch': 'tf_efficientnet_b4_ns', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':512,\n    'width':512,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-exp68-bce-best',\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'best', # best last\n}\n\nCFG14 = {\n    'weight': 0.899,\n    'model_arch': 'resnest50d_4s2x40d', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':512,\n    'width':512,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-exp70-resnest50',\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'best', # best last\n}\n\nCFG15 = {\n    'weight': 0.902,\n    'model_arch': 'tf_efficientnet_b4_ns', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':768,\n    'width':768,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-exp67-768-16',\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'best', # best last\n}\nCFG16 = {\n    'weight': 0.902,\n    'model_arch': 'seresnext50_32x4d', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':768,\n    'width':768,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-exp75-1',\n#     'clsw': [[0.00, 0.30, 0.40, 0.00, 0.40], \n#              [0.20, 0.10, 0.30, 0.40, 0.00], \n#              [0.40, 0.20, 0.00, 0.20, 0.10], \n#              [0.40, 0.20, 0.10, 0.20, 0.30], \n#              [0.00, 0.20, 0.00, 0.20, 0.20]],\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'best', # best last\n}\n\nCFG17 = {\n    'weight': 0.902,\n    'model_arch': 'regnety_032', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':768,\n    'width':768,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-exp70-2',\n#     'clsw': [[0.00, 0.30, 0.40, 0.00, 0.40], \n#              [0.20, 0.10, 0.30, 0.40, 0.00], \n#              [0.40, 0.20, 0.00, 0.20, 0.10], \n#              [0.40, 0.20, 0.10, 0.20, 0.30], \n#              [0.00, 0.20, 0.00, 0.20, 0.20]],\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'best', # best last\n}\n\nCFG18 = {\n    'weight': 0.902,\n    'model_arch': 'tf_efficientnet_b4_ns', # se_resnext50_32x4d tf_efficientnet_b0_ns seresnext50_32x4d regnety_080\n    'height':512,\n    'width':512,\n    'valid_bs': 32,\n    'tta': 3,\n    'ckpt_path': '../input/cassava-exp68-03',\n    'flag': False, # 使用pytorch-lightning的训练模型,就用False\n    'which': 'best', # best last\n}\n\nCFGS = [CFG18, CFG02, CFG04, CFG08]\n# CFGS = [CFG12, CFG15, CFG08]\nWS = [cfg['weight'] for cfg in CFGS]\n# WS = [1] * len(CFGS)\n\nUSE_WEIGHT = False\nassert len(WS) == len(CFGS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb\n\nclass CassavaDataset(Dataset):\n    def __init__(\n        self, df, data_root, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        \n        img  = get_img(path)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_valid_transforms():\n    return Compose([\n            CenterCrop(CFG['height'], CFG['width'], p=1.),\n            Resize(CFG['height'], CFG['width']),\n            Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\ndef get_inference_transforms(mean, std):\n    return Compose([\n            RandomResizedCrop(CFG['height'], CFG['width']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\ndef get_inference_transforms2(mean, std):\n    return Compose([\n            RandomResizedCrop(CFG['height'], CFG['width']),\n#             Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n#             VerticalFlip(p=0.5),\n#             HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n#             RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaModelTimm(nn.Module):\n    def __init__(self, model_arch, n_class=5, pretrained=False):\n        super().__init__()\n        self.model_arch = model_arch\n        self.model = timm.create_model(model_arch, pretrained=False, num_classes=768)\n#         self.gem = GeM(p=3, eps=1e-6)\n#         n_features = self.model.classifier.in_features\n#         self.model.classifier = nn.Linear(n_features, 768)\n        self.model.metric = nn.Linear(768, n_class)\n        \n    def forward(self, x):\n        if 'efficientnet' in self.model_arch:\n            x = self.model.conv_stem(x)\n            x = self.model.bn1(x)\n            x = self.model.act1(x)\n            x = self.model.blocks(x)\n            x = self.model.conv_head(x)\n            x = self.model.bn2(x)\n            fea_conv = x = self.model.act2(x)\n            x = self.model.global_pool(x)\n#             x = self.gem(x)\n            x = self.model.classifier(x)\n            x = self.model.metric(x)\n        elif 'seresnext' in self.model_arch:\n            x = self.model.conv1(x)\n            x = self.model.bn1(x)\n            x = self.model.act1(x)\n            x = self.model.maxpool(x)\n            x = self.model.layer1(x)\n            x = self.model.layer2(x)\n            x = self.model.layer3(x)\n            fea_conv = x = self.model.layer4(x)\n            x = self.model.global_pool(x)\n            x = self.model.fc(x)\n            x = self.model.metric(x)\n        elif 'regnet' in self.model_arch:\n            x = self.model.stem(x)\n            x = self.model.s1(x)\n            x = self.model.s2(x)\n            x = self.model.s3(x)\n            fea_conv = x = self.model.s4(x)\n            x = self.model.head(x)\n            x = self.model.metric(x)\n        elif 'resnest' in self.model_arch:\n            x = self.model.conv1(x)\n            x = self.model.bn1(x)\n            x = self.model.act1(x)\n            x = self.model.maxpool(x)\n            x = self.model.layer1(x)\n            x = self.model.layer2(x)\n            x = self.model.layer3(x)\n            fea_conv = x = self.model.layer4(x)\n            x = self.model.global_pool(x)\n            x = self.model.fc(x)\n            x = self.model.metric(x)\n        elif 'vit' in self.model_arch:\n            fea_conv = x = self.model(x)\n\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Main Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n    image_preds_all = []\n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)   #output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n\n    return image_preds_all\n#     return image_preds_all**0.5 # tsharp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(719) # 719\ntest = pd.DataFrame()\ntest['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\ndevice = torch.device('cuda:0')\nfinal_preds = []\ntst_preds = []\nfor idx, CFG in enumerate(CFGS):\n    mean=[0.485, 0.456, 0.406]\n    std=[0.229, 0.224, 0.225]\n    if 'vit' in CFG['model_arch']:\n        mean=[0.5,0.5,0.5]\n        std=[0.5,0.5,0.5]\n    transforms = get_inference_transforms(mean, std) if CFG['tta'] > 1 else get_valid_transforms(mean, std)\n    if 'fenghan' in CFG['ckpt_path']:\n        transforms = get_inference_transforms2(mean, std) if CFG['tta'] > 1 else get_valid_transforms(mean, std)\n    test_ds = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=transforms, output_label=False)\n    tst_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=4,\n            shuffle=False,\n            pin_memory=False,\n        )\n\n    ws = []\n    one_model_preds = []\n    for ii, name in enumerate(sorted(os.listdir(CFG['ckpt_path']))):\n        if CFG['which'] == 'best':\n            if 'last' in name:\n                continue\n        elif CFG['which'] == 'last':\n            if 'last' not in name:\n                continue\n        ckpt_path = os.path.join(CFG['ckpt_path'], name)\n        w = CFG['weight'] if USE_WEIGHT else 1\n        ws.append(w)\n        if 'exp75' in ckpt_path:\n            model = CassavaModelTimm(CFG['model_arch']).cuda()\n            state_dict = torch.load(ckpt_path)[\"state_dict\"]\n            model.load_state_dict(state_dict)\n        else:\n            model = timm.create_model(CFG['model_arch'], pretrained=False, num_classes=5).cuda()\n\n            state_dict = torch.load(ckpt_path)[\"state_dict\"]\n            new_state_dict = OrderedDict()\n            for k, v in state_dict.items():\n                if 'criterion' in k:\n                    continue\n                if 'model' in k:\n                    k = k[6:]\n                if 'model' in k:\n                    k = k[6:]\n                new_state_dict[k] = v \n            model.load_state_dict(new_state_dict)\n                \n        tta_preds = []\n        with torch.no_grad():\n            for _ in range(CFG['tta']):\n#                 tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]\n                if 'clsw' not in CFG.keys():\n                    tta_preds.append(w*inference_one_epoch(model, tst_loader, device))\n                else:\n                    tta_preds.append(w*inference_one_epoch(model, tst_loader, device)*np.array(CFG['clsw'][ii]))\n    \n        \n        tta_preds = np.sum(tta_preds, axis=0) / CFG['tta']\n        one_model_preds.append(tta_preds)\n\n        del model\n        torch.cuda.empty_cache()\n    one_model_preds = np.sum(one_model_preds, axis=0) / sum(ws)\n    final_preds.append(one_model_preds*WS[idx])\nfinal_preds = np.sum(final_preds, axis=0) / sum(WS)\nlabels = np.argmax(final_preds, axis=1)\ntest['label'] = labels\ntest.to_csv('submission.csv', index=False)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}