{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# !cd ../input/pretrainedmodels\n# !python setup.py install\n! pip install ../input/mlcollection/ml_collections-0.1.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"package_path = '../input/pytorch-image-models/pytorch-image-models-master' #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\nimport sys; sys.path.append(package_path)\nsys.path.insert(0, '../input/')\nsys.path.insert(0, '../input/ibnnet')\nsys.path.insert(0, '../input/cass-net')\nsys.path.insert(0, '../input/pretrainedmodels')\nsys.path.insert(0, '../input/local-libs')\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nfrom fastai.vision.all import *\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom  torch.cuda.amp import autocast, GradScaler\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\nimport timm #from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom\nfrom sklearn.metrics import log_loss\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 719,\n    'model_arch': 'resnext101_ibn_a',#'resnext101',#'resnext101_ibn_a',#'se50_softlabel',#'resnext101_ibn_a',#'tf_efficientnet_b4_ns',\n    'model_arch_eff':'tf_efficientnet_b4_ns',\n    'img_size': 512,\n    'epochs': 10,\n    'train_bs': 32,\n    'valid_bs': 32,\n    'lr': 1e-4,\n    'num_workers': 4,\n    'accum_iter': 1, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda' if torch.cuda.is_available() else 'cpu',\n    'tta': 4,\n\n}\n\nckpt_paths = ['../input/tf-efficientnet-b4-ns-fold-0-8-089533/tf_efficientnet_b4_ns_fold_0_4_0.89299',\n              '../input/tf-efficientnet-b4-ns-fold-0-8-089533/tf_efficientnet_b4_ns_fold_0_6_0.89065',\n             '../input/tf-efficientnet-b4-ns-fold-0-8-089533/tf_efficientnet_b4_ns_fold_0_8_0.89533',\n              '../input/tf-efficientnet-b4-ns-fold-0-8-089533/tf_efficientnet_b4_ns_fold_0_9_0.88995'\n             ]\n\n# version 4 tta=3\nckpt_paths = [\n             '../input/resnext101-ibn-a-01-22-10-05-bit-ls015-896/resnext101_ibn_a_fold_0_6_0.88435',\n             '../input/resnext101-ibn-a-01-22-10-05-bit-ls015-896/resnext101_ibn_a_fold_0_7_0.89299',\n             '../input/resnext101-ibn-a-01-22-10-05-bit-ls015-896/resnext101_ibn_a_fold_0_8_0.88995',\n             '../input/resnext101-ibn-a-01-22-10-05-bit-ls015-896/resnext101_ibn_a_fold_0_9_0.89603']\nckpt_weights =  [1,1,1,1]\n\nckpt_dir = '../input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds'\nckpt_paths = [os.path.join(ckpt_dir, sub_path) for sub_path in os.listdir(ckpt_dir) if not ckpt_dir.endswith('.txt')]\n\nckpt_paths = ['../input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_0_9_0.89439',\n              '../input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_0_19_0.89393',\n              '../input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_1_9_0.89603',\n              '../input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_1_8_0.89486',\n              '../input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_2_9_0.88759',\n              '../input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_2_17_0.88619',\n              '../input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_3_16_0.89928',\n              '../input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_3_19_0.89904',\n              '../input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_4_18_0.89857',\n              '../input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_4_9_0.89787',\n             ]\n\n\n#ckpt_dir = '../input/resnext101ibn0201select'\n#ckpt_paths = [os.path.join(ckpt_dir, fn) for fn in os.listdir(ckpt_dir)]\n\n\n#ckpt_dir = '../input/alldata-ema-resnext101-ibn-a'\n#ckpt_paths = [os.path.join(ckpt_dir, fn) for fn in os.listdir(ckpt_dir)]\n\n\nckpt_paths = [\n    \n              #'../input/seresnex101ibn-timm/checkpoint-21_f0_90.16.pth.tar',\n              #'../input/seresnex101ibn-timm/checkpoint-18_f1_9014.pth.tar',\n              #'../input/seresnex101ibn-timm/checkpoint-30_f2_90.04.pth.tar',\n              #'../input/seresnex101ibn-timm/checkpoint-30_f3_88.52.pth.tar',\n              #'../input/seresnex101ibn-timm/checkpoint-16_f4_89.57.pth.tar',\n                \n             '../input/vit-convert-tta-seed719/vit__1920-fold0_checkpoint-17.pth',\n              '../input/vit-convert-tta-seed719/vit__1920-fold1_checkpoint-14.pth',\n              '../input/vit-convert-tta-seed719/vit__1920-fold2_checkpoint-16.pth',\n              '../input/vit-convert-tta-seed719/vit__1920-fold3_checkpoint-8.pth',\n              '../input/vit-convert-tta-seed719/vit__1920-fold4_checkpoint-9.pth',\n    \n             '../input/alldata-ema-resnext101-ibn-a/resnext101_ibn_a_fold_0_11_0.89603',\n             '../input/alldata-ema-resnext101-ibn-a/resnext101_ibn_a_fold_1_10_0.89369',\n             '../input/alldata-ema-resnext101-ibn-a/resnext101_ibn_a_fold_2_13_0.88899',\n             '../input/alldata-ema-resnext101-ibn-a/resnext101_ibn_a_fold_3_12_0.89834',\n             '../input/alldata-ema-resnext101-ibn-a/resnext101_ibn_a_fold_4_11_0.89741',\n\n             '../input/alldata-ema-resnext101-32x4d-swsl-adam-02-05-17-28/resnext101_32x4d_swsl_fold_0_12_0.89369',\n              '../input/alldata-ema-resnext101-32x4d-swsl-adam-02-05-17-28/resnext101_32x4d_swsl_fold_1_12_0.89416',\n              '../input/alldata-ema-resnext101-32x4d-swsl-adam-02-05-17-28/resnext101_32x4d_swsl_fold_2_8_0.89203',\n              '../input/alldata-ema-resnext101-32x4d-swsl-adam-02-05-17-28/resnext101_32x4d_swsl_fold_3_12_0.90115',\n              '../input/alldata-ema-resnext101-32x4d-swsl-adam-02-05-17-28/resnext101_32x4d_swsl_fold_4_14_0.89530',\n             \n              #'../input/fork-of-cassava-classification-fastai/models/best_model_512_lce_sgdr_0.pth',\n              #'../input/fork-of-cassava-classification-fastai/models/best_model_512_lce_sgdr_1.pth',\n              #'../input/fork-of-cassava-classification-fastai/models/best_model_512_lce_sgdr_2.pth',\n              #'../input/fork-of-cassava-classification-fastai/models/best_model_512_lce_sgdr_3.pth',\n              #'../input/fork-of-cassava-classification-fastai/models/best_model_512_lce_sgdr_4.pth',\n            \n             \n              \n             ]\nmodel_names = [  \n               # 'se_resnet101_ibn_a',\n               # 'se_resnet101_ibn_a',\n               # 'se_resnet101_ibn_a',\n               # 'se_resnet101_ibn_a',\n               # 'se_resnet101_ibn_a',\n                \n               'vit',\n               'vit',\n               'vit',\n               'vit',\n               'vit',\n              'resnext101_ibn_a', \n               'resnext101_ibn_a',\n              'resnext101_ibn_a',\n              'resnext101_ibn_a',\n              'resnext101_ibn_a',\n               'resnext101',\n               'resnext101',\n               'resnext101',\n               'resnext101',\n               'resnext101',\n               #'b4',\n               #'b4',\n               #'b4',\n               #'b4',\n               #'b4',\n              \n]\nassert len(model_names) == len(ckpt_paths)\n\nprint(len(ckpt_paths))\n\nckpt_weights =  [1 for i in range(len(ckpt_paths))]\nprint(ckpt_paths)\n\nmodel_wts={}\n#wts={'w0': 0.6465801475470692, 'w1': 0.6582084173812914, 'w2': 0.16946139705016672}\n\n\n#model_wts['b4']=0.18833718\nmodel_wts['vit']=0.6582084173812914\nmodel_wts['resnext101']=0.16946139705016672\nmodel_wts['resnext101_ibn_a']=0.6465801475470692\n#[0.27363746 0.10715301 0.42271631 0.18833718]\n# ckpt_paths = [\n#              '../input/soft-label-se50-01-25-18-37-8949/se50_fold_0_16_0.89042',\n#              '../input/soft-label-se50-01-25-18-37-8949/se50_fold_0_19_0.89276',\n#              '../input/soft-label-se50-01-25-18-37-8949/se50_fold_0_29_0.89322',\n#              '../input/soft-label-se50-01-25-18-37-8949/se50_fold_0_18_0.89486']\n# ckpt_weights =  [1,1,1,1]\n\n\n# # version 5 tta=4\n# ckpt_paths = ['../input/resnext101-ibn-a-01-22-10-05-bit-ls015-896/resnext101_ibn_a_fold_0_6_0.88435',\n#              '../input/resnext101-ibn-a-01-22-10-05-bit-ls015-896/resnext101_ibn_a_fold_0_7_0.89299',\n#              '../input/resnext101-ibn-a-01-22-10-05-bit-ls015-896/resnext101_ibn_a_fold_0_8_0.88995',\n#              '../input/resnext101-ibn-a-01-22-10-05-bit-ls015-896/resnext101_ibn_a_fold_0_9_0.89603']\n# ckpt_weights =  [0.2,0.25,0.2,0.35]\n\n# # version 6\n# ckpt_paths = ['../input/resnext101-ibn-a-01-22-01-13-bit-ls035-896/resnext101_ibn_a_fold_0_6_0.88972',\n#              '../input/resnext101-ibn-a-01-22-01-13-bit-ls035-896/resnext101_ibn_a_fold_0_7_0.89393',\n#              '../input/resnext101-ibn-a-01-22-01-13-bit-ls035-896/resnext101_ibn_a_fold_0_8_0.89136',\n#              '../input/resnext101-ibn-a-01-22-01-13-bit-ls035-896/resnext101_ibn_a_fold_0_9_0.89650']\n\n# ckpt_weights =  [1,1,1,1]\n\n# #version 8\n# ckpt_paths = ['../input/resnext101-ibn-a-01-22-01-13-bit-ls035-896/resnext101_ibn_a_fold_0_6_0.88972',\n#              '../input/resnext101-ibn-a-01-22-01-13-bit-ls035-896/resnext101_ibn_a_fold_0_7_0.89393',\n#              '../input/resnext101-ibn-a-01-22-01-13-bit-ls035-896/resnext101_ibn_a_fold_0_8_0.89136',\n#              '../input/resnext101-ibn-a-01-22-01-13-bit-ls035-896/resnext101_ibn_a_fold_0_9_0.89650']\n\n# ckpt_weights =  [0.2, 0.25, 0.2, 0.35]\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(ckpt_weights)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> We could do stratified validation split in each fold to make each fold's train and validation set looks like the whole train set in target distributions."},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(\n        self, df, data_root, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        \n        img  = get_img(path)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define Train\\Validation Image Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n\n    return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\ndef get_inference_transforms():\n    return Compose([\n        RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n        Transpose(p=0.5),\n        HorizontalFlip(p=0.5),\n        VerticalFlip(p=0.5),\n        HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.2),\n        RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        ToTensorV2(p=1.0),\n    ], p=1.)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from ibnnet import *\nclass IBNResnextCassava(nn.Module):\n    def __init__(self, arch='resnext101_ibn_a', n_class=5, pre=False):\n        super().__init__()\n        #m = torch.hub.load('XingangPan/IBN-Net', arch, pretrained=pre) #'resnext101_ibn_a'\n        m = resnext101_ibn_a()\n        self.enc = nn.Sequential(*list(m.children())[:-2])\n        nc = list(m.children())[-1].in_features\n        self.head = nn.Sequential(\n            nn.AdaptiveAvgPool2d(1),\n            nn.Flatten(),\n            nn.Linear(2048, n_class)\n        )\n\n    def forward(self,  x):\n        x = self.enc(x)\n        x = self.head(x)\n\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MishFunction(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, x):\n        ctx.save_for_backward(x)\n        return x * torch.tanh(F.softplus(x))   # x * tanh(ln(1 + exp(x)))\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        x = ctx.saved_variables[0]\n        sigmoid = torch.sigmoid(x)\n        tanh_sp = torch.tanh(F.softplus(x)) \n        return grad_output * (tanh_sp + x * sigmoid * (1 - tanh_sp * tanh_sp))\n\nclass Mish(nn.Module):\n    def forward(self, x):\n        return MishFunction.apply(x)\n\ndef to_Mish(model):\n    for child_name, child in model.named_children():\n        if isinstance(child, nn.ReLU):\n            setattr(model, child_name, Mish())\n        else:\n            to_Mish(child)\n            \ndef _resnext(url, block, layers, pretrained, progress, **kwargs):\n    model = ResNet(block, layers, **kwargs)\n#     state_dict = load_state_dict_from_url(url, progress=progress)\n#     model.load_state_dict(state_dict)\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision.models.resnet import ResNet, Bottleneck\n\n\nsemi_weakly_supervised_model_urls = {\n    'resnet18':         'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnet18-118f1556.pth',\n    'resnet50':         'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnet50-16a12f1b.pth',    \n    'resnext50_32x4d':  'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnext50_32x4-72679e44.pth',\n    'resnext101_32x4d': 'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnext101_32x4-3f87e46b.pth',\n    'resnext101_32x8d': 'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnext101_32x8-b4712904.pth',\n    'resnext101_32x16d':'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnext101_32x16-f3559a9c.pth',\n}\n\n\nclass ResnextCassava(nn.Module):\n    def __init__(self, arch='resnext101_32x4d', n_class=5, pretrained=False):\n        super().__init__()\n\n        m = _resnext(semi_weakly_supervised_model_urls[arch], Bottleneck, [3, 4, 23, 3], False, \n                progress=False,groups=32,width_per_group=4)\n        \n        self.enc = nn.Sequential(*list(m.children())[:-2])       \n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.max_pool = nn.AdaptiveMaxPool2d(1)\n        self.logit = nn.Linear(2048,n_class)\n\n        \n    def forward(self,  x):\n        x = self.enc(x)\n        x = (self.avg_pool(x) + self.max_pool(x)).squeeze(-1).squeeze(-1)\n        x = self.logit(x)\n        return x\n\n\n\n# class ResnextCassava(nn.Module):\n#     def __init__(self, arch='resnext50_32x4d', n_class=5, pretrained=False):\n#         super().__init__()\n#         m = torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models', 'resnext101_32x4d_swsl')\n\n#         self.enc = nn.Sequential(*list(m.children())[:-2])       \n#         nc = list(m.children())[-1].in_features\n#         self.avg_pool = nn.AdaptiveAvgPool2d(1)\n#         self.max_pool = nn.AdaptiveMaxPool2d(1)\n        \n#         self.logit = nn.Linear(2048,n_class)\n#     def forward(self,  x):\n#         x = self.enc(x)\n#         #print('enc:', x.shape)\n#         x = (self.avg_pool(x) + self.max_pool(x)).squeeze(-1).squeeze(-1)\n#         x = self.logit(x)\n#         return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class timm_class(nn.Module):\n    def __init__(self, model, n=5 ):\n        super().__init__()\n        #m = torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models', arch,pre)\n        #m = _resnext(semi_supervised_model_urls[arch], Bottleneck, [3, 4, 6, 3], False, \n        #        progress=False,groups=32,width_per_group=4)\n        #self.enc = nn.Sequential(*list(m.children())[:-2])       \n        \n        self.effnet =  nn.Sequential(*list(model.children())[:-2])      \n        #nc = list(self.effnet.children())[-2].in_features\n        nc = self.effnet[-3].out_channels\n        #print(nc)\n        self.dropout = nn.Dropout(0.2)\n        #self.out = nn.Linear(1536, num_classes)\n        #self.arcmargin=ArcMarginProduct(in_features=512,out_features=n).to(device)\n        self.head = nn.Sequential(AdaptiveConcatPool2d(),Flatten(),nn.GroupNorm(32,2*nc)\n                                  ,nn.Linear(2*nc,512),\n                            Mish(),nn.GroupNorm(16,512), nn.Dropout(0.4), nn.Linear(512,n))\n        \n        #self.head = nn.Sequential(AdaptiveConcatPool2d(),Flatten(),\n        #                          nn.BatchNorm1d(2*nc) ,nn.Linear(2*nc,512),\n        #                    Mish(),nn.BatchNorm1d(512), nn.Dropout(0.4),nn.Linear(512,n))\n        \n        \n        \n        #layers=[AdaptiveConcatPool2d(),Flatten()]+bn_drop_lin(nc*2, 512, True, 0.5, Mish()) + \\\n        #    bn_drop_lin(512, n, True, 0.3) #nn.BatchNorm1d(512)\n        #self.head = nn.Sequential(*layers)\n        \n    def forward(self,  x):\n        #print(len(x))\n        '''\n        shape = x[0].shape\n        n = len(x)\n        x = torch.stack(x,1).view(-1,shape[1],shape[2],shape[3]).cpu()\n        # filter white features\n        #x: bs*N x 3 x 128 x 128\n        #l=[ get_emptiness(empty)  for empty in  x ]\n        #print(l)\n        #print(x[0])\n        #print(x.size())\n        #x1=[empty  for empty in  x if get_emptiness(empty)<0.80]\n        print(len(x))\n        #n=len(x)\n        #x = self.enc(torch.stack(x))\n        '''\n        #x = self.effnet.extract_features(x)\n        #print('shape',x.shape)\n        x=self.effnet(x)\n        \n        #x = self.enc(x)\n        #x: bs*N x C x 4 x 4\n        shape = x.shape\n        #print('shape',shape)\n        #concatenate the output for tiles into a single map\n        #x = x.view(-1,n,shape[1],shape[2],shape[3]).permute(0,2,1,3,4).contiguous()\\\n        #  .view(-1,shape[1],shape[2]*n,shape[3])\n        #x: bs x C x N*4 x 4\n        #print(x.size())\n        #print('x',x.size())\n        x = self.head(x)\n        #print('out',x.size())\n        #x: bs x n\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from timm import create_model\nclass CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        \n        self.model = create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        '''\n        self.model.classifier = nn.Sequential(\n            nn.Dropout(0.3),\n            #nn.Linear(n_features, hidden_size,bias=True), nn.ELU(),\n            nn.Linear(n_features, n_class, bias=True)\n        )\n        '''\n    def forward(self, x):\n        x = self.model(x)\n        return x\n\nclass IBN_CUSTOM1(nn.Module):\n    def __init__(self, arch='resnext101_ibn_a', n_class=5, pretrained=True):\n        super().__init__()\n        m = se_resnet101_ibn_a(pretrained=pretrained)\n        self.enc = nn.Sequential(*list(m.children())[:-2])\n        self.head = nn.Sequential(\n            nn.AdaptiveAvgPool2d(1),\n            nn.Flatten(),\n            nn.Linear(2048, n_class)\n        )\n\n    def forward(self,  x):\n        x = self.enc(x)\n        x = self.head(x)\n\n        return x\n    \n\ndef get_model(model_name, device='cuda'):\n    if 'efficientnet' in model_name:\n#         from timm import create_model\n#         #model = CassvaImgClassifier(model_name, train.label.nunique(), pretrained=False)\n#         model = create_model(\n#             model_name,\n#             num_classes=5) #bs =16\n        model = CassvaImgClassifier(model_name, 5, pretrained=False).to(device)\n\n    elif model_name == 'ibn50a':\n        from cassnet import resnet50_ibn_a\n        model = resnet50_ibn_a(pretrained=False).to(device)\n        model.avgpool = nn.AdaptiveAvgPool2d(1)\n        model.fc = nn.Linear(2048, train.label.nunique())\n    elif model_name == 'ibnse50a':\n        from cassnet import se_resnet50_ibn_a\n        model = se_resnet50_ibn_a(pretrained=False).to(device)\n        model.avgpool = nn.AdaptiveAvgPool2d(1)\n        model.fc = nn.Linear(2048, train.label.nunique())\n    \n    elif model_name == 'resnext101_ibn_a':\n        model = IBNResnextCassava(arch=model_name)\n    elif model_name == 'se50_softlabel':\n        from net_soft_label import se_resnext50_32x4d\n        model = se_resnext50_32x4d(pretrained=None).to(device)\n    elif model_name == 'resnext101':\n        model = ResnextCassava(pretrained=False).to(device)\n    elif model_name == 'b4':\n        b4_model = timm.create_model('tf_efficientnet_b4_ns', pretrained=False)\n        \n        model=timm_class(b4_model,n=5).to(device)    \n    elif model_name == 'vit':\n        from vitnet import VisionTransformer\n        import vitnet.configs as configs\n        CONFIGS = {\n            'ViT-B_16': configs.get_b16_config(),\n            'ViT-B_32': configs.get_b32_config(),\n            'ViT-L_16': configs.get_l16_config(),\n            'ViT-L_32': configs.get_l32_config(),\n            'ViT-H_14': configs.get_h14_config(),\n            'testing': configs.get_testing(),\n        }\n        config = CONFIGS[\"ViT-B_16\"]\n        model = VisionTransformer(config, CFG['img_size'], zero_head=True, num_classes=5).to(device)\n    \n    elif model_name == 'se_resnet101_ibn_a':\n        model = IBN_CUSTOM1(pretrained=False).to(device) #bs=18\n\n        \n    \n    \n    model = model.to(device)\n    \n    #model = nn.DataParallel(model.to(device))\n    return model\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Main Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install ../input/mlcollection/ml_collections-0.1.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AdaptiveConcatPool2d(nn.Module):\n    \n    \"Layer that concats `AdaptiveAvgPool2d` and `AdaptiveMaxPool2d`\"\n    def __init__(self, size=None):\n        super().__init__()\n        self.size = size or 1\n        self.ap = nn.AdaptiveAvgPool2d(self.size)\n        self.mp = nn.AdaptiveMaxPool2d(self.size)\n    def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        image_preds = model(imgs)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    #image_preds_min=np.min(image_preds_all,axis=-1)\n    #image_preds_max=np.max(image_preds_all,axis=-1)\n    #image_preds_norm=(image_preds_all-image_preds_min[:,np.newaxis])/ (image_preds_max[:,np.newaxis]-image_preds_min[:,np.newaxis])\n \n    return image_preds_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n     # for training only, need nightly build pytorch\n    VALID = False#False\n    test_num = len(os.listdir('../input/cassava-leaf-disease-classification/test_images'))    \n    print('test_num:', test_num)\n\n    seed_everything(CFG['seed'])\n    \n    #folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    folds = StratifiedKFold(n_splits=CFG['fold_num'], shuffle=True, random_state=CFG['seed']).split(np.arange(train.shape[0]), train.label.values)\n\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        # we'll train fold 0 first\n        if fold > 0:\n            break \n\n        print('Inference fold {} started'.format(fold))\n\n        valid_ = train.loc[val_idx,:].reset_index(drop=True)\n        valid_ds = CassavaDataset(valid_, '../input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms(), output_label=False)\n        \n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms(), output_label=False)\n        \n        val_loader = torch.utils.data.DataLoader(\n            valid_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n        \n        tst_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n\n        device = torch.device(CFG['device'])\n\n        val_preds = []\n        tst_preds = []\n        \n        for i in range(len(ckpt_paths)):   \n            ckpt_path = ckpt_paths[i]\n            model = get_model(model_names[i], device)\n            \n            if model_names[i] == 'se_resnet101_ibn_a':\n                model.load_state_dict(torch.load(ckpt_path)['state_dict'])\n            else:\n                model.load_state_dict(torch.load(ckpt_path,map_location=device))\n            \n            with torch.no_grad():\n                for _ in range(CFG['tta']):\n                    if VALID:\n                        if test_num == 1:\n                            val_preds += [ckpt_weights[i]/sum(ckpt_weights)/CFG['tta']*inference_one_epoch(model, val_loader, device)]\n                    \n                    #tst_preds += [ckpt_weights[i]/sum(ckpt_weights)/CFG['tta']*\n                    #              inference_one_epoch(model, tst_loader, device)] model_wts[model_names[i]]\n                    tst_preds += [ \n                                  inference_one_epoch(model, tst_loader, device)*model_wts[model_names[i]]]\n                    #print(len(tst_preds))\n            del model\n            torch.cuda.empty_cache()\n        if VALID:\n            if test_num == 1:\n                val_preds = np.mean(val_preds, axis=0) \n        tst_preds = np.mean(tst_preds, axis=0) *3  #multiply by number of models if weighted\n        \n        if VALID:\n            if test_num == 1:\n                print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))\n            print('fold {} validation accuracy = {:.5f}'.format(fold, (valid_.label.values==np.argmax(val_preds, axis=1)).mean()))\n\n\n            from sklearn.metrics import confusion_matrix\n            cm = confusion_matrix(valid_.label.values, np.argmax(val_preds, axis=1), labels=[0, 1, 2, 3, 4])\n            print('cm')\n            print(cm)\n            import seaborn as sns\n            sns.set()\n            f, ax=plt.subplots()\n            sns.heatmap(cm,annot=True,ax=ax) #画热力图\n\n            ax.set_title('confusion matrix') #标题\n            ax.set_xlabel('predict') #x轴\n            ax.set_ylabel('true') #y轴\n\n#         del model\n#         torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tst_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['label'] = np.argmax(tst_preds, axis=1)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train part is here: https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-train-amp-aug"},{"metadata":{"trusted":true},"cell_type":"code","source":"len(tst_preds)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}