{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"package_path = '../input/pytorch-image-models/pytorch-image-models-master'\n\nimport sys\nsys.path.append(package_path)\n\nDATA_DIR = '../input/cassava-leaf-disease-classification'\n\nMODEL_DIR_0 = '../input/gpu-vit-noisearch-amp-aug-fold-0'\nMODEL_DIR_1 = '../input/gpu-vit-noisearch-amp-aug-fold-1'\nMODEL_DIR_2 = '../input/gpu-vit-noisearch-amp-aug-fold-2'\nMODEL_DIR_3 = '../input/gpu-vit-noisearch-amp-aug-fold-3'\nMODEL_DIR_4 = '../input/gpu-vit-noisearch-amp-aug-fold-4'\n\nMODEL_DIR_01 = '../input/gpu-en-b4-ns-noisearch-amp-aug-fold-0'\nMODEL_DIR_11 = '../input/gpu-en-b4-ns-noisearch-amp-aug-fold-1'\nMODEL_DIR_21 = '../input/gpu-en-b4-ns-noisearch-amp-aug-fold-2'\nMODEL_DIR_31 = '../input/gpu-en-b4-ns-noisearch-amp-aug-fold-3'\nMODEL_DIR_41 = '../input/gpu-en-b4-ns-noisearch-amp-aug-fold-4'\n\nMODEL_DIR_02 = '../input/gpu-seresnext-noisearch-amp-aug-fold-0'\nMODEL_DIR_12 = '../input/gpu-seresnext-noisearch-amp-aug-fold-1'\nMODEL_DIR_22 = '../input/gpu-seresnext-noisearch-amp-aug-fold-2'\nMODEL_DIR_32 = '../input/gpu-seresnext-noisearch-amp-aug-fold-3'\nMODEL_DIR_42 = '../input/gpu-seresnext-noisearch-amp-aug-fold-4'\n\nMODEL_DIR_03 = '../input/gpu-en-b4-ns-amp-aug-fold-0'\nMODEL_DIR_13 = '../input/gpu-en-b4-ns-amp-aug-fold-1'\nMODEL_DIR_23 = '../input/gpu-en-b4-ns-amp-aug-fold-2'\nMODEL_DIR_33 = '../input/gpu-en-b4-ns-amp-aug-fold-3'\nMODEL_DIR_43 = '../input/gpu-en-b4-ns-amp-aug-fold-4'\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from datetime import datetime\nimport os\nimport random\nimport time\nimport warnings\n\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport cv2\nimport matplotlib.pyplot as plt\nimport sklearn\nfrom sklearn.metrics import log_loss\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nfrom torch import nn\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nimport timm\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\nfrom albumentations.pytorch import ToTensorV2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 719,\n    'model_arch': 'vit_base_patch16_384',\n    'img_size': 384,\n    'epochs': 10,\n    'train_bs': 16,\n    'valid_bs': 16,\n    'lr': 1e-4,\n    'num_workers': 4,\n    'accum_iter': 1, # support to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'tta': 3,\n    'used_epochs': [7,8,9],\n    'weights': [1,1,1,1,1,1]\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS0 = {\n    0: [9,8,6,5],\n    1: [5,6,4,8],\n    2: [8,9,7,6],\n    3: [8,7,9,6],\n    4: [9,8,7,4]\n         }\n\nEPOCHS1 = {\n    0: [8,9,7,6],\n    1: [9,4,8,6],\n    2: [9,7,8,4],\n    3: [5,8,9,3],\n    4: [6,7,8,9]\n         }\n\nEPOCHS2 = {\n    0: [8,9,6,7],\n    1: [9,8,5,6],\n    2: [9,7,5,4],\n    3: [5,8,9,4],\n    4: [5,8,9,7]\n         }\n\nEPOCHS3 = {\n    0: [8,9,7,6],\n    1: [2,7,9,5],\n    2: [5,6,9,7],\n    3: [8,9,7,6],\n    4: [4,5,8,7]\n         }\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(f'{DATA_DIR}/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    return im_rgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, df, data_root, transforms=None, output_label=True):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        img  = get_img(f\"{self.data_root}/{self.df.loc[index]['image_id']}\")\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        # do label smoothing\n        if self.output_label:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_inference_transforms_384():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.0)\n\n\ndef get_inference_transforms():\n    return Compose([\n            RandomResizedCrop(512, 512),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassvaImgClassifierN(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.n_class = n_class\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        \n        if 'vit' in model_arch:\n            n_features = self.model.head.in_features\n            self.model.head = nn.Identity()\n        \n        if 'eff' in model_arch:\n            n_features = self.model.classifier.in_features\n            self.model.classifier = nn.Identity()\n            \n        if 'resnext' in model_arch:\n            n_features = self.model.fc.in_features\n            self.model.fc = nn.Identity()\n        \n        self.layer_noise = nn.Linear(n_features, n_class)\n        self.layer_pure = nn.Linear(n_features, n_class)\n        \n\n    def forward(self, x):\n        x = self.model(x)\n        x1 = self.layer_noise(x)\n        x2 = self.layer_pure(x)\n        x1 = x1 + x2\n        return x1, x2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        _, image_preds = model(imgs)\n        image_preds_all += [torch.sigmoid(image_preds).detach().cpu().numpy()]\n        \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_one_epochN(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)\n        image_preds_all += [torch.sigmoid(image_preds).detach().cpu().numpy()]\n        \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_inference(fold, MODEL_DIR, model_arch, epoch_list):\n    print(f'Inference fold {fold} started')\n\n\n    test = pd.DataFrame()\n    test['image_id'] = list(os.listdir(f'{DATA_DIR}/test_images/'))\n    \n    if 'vit' in model_arch:\n        T = get_inference_transforms_384()\n    else:\n        T = get_inference_transforms()\n    \n    test_ds = CassavaDataset(\n        test, f'{DATA_DIR}/test_images/',\n        transforms=T, output_label=False)\n\n\n    tst_loader = torch.utils.data.DataLoader(\n        test_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n\n    device = torch.device(CFG['device'])\n    model = CassvaImgClassifier(model_arch, train.label.nunique()).to(device)\n\n    tst_preds = []\n\n    for i, epoch in enumerate(epoch_list[fold][:1]):\n\n        model.load_state_dict(torch.load(f'{MODEL_DIR}/{model_arch}_fold_{fold}_{epoch}'))\n\n        with torch.no_grad():\n            for _ in range(CFG['tta']):\n                tst_image_preds = inference_one_epoch(model, tst_loader, device)\n                tst_preds += [(1/CFG['tta'])*tst_image_preds]\n\n    tst_preds = np.sum(tst_preds, axis=0) \n\n    del model\n    torch.cuda.empty_cache()\n    return tst_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_inferenceN(fold, MODEL_DIR, model_arch):\n    print(f'Inference fold {fold} started')\n    test = pd.DataFrame()\n    test['image_id'] = list(os.listdir(f'{DATA_DIR}/test_images/'))\n    test_ds = CassavaDataset(\n        test, f'{DATA_DIR}/test_images/',\n        transforms=get_inference_transforms(), output_label=False)\n\n\n    tst_loader = torch.utils.data.DataLoader(\n        test_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n\n    device = torch.device(CFG['device'])\n    model = CassvaImgClassifierN(model_arch, train.label.nunique()).to(device)\n\n    tst_preds = []\n\n    for i, epoch in enumerate(CFG['used_epochs']):\n\n        model.load_state_dict(\n            torch.load(f'{MODEL_DIR}/{model_arch}_fold_{fold}_{epoch}'))\n\n        with torch.no_grad():\n            for _ in range(CFG['tta']):\n                tst_image_preds = inference_one_epochN(model, tst_loader, device)\n                tst_preds += [(1/ CFG['tta'])*tst_image_preds]\n\n    tst_preds = np.mean(tst_preds, axis=0) \n\n    del model\n    torch.cuda.empty_cache()\n    return tst_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds0 = run_inference(0, MODEL_DIR_0, 'vit_base_patch16_384', EPOCHS0)\npreds1 = run_inference(1, MODEL_DIR_1, 'vit_base_patch16_384', EPOCHS0)\npreds2 = run_inference(2, MODEL_DIR_2, 'vit_base_patch16_384', EPOCHS0)\npreds3 = run_inference(3, MODEL_DIR_3, 'vit_base_patch16_384', EPOCHS0)\npreds4 = run_inference(4, MODEL_DIR_4, 'vit_base_patch16_384', EPOCHS0)\n\n# tst_preds = preds0\nPRED0 = (preds0 + preds1 + preds2 + preds3 + preds4)/5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds0 = run_inference(0, MODEL_DIR_01, 'tf_efficientnet_b4_ns', EPOCHS1)\npreds1 = run_inference(1, MODEL_DIR_11, 'tf_efficientnet_b4_ns', EPOCHS1)\npreds2 = run_inference(2, MODEL_DIR_21, 'tf_efficientnet_b4_ns', EPOCHS1)\npreds3 = run_inference(3, MODEL_DIR_31, 'tf_efficientnet_b4_ns', EPOCHS1)\npreds4 = run_inference(4, MODEL_DIR_41, 'tf_efficientnet_b4_ns', EPOCHS1)\n\n# tst_preds = preds0\nPRED1 = (preds0 + preds1 + preds2 + preds3 + preds4)/5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds0 = run_inference(0, MODEL_DIR_02, 'seresnext50_32x4d', EPOCHS2)\npreds1 = run_inference(1, MODEL_DIR_12, 'seresnext50_32x4d', EPOCHS2)\npreds2 = run_inference(2, MODEL_DIR_22, 'seresnext50_32x4d', EPOCHS2)\npreds3 = run_inference(3, MODEL_DIR_32, 'seresnext50_32x4d', EPOCHS2)\npreds4 = run_inference(4, MODEL_DIR_42, 'seresnext50_32x4d', EPOCHS2)\n\n# tst_preds = preds0\nPRED2 = (preds0 + preds1 + preds2 + preds3 + preds4)/5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds0 = run_inferenceN(0, MODEL_DIR_03, 'tf_efficientnet_b4_ns',)\npreds1 = run_inferenceN(1, MODEL_DIR_13, 'tf_efficientnet_b4_ns',)\npreds2 = run_inferenceN(2, MODEL_DIR_23, 'tf_efficientnet_b4_ns',)\npreds3 = run_inferenceN(3, MODEL_DIR_33, 'tf_efficientnet_b4_ns',)\npreds4 = run_inferenceN(4, MODEL_DIR_43, 'tf_efficientnet_b4_ns')\n\n# tst_preds = preds0\nPRED3 = (preds0 + preds1 + preds2 + preds3 + preds4)/5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tst_preds = (PRED0 + 2*PRED1 + PRED2 + PRED3)/5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.DataFrame()\ntest['image_id'] = list(os.listdir(f'{DATA_DIR}/test_images/'))\n\ntest['label'] = np.argmax(tst_preds, axis=1)\ntest.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}