{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Library"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nimport timm\n\nimport random\nimport os\nimport torch\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\nimport torch.nn as nn\nimport torchvision\nfrom torchvision import models as tvmodels\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, utils\nfrom tqdm import tqdm\nimport torch.nn.functional as F\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import KFold\nimport time\n\nimport math\nfrom matplotlib.pyplot import imread\nimport albumentations as A\nfrom albumentations import Compose\nfrom albumentations.pytorch import ToTensorV2\nimport numpy as np\nimport cv2\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\n\nimport time\n# Ignore warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nstart_time = time.time()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"#for efficientnet\nBATCH_SIZE = 1\nimage_size = 512\nenet_type = ['tf_efficientnet_b4_ns'] * 5\nmodel_path = ['../input/moa-b4-baseline/baseline_cld_fold0_epoch8_tf_efficientnet_b4_ns_512.pth', \n              '../input/moa-b4-baseline/baseline_cld_fold1_epoch9_tf_efficientnet_b4_ns_512.pth', \n              '../input/moa-b4-baseline/baseline_cld_fold2_epoch9_tf_efficientnet_b4_ns_512.pth',\n              '../input/moa-b4-baseline/baseline_cld_fold3_epoch5_tf_efficientnet_b4_ns_512.pth',\n              '../input/moa-b4-baseline/baseline_cld_fold4_epoch11_tf_efficientnet_b4_ns_512.pth']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Config\n# ====================================================\nDATA_PATH = '../input/cassava-leaf-disease-classification/'\nTRAIN_DIR = DATA_PATH + 'train_images/'\nTEST_DIR = DATA_PATH + 'test_images/'\nMODEL_PATH = '../input/cassavanet-baseline-models/'\n\nN_TTA = 8\n\nHEIGHT = 512\nWIDTH = 512\nCHANNELS = 3\n\nN_CLASSES = 5\n\nMODEL_LIST = [0,1,2,3,4,5]\n\nIMG_MEAN = [0.485, 0.456, 0.406] #Mean for normalization Transform cassava = [0.4303, 0.4967, 0.3134] imgnet = [0.485, 0.456, 0.406]\nIMG_STD = [0.229, 0.224, 0.225] #STD for normalization Transform cassava = [0.2142, 0.2191, 0.1954] imgnet = [0.229, 0.224, 0.225]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Seed"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Seed\n# ====================================================\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nSEED = 1234\nseed_everything(SEED)  \nDEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Model\n# ====================================================\nclass CassavaNet(nn.Module):\n    def __init__(self, model_name=None, pretrained=False):\n        super().__init__()\n        self.model_name = model_name\n        if model_name == 'deit_base_patch16_224' or model_name == 'deit_base_patch16_384':\n            self.model = torch.hub.load('facebookresearch/deit:main', model_name, pretrained=pretrained)\n        else:\n            self.model = timm.create_model(model_name, pretrained=pretrained)\n        if 'efficientnet' in model_name:\n            self.n_features = self.model.classifier.in_features\n            self.model.classifier = nn.Linear(self.n_features, N_CLASSES)\n        elif model_name == 'vit_large_patch16_384' or model_name == 'deit_base_patch16_224' or model_name == 'deit_base_patch16_384':\n            self.n_features = self.model.head.in_features\n            self.model.head = nn.Linear(self.n_features, N_CLASSES)\n        elif 'resnext' in model_name:\n            self.n_features = self.model.fc.in_features\n            self.model.fc = nn.Linear(self.n_features, N_CLASSES)\n        \n    def forward(self, x):\n        return self.model(x)\n    \n    def freeze(self):\n        # To freeze the residual layers\n        for param in self.model.parameters():\n            param.requires_grad = False\n            \n        if 'efficientnet' in self.model_name:\n            for param in self.model.classifier.parameters():\n                param.requires_grad = True\n        elif self.model_name == 'vit_large_patch16_384' or 'deit_base_patch16_224':\n            for param in self.model.head.parameters():\n                param.requires_grad = True\n        elif 'resnext' in self.model_name:\n            for param in self.model.fc.parameters():\n                param.requires_grad = True\n            \n    def unfreeze(self):\n        # Unfreeze all layers\n        for param in self.model.parameters():\n            param.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\nclass GetData(Dataset):\n    def __init__(self, Dir, FNames, labels,Type):\n        self.dir = Dir\n        self.fnames = FNames\n        self.lbs = labels\n        self.type = Type\n        \n    def __len__(self):\n        return len(self.fnames)\n\n    def __getitem__(self, index):\n        x = imread(os.path.join(self.dir, self.fnames[index]))\n        if \"train\" in self.type:\n            aug_data = train_transforms(image = x)\n            return aug_data['image'], self.lbs[index]            \n        elif \"valid\" in self.type:\n            aug_data = valid_transforms(image = x)\n            return aug_data['image'], self.lbs[index]\n        elif \"tr-tst\" in self.type:\n            return x, self.lbs[index]\n        elif \"test\" in self.type:\n            return x, self.fnames[index]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Augmentation\n# ====================================================\nAug_Norm = A.Normalize(mean=IMG_MEAN, std=IMG_STD, max_pixel_value=255.0, p=1.0)\ntest_aug = Compose([\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.ShiftScaleRotate(p = 1.0),\n            A.ColorJitter(brightness=0.1, contrast=0.2, saturation=0.2, hue=0.00, always_apply=False, p=1.0),\n            A.RandomCrop(height= HEIGHT, width = WIDTH,always_apply=True, p=1.0),\n            Aug_Norm,\n            ToTensorV2(p=1.0)\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir(MODEL_PATH)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Model Loading\n# ====================================================\nmodels = []\ncount = 0\nfor model_fpath in os.listdir(MODEL_PATH):\n    if count in MODEL_LIST:\n        print(\"Model Loaded:\",model_fpath)\n        model_name_split = model_fpath.split('_f')[0]\n        model = CassavaNet(model_name_split,pretrained = False)\n        info = torch.load(MODEL_PATH + model_fpath,map_location = torch.device(DEVICE))\n        model.load_state_dict(info)\n        models.append(model)\n    count+=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#This sets up the submission dataframe\nsubmission = pd.DataFrame()\nlist_files = os.listdir(TEST_DIR)\nsubmission['image_id'] = pd.Series(list_files)\nsubmission['label'] = 0\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TTA"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# TTA\n# ====================================================\n\nBATCH_SIZE = 1\ntest_set = GetData(TEST_DIR,submission['image_id'], submission['label'], Type = 'test')\ntest_loader = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=False, num_workers=8,pin_memory = True)\nwith torch.no_grad():\n    for i, (images,labels) in enumerate(test_loader):\n        voting = np.zeros((len(models),N_TTA,N_CLASSES))\n        aug_images = np.zeros((N_TTA,CHANNELS,HEIGHT,WIDTH))\n        for aug_no in range(N_TTA):\n            img_np = images.numpy()\n\n            aug_data = test_aug(image = np.reshape(img_np,(600,800,CHANNELS)))\n            aug_images[aug_no,:,:,:] = aug_data['image'].numpy()\n        aug_images = torch.from_numpy(aug_images).to(torch.float32).to(DEVICE)\n        for model_no in range(len(models)):\n            model = models[model_no]\n            model = model.to(DEVICE)\n            model.eval()            \n\n            logits = model(aug_images)\n            voting[model_no,:,:] = F.softmax(logits).cpu().numpy()\n\n        voting = np.sum(voting,axis = 1) / N_TTA\n        voting = np.sum(voting,axis = 0) / len(models)\n\n        \n\n\nprint(voting)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\nOUTPUT_DIR = './'\nMODEL_DIR = '../input/cassavaresnext/'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n    \nTRAIN_PATH = '../input/cassava-leaf-disease-classification/train_images'\nTEST_PATH = '../input/cassava-leaf-disease-classification/test_images'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations\nimport albumentations\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nfrom scipy.special import softmax\n\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\n\nfrom PIL import Image\n\n\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\n\nimport warnings \nwarnings.filterwarnings('ignore')\ntransforms_valid = albumentations.Compose([\n    albumentations.CenterCrop(image_size, image_size, p=1),\n    albumentations.Resize(image_size, image_size),\n    albumentations.Normalize()\n])\nclass CFG:\n    debug=False\n    num_workers=8\n    model_name='resnext50_32x4d'\n    size=512\n    batch_size=32\n    seed=2020\n    target_size=5\n    target_col='label'\n    n_fold=5\n    trn_fold=[0, 1, 2, 3, 4]\n    inference=True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n\ndef init_logger(log_file=OUTPUT_DIR+'inference.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\n#LOGGER = init_logger()\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CLDDataset(Dataset):\n    def __init__(self, df, mode, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.mode = mode\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        row = self.df.loc[index]\n        image = cv2.imread(row.filepath)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if self.transform is not None:\n            res = self.transform(image=image)\n            image = res['image']\n        \n        image = image.astype(np.float32)\n        image = image.transpose(2,0,1)\n        if self.mode == 'test':\n            return torch.tensor(image).float()\n        else:\n            return torch.tensor(image).float(), torch.tensor(row.label).float()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\ntest['filepath'] = test.image_id.apply(lambda x: os.path.join('../input/cassava-leaf-disease-classification/test_images', f'{x}'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset_efficient = CLDDataset(test, 'test', transform=transforms_valid)\ntest_loader_efficient = torch.utils.data.DataLoader(test_dataset_efficient, batch_size=BATCH_SIZE, shuffle=False,  num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_transforms(*, data):\n    if data == 'valid':\n        return A.Compose([\n            A.Resize(CFG.size, CFG.size),\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomResNext(nn.Module):\n    def __init__(self, model_name='resnext50_32x4d', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class enet_v2(nn.Module):\n\n    def __init__(self, backbone, out_dim, pretrained=False):\n        super(enet_v2, self).__init__()\n        self.enet = timm.create_model(backbone, pretrained=pretrained)\n        in_ch = self.enet.classifier.in_features\n        self.myfc = nn.Linear(in_ch, out_dim)\n        self.enet.classifier = nn.Identity()\n\n    def forward(self, x):\n        x = self.enet(x)\n        x = self.myfc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_state(model_path):\n    model = CustomResNext(CFG.model_name, pretrained=False)\n    try:  # single GPU model_file\n        model.load_state_dict(torch.load(model_path)['model'], strict=True)\n        state_dict = torch.load(model_path)['model']\n    except:  # multi GPU model_file\n        state_dict = torch.load(model_path)['model']\n        state_dict = {k[7:] if k.startswith('module.') else k: state_dict[k] for k in state_dict.keys()}\n\n    return state_dict\n\n\ndef inference(model, states, test_loader, device):\n    model.to(device)\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state)\n            model.eval()\n            with torch.no_grad():\n                y_preds = model(images)\n            avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_func(test_loader):\n    model.eval()\n    bar = tqdm(test_loader)\n\n    LOGITS = []\n    PREDS = []\n    \n    with torch.no_grad():\n        for batch_idx, images in enumerate(bar):\n            x = images.to(device)\n            logits = model(x)\n            LOGITS.append(logits.cpu())\n            PREDS += [torch.softmax(logits, 1).detach().cpu()]\n        PREDS = torch.cat(PREDS).cpu().numpy()\n        LOGITS = torch.cat(LOGITS).cpu().numpy()\n    return PREDS\n\ndef tta_inference_func(test_loader):\n    model.eval()\n    bar = tqdm(test_loader)\n    PREDS = []\n    LOGITS = []\n\n    with torch.no_grad():\n        for batch_idx, images in enumerate(bar):\n            x = images.to(device)\n            x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),\n            x.transpose(-1,-2),x.transpose(-1,-2).flip(-1),\n            x.transpose(-1,-2).flip(-2),x.transpose(-1,-2).flip(-1,-2)],0)\n            x = x.view(-1, 3, image_size, image_size)\n            logits = model(x)\n            logits = logits.view(BATCH_SIZE, 8, -1).mean(1)\n            PREDS += [torch.softmax(logits, 1).detach().cpu()]\n            LOGITS.append(logits.cpu())\n\n        PREDS = torch.cat(PREDS).cpu().numpy()\n        \n    return PREDS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel = CustomResNext(CFG.model_name, pretrained=False)\n#model = enet_v2(enet_type[i], out_dim=5)\nstates = [load_state(MODEL_DIR+f'{CFG.model_name}_fold{fold}.pth') for fold in CFG.trn_fold]\ntest_dataset = TestDataset(test, transform=get_transforms(data='valid'))\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, \n                         num_workers=CFG.num_workers, pin_memory=True)\npredictions = inference(model, states, test_loader, device)\n\n#for Efficientnet\ntest_preds = []\nfor i in range(len(enet_type)):\n    model = enet_v2(enet_type[i], out_dim=5)\n    model = model.to(device)\n    model.load_state_dict(torch.load(model_path[i]))\n    test_preds += [tta_inference_func(test_loader_efficient)]\n\n# submission\nprint(type(predictions))\nprint(type(test_preds))\nprint(voting)\npred =np.mean(test_preds,axis=0)*0.333 +voting*0.3333+predictions*0.3333\nprint(pred)\ntest['label'] = softmax(pred).argmax(1)\ntest[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)\n\nprint(time.time()-start_time)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}