{"cells":[{"metadata":{},"cell_type":"markdown","source":"<center><h1>Cassava Inference</h1></center>"},{"metadata":{},"cell_type":"markdown","source":"This notebook presents our inference code for the cassava leaf disease competition\n\nThe following types of models were ensembled for the submission:\n* Resnext 50\n* Efficientnet B4\n* Vision Transformer\n* Efficientnet B3"},{"metadata":{},"cell_type":"markdown","source":"## Dependencies"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --quiet ../input/cassava-models/inputs/keras-applications\n!pip install --quiet ../input/cassava-models/inputs/efficientnet_git","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\n\npackage_path = '../input/cassava-models/inputs/Vision Transformer Pytorch/VisionTransformer-Pytorch'\nsys.path.append(package_path)\nfrom vision_transformer_pytorch import VisionTransformer\n\npackage_path = '../input/cassava-models/inputs/pytorch image models/pytorch-image-models-master'\nsys.path.append(package_path)\nimport timm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import cv2\nfrom skimage import io\nimport time\nimport random\nfrom torchvision import transforms\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\n\nimport os, re, glob\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras import Model\nimport efficientnet.tfkeras as efn\n\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nimport torchvision.models as models\nfrom albumentations.pytorch import ToTensorV2\nimport albumentations\n\nfrom pathlib import Path\nfrom contextlib import contextmanager\n\nfrom scipy.special import softmax\nfrom sklearn.metrics import accuracy_score\nfrom tqdm.auto import tqdm\nfrom PIL import Image\n\n\nimport warnings \nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = 512\nCLASSES = 5\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Pytorch dataset\nclass PytorchCassavaDataset(Dataset):\n    def __init__(self, df = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv'),\n                 data_root = \"../input/cassava-leaf-disease-classification/test_images\", transforms=None):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        \n    def get_img(self, path):\n        im_bgr = cv2.imread(path)\n        im_rgb = cv2.cvtColor(im_bgr, cv2.COLOR_BGR2RGB)\n        return im_rgb\n\n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        img  = self.get_img(path)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        return img\n    \n    \n# Tensorflow dataset\nAUTO = tf.data.experimental.AUTOTUNE\nBATCH_SIZE = 32\n\ndef get_name(file_path):\n    parts = tf.strings.split(file_path, os.path.sep)\n    name = parts[-1]\n    return name\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    return image\n\ndef center_crop(image):\n    image = tf.reshape(image, [600, 800, 3]) # Original shape\n    \n    h, w = image.shape[0], image.shape[1]\n    if h > w:\n        image = tf.image.crop_to_bounding_box(image, (h - w) // 2, 0, w, w)\n    else:\n        image = tf.image.crop_to_bounding_box(image, 0, (w - h) // 2, h, h)\n        \n    image = tf.image.resize(image, [IMAGE_SIZE, IMAGE_SIZE]) # Expected shape\n    return image\n\ndef resize_image(image, label):\n    image = tf.image.resize(image, [IMAGE_SIZE, IMAGE_SIZE])\n    image = tf.reshape(image, [IMAGE_SIZE, IMAGE_SIZE, 3])\n    return image, label\n\ndef process_path(file_path):\n    name = get_name(file_path)\n    img = tf.io.read_file(file_path)\n    img = decode_image(img)\n    return img, name\n\ndef get_dataset(files_path, shuffled=False, tta=False, extension='jpg'):\n    dataset = tf.data.Dataset.list_files(f'{files_path}*{extension}', shuffle=shuffled)\n    dataset = dataset.map(process_path, num_parallel_calls=AUTO)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.map(resize_image, num_parallel_calls=AUTO)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Pytorch augmentations\nefficientnet_transforms = albumentations.Compose([\n    albumentations.CenterCrop(IMAGE_SIZE, IMAGE_SIZE, p=1),\n    albumentations.Resize(IMAGE_SIZE, IMAGE_SIZE),\n    albumentations.Normalize(),\n    ToTensorV2(p=1.0)\n])\n\nresnext_transforms = albumentations.Compose([\n    albumentations.Resize(IMAGE_SIZE, IMAGE_SIZE),\n    albumentations.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ToTensorV2(),\n])\n\nvitEfficientnet_transforms = albumentations.Compose([\n    albumentations.RandomResizedCrop(384, 384),\n    albumentations.Transpose(p=0.5),\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.VerticalFlip(p=0.5),\n    albumentations.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n    albumentations.RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n    albumentations.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n    ToTensorV2(p=1.0),\n], p=1.)\n\n\n# Tensorflow augmentations\ndef data_augment(image, label):\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n            \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > .75:\n        image = tf.image.transpose(image)\n        \n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n        \n    # Pixel-level transforms\n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n        \n    # Crops\n    if p_crop > .7:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.8)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.9)\n    elif p_crop > .4:\n        crop_size = tf.random.uniform([], int(IMAGE_SIZE*.8), IMAGE_SIZE, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, 3])\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Architectures"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Pytorch models\nclass TimmModel(nn.Module):\n    def __init__(self, model_name, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, CLASSES)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \nclass enet_v2(nn.Module):\n    def __init__(self, backbone, out_dim, pretrained=False):\n        super(enet_v2, self).__init__()\n        self.enet = timm.create_model(backbone, pretrained=pretrained)\n        in_ch = self.enet.classifier.in_features\n        self.myfc = nn.Linear(in_ch, out_dim)\n        self.enet.classifier = nn.Identity()\n\n    def forward(self, x):\n        x = self.enet(x)\n        x = self.myfc(x)\n        return x\n    \nclass CassavaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \nclass ViTEfficientnetClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model1 = VisionTransformer.from_name('ViT-B_16', num_classes=5) \n        self.model1.load_state_dict(torch.load('../input/cassava-models/inputs/ViT/ViT-B_16.pt'))\n        self.model2 = CassavaImgClassifier(model_arch, n_class, pretrained)\n        \n    def forward(self, x):\n        x1 = self.model1(x)\n        x2 = self.model2(x)\n        return 0.6 * x1 + 0.4 * x2\n    \n    def load(self, state_dict):\n        self.model2.load_state_dict(state_dict)\n    \n    \n# Tensorflow efficientnet\ndef model_fn(input_shape, n_class):\n    inputs = L.Input(shape=input_shape, name='input_image')\n    base_model = efn.EfficientNetB4(input_tensor=inputs, \n                                    include_top=False, \n                                    weights=None, \n                                    pooling='avg')\n    x = L.Dropout(.5)(base_model.output)\n    output = L.Dense(n_class, activation='softmax', name='output')(x)\n    model = Model(inputs=inputs, outputs=output)\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Resnext+EfficientnetB4"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResnextEfficientnet_CFG:\n    resnext_model='resnext50_32x4d'\n    efficientnet_model='tf_efficientnet_b4_ns'\n    num_workers=8\n    batch_size=32\n    trn_fold=[0, 1, 2, 3, 4]\n    efficientnet_model_path=glob.glob('../input/cassava-models/inputs/Efficientnet-B4-2/*')\n\n\ndef load_resnext_state(model_path):\n    model = TimmModel(ResnextEfficientnet_CFG.resnext_model, pretrained=False)\n    try:  # single GPU model_file\n        model.load_state_dict(torch.load(model_path)['model'], strict=True)\n        state_dict = torch.load(model_path)['model']\n    except:  # multi GPU model_file\n        state_dict = torch.load(model_path)['model']\n        state_dict = {k[7:] if k.startswith('module.') else k: state_dict[k] for k in state_dict.keys()}\n\n    return state_dict\n\n\ndef inference(model, states, test_loader, device):\n    model.to(device)\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state)\n            model.eval()\n            with torch.no_grad():\n                y_preds = model(images)\n            avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs\n\ndef tta_inference_func(test_loader):\n    model.eval()\n    bar = tqdm(test_loader)\n    PREDS = []\n    LOGITS = []\n\n    with torch.no_grad():\n        for batch_idx, images in enumerate(bar):\n            x = images.to(device)\n            x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),\n            x.transpose(-1,-2),x.transpose(-1,-2).flip(-1),\n            x.transpose(-1,-2).flip(-2),x.transpose(-1,-2).flip(-1,-2)],0)\n            x = x.view(-1, 3, IMAGE_SIZE, IMAGE_SIZE)\n            logits = model(x)\n            logits = logits.view(1, 8, -1).mean(1)\n            PREDS += [torch.softmax(logits, 1).detach().cpu()]\n            LOGITS.append(logits.cpu())\n\n        PREDS = torch.cat(PREDS).cpu().numpy()\n        \n    return PREDS\n\n# Create loaders\ntest_dataset_efficient = PytorchCassavaDataset(transforms=efficientnet_transforms)\ntest_loader_efficient = torch.utils.data.DataLoader(test_dataset_efficient, batch_size=1, shuffle=False,  num_workers=4)\ntest_dataset_resnext = PytorchCassavaDataset(transforms=resnext_transforms)\ntest_loader_resnext = DataLoader(test_dataset_resnext, batch_size=ResnextEfficientnet_CFG.batch_size, shuffle=False, num_workers=ResnextEfficientnet_CFG.num_workers, pin_memory=True)\n\n# Resnext predictions\nmodel = TimmModel(ResnextEfficientnet_CFG.resnext_model, pretrained=False)\nstates = [load_resnext_state(path) for path in glob.glob('../input/cassava-models/inputs/Resnext/*')]\nresnext_predictions = inference(model, states, test_loader_resnext, device)\n\n# Efficientnet predictions\ntest_preds = []\nfor i in range(len(ResnextEfficientnet_CFG.efficientnet_model_path)):\n    model = enet_v2(ResnextEfficientnet_CFG.efficientnet_model, out_dim=5)\n    model = model.to(device)\n    model.load_state_dict(torch.load(ResnextEfficientnet_CFG.efficientnet_model_path[i]))\n    test_preds += [tta_inference_func(test_loader_efficient)]\nefficientnet_predictions = np.mean(test_preds, axis=0)\n\n# Combine resnext and efficientnet predictions\ntest = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\npred = 0.5*resnext_predictions + 0.5*efficientnet_predictions\ntest['label'] = softmax(pred).tolist()\nresnext_b4_dict = dict(zip(list(test['image_id']), pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Vision Transformer+EfficientnetB3"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ViTEfficientnet_CFG:\n    efficientnet_model='tf_efficientnet_b3_ns'\n    img_size=384\n    valid_bs=32\n    num_workers=4\n    tta=1\n        \ndef inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)   #output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all\n\nseed_everything(719)\ntest = pd.DataFrame()\ntest['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\ntest_ds = PytorchCassavaDataset(df=test, data_root = \"../input/cassava-leaf-disease-classification/test_images\", transforms=vitEfficientnet_transforms)\n\ntst_loader = torch.utils.data.DataLoader(\n    test_ds, \n    batch_size=ViTEfficientnet_CFG.valid_bs,\n    num_workers=ViTEfficientnet_CFG.num_workers,\n    shuffle=False,\n    pin_memory=False,\n)\n\nmodel = ViTEfficientnetClassifier(ViTEfficientnet_CFG.efficientnet_model, CLASSES).to(device)\n\ntst_preds = []\nfor file in glob.glob('../input/cassava-models/inputs/Efficientnet-B3/*'):\n    model.load(file)\n    with torch.no_grad():\n        for _ in range(ViTEfficientnet_CFG.tta):\n            tst_preds += [inference_one_epoch(model, tst_loader, device)]\ntst_preds = np.mean(tst_preds, axis=0)\ntest[\"label\"] = list(tst_preds)\nvit_dict = dict(zip(list(test['image_id']), list(test['label'])))\n\ndel model\ntorch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## VisionTransformer+EfficientnetB3+EfficientnetB4"},{"metadata":{"trusted":true},"cell_type":"code","source":"TTA_STEPS = 3 # Do TTA if > 0\n\nfiles_path = '/kaggle/input/cassava-leaf-disease-classification/test_images/'\ntest_size = len(os.listdir(files_path))\ntest_preds = np.zeros((test_size, CLASSES))\n\n# Create efficientnet model\nmodel_path_list = glob.glob('../input/cassava-models/inputs/Efficientnet-B4/*.h5')\nmodel_path_list.sort()\nmodel = model_fn((None, None, 3), CLASSES)\n\n# Load model weights and make predictions\nfor model_path in model_path_list:\n    print(model_path)\n    K.clear_session()\n    model.load_weights(model_path)\n\n    if TTA_STEPS > 0:\n        test_ds = get_dataset(files_path, tta=True).repeat()\n        ct_steps = TTA_STEPS * ((test_size/BATCH_SIZE) + 1)\n        preds = model.predict(test_ds, steps=ct_steps, verbose=1)[:(test_size * TTA_STEPS)]\n        preds = np.mean(preds.reshape(test_size, TTA_STEPS, CLASSES, order='F'), axis=1)\n        test_preds += preds / len(model_path_list)\n    else:\n        test_ds = get_dataset(files_path, tta=False)\n        x_test = test_ds.map(lambda image, image_name: image)\n        test_preds += model.predict(x_test) / len(model_path_list)\n    \n# test_preds = np.argmax(test_preds, axis=-1)\ntest_names_ds = get_dataset(files_path)\nimage_names = [img_name.numpy().decode('utf-8') for img, img_name in iter(test_names_ds.unbatch())]\nb4_dict = dict(zip(image_names, test_preds))\n\nvit_b3_b4_dict = {image_name:0.5*b4_dict[image_name]+0.5*vit_dict[image_name] for image_name in vit_dict.keys()}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Combine all models"},{"metadata":{"trusted":true},"cell_type":"code","source":"final_dict = {image_name:np.argmax(resnext_b4_dict[image_name]+vit_b3_b4_dict[image_name], axis=-1) for image_name in vit_b3_b4_dict.keys()}\nsubmission_df = pd.DataFrame()\nsubmission_df['image_id'] = final_dict.keys()\nsubmission_df['label'] = final_dict.values()\nsubmission_df.sort_values(by=[\"image_id\"])\n\nsubmission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Sources\n* https://www.kaggle.com/luonganhtuan93/ensemble-resnext50-32x4d-efficientnet\n* https://www.kaggle.com/szuzhangzhi/vit-cuda-as-usual-ensemble-inference\n* https://www.kaggle.com/dimitreoliveira/cassava-leaf-disease-tpu-v2-pods-inference"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}