{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 1st Place Solution \"Cassava Leaf Disease Classification\"\n\nThis is the inference notebook of our final submission which scored ~91.3% on public and private leaderboard. We used an ensemble of four different models and stacked those models together using a mean approach.\n\nYou can find the according training code in these notebooks:\n\n* [EfficientNet B4 (TPU Training)](https://www.kaggle.com/jannish/cassava-leaf-disease-efficientnetb4-tpu)\n* [ResNext50_32x4d (GPU Training)](https://www.kaggle.com/hiarsl/cassava-leaf-disease-resnext50)\n* [ViT (TPU Training)](https://www.kaggle.com/sebastiangnther/cassava-leaf-disease-vit-tpu-training)\n\nIn order to find the final combination of all the models we tested, we iteratively tried different ensembles using this notebook:\n\n* [Ensembling by using OOF predictions](https://www.kaggle.com/jannish/cassava-leaf-disease-finding-final-ensembles)\n\nOur final submission first averaged the probabilities of the predicted classes of ViT and ResNext. This averaged probability vector was then merged with the predicted probabilities of EfficientnetB4 and MobileNet(CropNet) in a second stage. For this purpose, the values were simply summed up.\n\nFinally, we would like to thank all the Kagglers who posted their notebooks and gave valuable hints on which models to try!\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nimport os\nimport random\nimport json\nimport gc\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\n\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom sklearn.metrics import accuracy_score\nfrom functools import partial\nfrom albumentations import (Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, CenterCrop, \n                            HorizontalFlip, VerticalFlip, Rotate, ShiftScaleRotate, Transpose)\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nfrom tensorflow import keras\n\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as models\nfrom torch.utils.data import DataLoader, Dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = \"/kaggle/input/cassava-leaf-disease-classification/\"\nimage_path = path+\"test_images/\"\n\nIMAGE_SIZE = (512,512)\nsubmission_df = pd.DataFrame(columns={\"image_id\",\"label\"})\nsubmission_df[\"image_id\"] = os.listdir(image_path)\nsubmission_df[\"label\"] = 0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Used models in the final submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"# We used this flag to test combinations using only TF.Keras models\nonlykeras = False\n        \nused_models_pytorch = {\"vit2020\": [f'../input/cassava-leaf-disease-1st-place-models/vit/vit_base_patch16_384_fold_{fold}.h5' for fold in [0,1,2,3,4]],\n                       \"resnext\": [f'../input/cassava-leaf-disease-1st-place-models/resnext50_32x4d/resnext50_32x4d_fold{fold}_best.pth' for fold in [0,1,2,3,4]]}\n\nused_models_keras = {\"mobilenet\": \"../input/cassava-leaf-disease-1st-place-models/cropnet_mobilenetv3/cropnet\",\n                     \"efficientnetb4\": \"../input/cassava-leaf-disease-1st-place-models/efficientnetb4/efficientnetb4_all_e14.h5\"}\n\n# We used this flag for testing different ensembling approaches\nstacked_mean = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ResNext50_32x4d"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomResNext(nn.Module):\n        def __init__(self, model_name='resnext50_32x4d', pretrained=False):\n            super().__init__()\n            self.model = timm.create_model(model_name, pretrained=pretrained)\n            n_features = self.model.fc.in_features\n            self.model.fc = nn.Linear(n_features, 5)\n\n        def forward(self, x):\n            x = self.model(x)\n            return x\n\nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_path_id'].values\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        image = cv2.imread(file_name)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image\n\nif \"resnext\" in used_models_pytorch:\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    def get_transforms():\n        return Compose([Resize(512, 512),\n                        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n                        ToTensorV2()])\n\n    def inference(model, states, test_loader, device):\n        model.to(device)\n\n        probabilities = []\n        for i, (images) in enumerate(test_loader):\n            images = images.to(device)\n            avg_preds = []\n            for state in states:\n                model.load_state_dict(state['model'])\n                model.eval()\n                with torch.no_grad():\n                    y_preds = model(images)\n                avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n            avg_preds = np.mean(avg_preds, axis=0)\n            probabilities.append(avg_preds)\n        return np.concatenate(probabilities)\n    \n\n    predictions_resnext = pd.DataFrame(columns={\"image_id\"})\n    predictions_resnext[\"image_id\"] = submission_df[\"image_id\"].values\n    predictions_resnext['image_path_id'] = image_path + predictions_resnext['image_id'].astype(str)\n\n    model = CustomResNext('resnext50_32x4d', pretrained=False)\n    states = [torch.load(f) for f in used_models_pytorch[\"resnext\"]]\n\n    test_dataset = TestDataset(predictions_resnext, transform=get_transforms())\n    test_loader = DataLoader(test_dataset, batch_size=16, shuffle=False, num_workers=4, pin_memory=True)\n    predictions = inference(model, states, test_loader, device)\n\n    predictions_resnext['resnext'] = [np.squeeze(p) for p in predictions]\n    predictions_resnext = predictions_resnext.drop([\"image_path_id\"], axis=1)\n    \n\n    torch.cuda.empty_cache()\n    try:\n        del(model)\n        del(states)\n    except:\n        pass\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ViT"},{"metadata":{"trusted":true},"cell_type":"code","source":"if \"vit2020\" in used_models_pytorch:\n    \n    vit_image_size = 384\n    \n    class CustomViT(nn.Module):\n        def __init__(self, model_arch, n_class, pretrained=False):\n            super().__init__()\n            self.model = timm.create_model(model_arch, pretrained=pretrained)\n            n_features = self.model.head.in_features\n            self.model.head = nn.Linear(n_features, n_class)\n\n        def forward(self, x):\n            x = self.model(x)\n            return x\n        \n    class TestDataset(Dataset):\n        def __init__(self, df, transform=None):\n            self.df = df\n            self.file_names = df['image_path_id'].values\n            self.transform = transform\n\n        def __len__(self):\n            return len(self.df)\n\n        def __getitem__(self, idx):\n            file_name = self.file_names[idx]\n            im_bgr = cv2.imread(file_name)\n            image = im_bgr[:, :, ::-1]\n            if self.transform:\n                augmented = self.transform(image=image)\n                image = augmented['image']\n            return image\n\n    def get_tta_transforms():\n        return Compose([CenterCrop(vit_image_size, vit_image_size, p=1.),\n                Resize(vit_image_size, vit_image_size),\n                Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n                ToTensorV2(p=1.0)], p=1.)\n\n    def inference(models, test_loader, device):\n        tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n        probs = []\n        for i, (images) in tk0:\n            avg_preds = []\n            for model in models:\n                images = images.to(device)\n                model.to(device)\n                model.eval()\n                with torch.no_grad():\n                    y_preds = model(images)\n                avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n            avg_preds = np.mean(avg_preds, axis=0)\n            probs.append(avg_preds)\n        probs = np.concatenate(probs)\n        return probs\n\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    predictions_vit = pd.DataFrame(columns={\"image_id\"})\n    predictions_vit[\"image_id\"] = submission_df[\"image_id\"].values\n    predictions_vit['image_path_id'] = image_path + predictions_vit['image_id'].astype(str)\n\n    def load_cassava_vit(modelpath):\n        _model = CustomViT('vit_base_patch16_384', 5, pretrained=False)\n        _model.load_state_dict(torch.load(modelpath))\n        _model.eval()\n        return _model\n\n    models = [load_cassava_vit(f) for f in used_models_pytorch[\"vit2020\"]]\n\n    test_dataset = TestDataset(predictions_vit, transform=get_tta_transforms())\n    test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4, pin_memory=True)\n\n    predictions_raw_vit = inference(models, test_loader, device)\n\n    predictions_vit['vit2020'] = [np.squeeze(p) for p in predictions_raw_vit]\n    predictions_vit = predictions_vit.drop([\"image_path_id\"], axis=1)\n    \n    torch.cuda.empty_cache()\n    try:\n        for model in models:\n            del(model)\n    except:\n        pass\n    models = []\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Mobilenet V3 (CropNet)\n\nThere are multiple ways to include pretrained models from [TensorFlow Hub](https://www.tensorflow.org/hub/tutorials/cropnet_cassava), if internet has to be turned of during submission:\n\n* Accessing and storing the .tar.gz file (see [this](https://xianbao-qian.medium.com/how-to-run-tf-hub-locally-without-internet-connection-4506b850a915) Medium post) \n<code>\n!curl -LO https://storage.googleapis.com/tfhub-modules/google/cropnet/classifier/cassava_disease_V1/2.tar.gz\n!mkdir cropnet_mobilenetv3\n!tar -xf 2.tar.gz  --directory cropnet_mobilenetv3    \n</code>\n<br>\n\n* Downloading and caching the weights using\n<code>\nos.environ[\"TFHUB_CACHE_DIR\"] = \"/kaggle/working\"\nhub.KerasLayer('https://tfhub.dev/google/cropnet/classifier/cassava_disease_V1/2', trainable=False)\n</code>\n<br>\n\nYou can find more [information on caching on the official tfhub website](https://www.tensorflow.org/hub/caching) and more information on the [pretrained CropNet model ](https://tfhub.dev/google/cropnet/classifier/cassava_disease_V1/2). For the offline submissions we included these weights into a Kaggle Dataset bucket.\n\nRemark: In the meantime, TFHub models can apparently be integrated directly into the TPU training via Kaggle. Check out the[ Kaggle TPU FAQs](https://www.kaggle.com/product-feedback/216256)\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow_hub as hub\n\ndef build_mobilenet3(img_size=(224,224), weights=\"../input/cassava-leaf-disease-1st-place-models/cropnet_mobilenetv3/cropnet\"):\n    classifier = hub.KerasLayer(weights)\n    model = tf.keras.Sequential([\n    tf.keras.layers.InputLayer(input_shape=img_size + (3,)),\n    hub.KerasLayer(classifier, trainable=False)])\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Keras Inference with TTA\n\nFor the included EfficientNets we used simple test time augmentations (Flip, Rotate, Transpose). To do this, we cropped 4 overlapping patches of size 512x512 from the .jpg images and applied 2 augmentations to each patch. We retain two additional center-cropped patches of the image to which no augmentations were applied. To get an overall prediction, we took the average of all these image tiles.\n\nFor the CropNet, we just center-cropped and resized the image. In addition, we distributed the unknown class evenly over the 5 leaf diseases."},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_augmentations(image):\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n    \n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    \n    if p_spatial > 0.75:\n        image = tf.image.transpose(image)\n        \n    if p_rotate > 0.75:\n        image = tf.image.rot90(image, k = 3)\n    elif p_rotate > 0.5:\n        image = tf.image.rot90(image, k = 2)\n    elif p_rotate > 0.25:\n        image = tf.image.rot90(image, k = 1)\n\n    image = tf.image.resize(image, size = IMAGE_SIZE)\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    \n    return image\n\ndef read_preprocess_file(img_path, normalize=False):\n    image = Image.open(img_path)\n    if normalize:\n        img_scaled = np.array(image)/ 255.0\n    else:\n        img_scaled = np.array(image)\n    img_scaled = img_scaled.astype(np.float32)\n    return (image.size[0], image.size[1]), img_scaled\n\ndef create_image_tiles(origin_dim, processed_img):\n    crop_size = 512\n    img_list = []\n    # Cut image into 4 overlapping patches\n    for x in [0, origin_dim[1] - crop_size]:\n        for y in [0, origin_dim[0] - crop_size]:\n            img_list.append(processed_img[x:x+crop_size , y:y+crop_size,:])\n    # Keep one additional center cropped image \n    img_list.append(cv2.resize(processed_img[:, 100:700 ,:], dsize=(crop_size, crop_size)))\n    return np.array(img_list)\n\ndef augment_tiles_light(tiles, ttas=2):\n  # Copy central croped image to have same ratio to augmented images\n  holdout = np.broadcast_to(tiles[-1,:,:,:],(ttas,) + tiles.shape[1:])\n  augmented_batch = tf.map_fn(lambda x: image_augmentations(x), tf.concat(\n      [tiles[:-1,:,:,:] for _ in range(ttas)], axis=0))\n  return tf.concat([augmented_batch, holdout], axis=0)\n\ndef cut_crop_image(processed_img):\n    image = tf.image.central_crop(processed_img, 0.8)\n    image = tf.image.resize(image, (224, 224))\n    return np.expand_dims(image, 0)\n\n# CropNet class 6 (unknown) is distributed evenly over all 5 classes to match problem setting\ndef distribute_unknown(propabilities):\n    return propabilities[:,:-1] + np.expand_dims(propabilities[:,-1]/5, 1)\n\ndef multi_predict_tfhublayer(img_path, modelinstance):\n    img = cut_crop_image(read_preprocess_file(img_path, True)[1])\n    yhat = modelinstance.predict(img)\n    return np.mean(distribute_unknown(yhat), axis=0)\n\ndef multi_predict_keras(img_path, modelinstance, *args):\n    augmented_batch = augment_tiles_light(create_image_tiles(\n        *read_preprocess_file(img_path)))\n    Yhat = modelinstance.predict(augmented_batch)\n    return np.mean(Yhat, axis=0)\n\ndef predict_and_vote(image_list, modelinstances, onlykeras):\n    predictions = [] \n    with tqdm(total=len(image_list)) as process_bar:       \n      for img_path in image_list:\n        process_bar.update(1)  \n        Yhats = np.vstack([func(img_path, modelinstance) for func, modelinstance in modelinstances])\n        if onlykeras:\n            predictions.append(np.argmax(np.sum(Yhats, axis=0)))\n        else:\n            predictions.append(Yhats)    \n    return predictions\n\n\ninference_models = []\n\nif \"mobilenet\" in used_models_keras:\n    model_mobilenet = build_mobilenet3(weights=used_models_keras[\"mobilenet\"])\n    inference_models.append((multi_predict_tfhublayer, model_mobilenet))\n    \nif \"efficientnetb4\" in used_models_keras:\n    model_efficientnetb4 =  keras.models.load_model(used_models_keras[\"efficientnetb4\"], compile=False)\n    inference_models.append((multi_predict_keras, model_efficientnetb4))\n    \nif \"efficientnetb5\" in used_models_keras:\n    model_efficientnetb5 =  keras.models.load_model(used_models_keras[\"efficientnetb5\"])\n    inference_models.append((multi_predict_keras, model_efficientnetb5))\n\nsubmission_df[\"label\"] = predict_and_vote([image_path+id for id in submission_df[\"image_id\"].values], inference_models, onlykeras)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.backend.clear_session()\n\ntry:\n    del inference_models[:]\nexcept:\n    pass\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Final Ensembling\n\nOur winning submission just included CropNet, EfficientNet B4, ResNext50 and ViT and a mean approach. We took the mean of the class weights from the ResNext and ViT model and combined this combination with the MobileNet and the EfficientnetB4 in the second stage."},{"metadata":{"trusted":true},"cell_type":"code","source":"if len(list(used_models_keras.keys())) <= 1:\n    submission_df.loc[:,list(used_models_keras)[0]] = submission_df[\"label\"].explode()\nelse:\n    tmp = (submission_df['label'].transform([lambda x:x[0], lambda x:x[1]]).set_axis(list(used_models_keras.keys()), axis=1, inplace=False))\n    submission_df = submission_df.merge(tmp, right_index=True, left_index=True)\n    \nsubmission_df[\"label\"] = 0\n\nif \"resnext\" in used_models_pytorch:\n    submission_df = submission_df.merge(predictions_resnext, on=\"image_id\")\n    \nif \"efficientnetb3\" in used_models_pytorch:\n    submission_df = submission_df.merge(predictions_cutmix, on=\"image_id\")\n    \nif \"vit2020\" in used_models_pytorch:\n    submission_df = submission_df.merge(predictions_vit, on=\"image_id\")\n    \nif \"vit2019\" in used_models_pytorch:\n    submission_df = submission_df.merge(predictions_vit2019, on=\"image_id\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if stacked_mean:\n    submission_df[\"stage_1\"] = submission_df.apply(lambda row: [np.mean(e) for e in zip(row[\"vit2020\"], row[\"resnext\"])], axis=1)\n    submission_df[\"label\"] = submission_df.apply(lambda row: np.argmax(\n        [np.sum(e) for e in zip(row[\"mobilenet\"],row[\"stage_1\"], row[\"efficientnetb4\"])]), axis=1)        \nelse:\n    submission_df[\"label\"] = submission_df.apply(lambda row: np.argmax(\n        [np.sum(e) for e in zip(*[row[m] for m in list(used_models_pytorch.keys())+list(used_models_keras.keys())])]), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.head(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df[[\"image_id\",\"label\"]].to_csv(\"submission.csv\", index=False)\n!head submission.csv","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}