{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import Libraries"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!pip install -U -q ../input/resnest/resnest-0.0.5-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os\nimport random\nimport time\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn.functional as F\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, Transpose, RandomResizedCrop, Compose, Normalize, ShiftScaleRotate, CenterCrop, Resize, RandomResizedCrop\n)\nfrom albumentations.pytorch import ToTensorV2\nfrom glob import glob\nfrom resnest.torch import resnest50, resnest50_fast_4s2x40d\nfrom torch import nn\nfrom torch.utils.data import Dataset\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import sys\n\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import timm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configuration"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"CFG = {\n    'seed': 1337,\n    'img_size': 512,\n    'bs': 32,\n    'num_workers': 4,\n    'tta': 4,\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Data"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def seed_everything(seed: int):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def get_img(path):\n    return cv2.imread(path)[:, :, ::-1]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"seed_everything(CFG['seed'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, df, data_root, transforms=None):\n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, index: int):\n        img  = get_img(f\"{self.data_root}/{self.df.iloc[index]['image_id']}\")\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def get_inference_transforms():\n    return Compose([\n        # RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n        # Resize(CFG['img_size'], CFG['img_size']),\n        CenterCrop(CFG['img_size'], CFG['img_size']),\n        # Transpose(p=0.5),\n        HorizontalFlip(p=0.5),\n        VerticalFlip(p=0.5),\n        # HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n        # RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        ToTensorV2(p=1.0),\n    ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class CassvaClassifierV1(nn.Module):\n    def __init__(self, n_classes: int = 5, dropout: float = .5):\n        super().__init__()\n        self.backbone = resnest50_fast_4s2x40d(pretrained=False)\n\n        self.pool = nn.AdaptiveAvgPool2d(1)\n        self.dropout = nn.Dropout(p=dropout)\n        self.emb_size: int = 2048\n\n        self.classifier = nn.Linear(self.emb_size, n_classes)\n\n    def cnn_feature_extractor(self, x):\n        x = self.backbone.conv1(x)\n        x = self.backbone.bn1(x)\n        x = self.backbone.relu(x)\n        x = self.backbone.maxpool(x)\n\n        x1 = self.backbone.layer1(x)\n        x2 = self.backbone.layer2(x1)\n        x3 = self.backbone.layer3(x2)\n        x4 = self.backbone.layer4(x3)\n        return x4\n\n    def forward(self, x):\n        x = self.cnn_feature_extractor(x)\n        x = self.pool(x)\n        x = self.dropout(x)\n        x = x.view(-1, self.emb_size)\n\n        x = self.classifier(x)\n\n        return x","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class CassvaClassifierV2(nn.Module):\n    def __init__(self, n_classes: int = 5):\n        super().__init__()\n        self.model = timm.create_model('tf_efficientnet_b4_ns', pretrained=False)\n        self.model.classifier = nn.Linear(self.model.classifier.in_features, n_classes)\n\n    def forward(self, x):\n        return self.model(x)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class CassvaClassifierV3(nn.Module):\n    def __init__(self, n_classes: int = 5):\n        super().__init__()\n        self.model = timm.create_model('tf_efficientnet_b3_ns', pretrained=False)\n        self.model.classifier = nn.Linear(self.model.classifier.in_features, n_classes)\n\n    def forward(self, x):\n        return self.model(x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference Function"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    for imgs in data_loader:\n        image_preds = model(imgs.to(device).float())\n        image_preds_all += [torch.softmax(image_preds, 1).cpu().numpy()]\n\n    return np.concatenate(image_preds_all, axis=0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Models"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"model_paths = sorted(glob(os.path.join('../input/leaf-disease-resnest50', '*')))\nmodel_paths.extend(sorted(glob(os.path.join('../input/leaf-disease-effnet', '*'))))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def get_models(model_paths, recipe: str):\n    models = []\n    for model_path in model_paths:\n        model_name: str = model_path.split('/')[-1]\n\n        n_folds: int = int([x[-1] for x in model_name.split('-') if x.startswith('fold')][0])\n        n_epochs: int = int([x[6:] for x in model_name.split('-') if x.startswith('epochs')][0])\n            \n        if not model_name.startswith(recipe):\n            continue\n\n        if model_name.startswith('resnest50_fast_4s2x40d-fmix-cutmix-'):\n            model = CassvaClassifierV1().to(device)\n        elif model_name.startswith('resnest50_fast_4s2x40d-cutmix-fmix-'):\n            model = CassvaClassifierV1().to(device)\n        elif model_name.startswith('resnest50_fast_4s2x40d-fcl-'):\n            model = CassvaClassifierV1().to(device)\n        elif model_name.startswith('effnetb4-fcl-'):\n            model = CassvaClassifierV2().to(device)\n        elif model_name.startswith('effnetb3-cutmix-scce-'):\n            model = CassvaClassifierV3().to(device)\n        elif model_name.startswith('effnetb4-cutmix-fmix-'):\n            model = CassvaClassifierV2().to(device)\n        elif model_name.startswith('effnetb4-pseudo-cutmix-fmix-'):\n            model = CassvaClassifierV2().to(device)\n        else:\n            continue\n\n        print(f'[+] load {model_name}')\n\n        model.load_state_dict(torch.load(model_path, map_location=device))\n        model.eval()\n\n        models.append(model)\n\n    return models","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Test Data"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"test = pd.DataFrame()\ntest['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\ntest_ds = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms())\n\ntst_loader = torch.utils.data.DataLoader(\n    test_ds, \n    batch_size=CFG['bs'],\n    num_workers=CFG['num_workers'],\n    shuffle=False,\n    pin_memory=True\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"recipe_list = [\n    # 'effnetb3-cutmix-scce-',\n    'effnetb4-cutmix-fmix-',\n    'effnetb4-fcl-',\n    # 'effnetb4-pseudo-cutmix-fmix-',\n    'resnest50_fast_4s2x40d-cutmix-fmix-',\n    'resnest50_fast_4s2x40d-fcl-',\n    'resnest50_fast_4s2x40d-fmix-cutmix-',\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"preds = []\nwith torch.no_grad():\n    for recipe in recipe_list:\n        models = get_models(model_paths, recipe)\n\n        preds_per_recipe = np.mean(\n            [\n                np.mean([inference_one_epoch(model, tst_loader, device) for _ in range(CFG['tta'])], axis=0)\n                for model in models\n            ], axis=0\n        )\n\n        preds.append(preds_per_recipe)\n\n        del models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# weights = [0.33547759, 0.30181755, 0.28914882]                          # Ensembles 9 weights\n# weights = [0.32264375, 0.19517635, 0.10858799, 0.33353971]              # Ensembles 11 weights\n\n# weights = [0.31301767, 0.29456512, 0.31728454]                          # Ensembles 20 weights\n# weights_v1 = [0.21145703, 0.18271946, 0.26915887, 0.29597817]           # Ensembles 21 weights\n# weights_v2 = [0.10884351, 0.2009535, 0.18187667, 0.20405396]            # Ensembles 21 weights - v3\n# weights_v3 = [0.21411924, 0.18756664, 0.26507698, 0.31649887]           # Ensembles 21 weights - v4\n# weights = [0.28008428, 0.08930099, 0.19287446, 0.13415098, 0.2855688]   # Ensembles 22 weights\n# weights = [0.33566536, 0.10897427, 0.19606722, 0.31792425]              # Ensembles 23 weights\n# weights = [0.22155271, 0.1881944, 0.38943474, 0.1644162]                # Ensembles 24 weigths - v1 (Simplex)\n# weights = [0.29872177, 0.41167376, 0.92104135, 0.51346469]              # Ensembles 24 weights - v2 (Optuna)\nweights_v1 = [0.16956903, 0.11774465, 0.3500565, 0.06345556, 0.27596693]   # Ensembles 25 weights - v1 (Simplex)\nweights_v2 = [0.40763181, 0.23739473, 0.89755062, 0.15559366, 0.78777895]  # Ensembles 25 weights - v2 (Optuna)\nweights_v3 = [0.213048, 0.1057116, 0.37792647, 0.10013445, 0.33992517]     # Ensembles 25 weights - v3 (Optuna) v1 wise\n\n# weights = [0.34618164, 0.19092364, 0.38515934, 0.91232422, 0.00026995, 0.70023081]  # Ensembles 27 weights - v2 (Optuna)\n# weights = [0.17911332, 0.10146231, 0.40657403, 0.30869513]              # Ensembles 28 weights - v1 (Simplex)\n# weights = [0.13729324, 0.19570104, 0.81195183, 0.38014906]              # Ensembles 28 weights - v2 (Optuna)\n\n# weights = [0.30978996, 0.22192819, 0.17375666, 0.29452519]              # Ensembles 21 weights w/ Optuna\n# weights = [0.23779558, 0.24279284, 0.10746264, 0.41194895]              # Ensembles 21 weights + 2020 validation w/ Optuna\n# weights = [0.22772560, 0.10498674, 0.19491591, 0.18363636, 0.28873539]  # Ensembles 22 weights w/ Optuna\n# weights = [0.22084675, 0.08012831, 0.24442862, 0.07493153, 0.37966478]  # Ensembles 22 weights + 2020 validation w/ Optuna\n\n# weights = [0.31792425, 0.33566536, 0.19606722, 0.10897427]              # Ensembles 21 v1-corr\n# weights = [0.90971052, 0.82730546, 0.38318065, 0.26230337]              # Ensembles 21 v2-corr\n# weights = [0.16892105, 0.13211221, 0.54735528, 0.06826233, 0.06961585]  # Ensembles 25 v1-corr\n# weights = [0.3306592, 0.25242486, 0.8238158, 0.11380859, 0.22893606]    # Ensembles 25 v2-corr\n# weights = [0.64438387, 0.06787352, 0.21374317, 0.92894338, 0.30073056, 0.25681572]  # Ensembles 27 v2-corr\n\nweights = [round((x + y + z) / 3., 4) for x, y, z in zip(weights_v1, weights_v2, weights_v3)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# weights = [0.16127426, 0.14639634, 0.40610428, 0.25388546]                          # Ensembles 29 v1\n# weights = [0.14392024, 0.26999754, 0.66471911, 0.10123768]                          # Ensembles 29 v2\n# weights = [0.62811276, 0.23633465, 0.25867451, 0.95243224, 0.10789748, 0.5444512]   # Ensembles 30 v2\n# weights = [0.15856572, 0.03564624, 0.09446308, 0.06490467, 0.51857468, 0.12564358]  # Ensembles 31 v1\n# weights = [0.37460579, 0.06460235, 0.06938154, 0.22316517, 0.83760474, 0.26799389]  # Ensembles 31 v2\n# weights = [0.03564624, 0.15856572, 0.12564358, 0.51857468, 0.06490467, 0.09446308]  # Ensembles 32 v1\n# weights = [2.47e-05, 0.58436138, 0.5120863, 0.87538525, 0.61622132, 0.31402191]     # Ensembles 32 v2\n# weights = [0.50718439, 0.16996559, 0.25776149, 0.13448321, 0.92798265, 0.2179295, 0.22300932]  # Ensembles 33 v2\n# weights = [0.72592935, 0.90095763, 0.65667935, 0.53306918]                          # Ensembles 34 v2\n# weights = [0.29556101, 0.95351582, 0.96590418, 0.82782731, 0.73989029]              # Ensembles 35 v2\n# weights = [0.24627387, 0.12512951, 0.18096359, 0.07093961, 0.37141577]              # Ensembles 35 v3\n\nweights","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"# tst_preds = np.mean(preds, axis=0)\n\n# tst_preds = preds[0]\n# tst_preds = weights[0] * preds[0] + weights[1] * preds[1] + weights[2] * preds[2]\n# tst_preds = weights[0] * preds[0] + weights[1] * preds[1] + weights[2] * preds[2] + weights[3] * preds[3]\ntst_preds = weights[0] * preds[0] + weights[1] * preds[1] + weights[2] * preds[2] + weights[3] * preds[3] + weights[4] * preds[4]\n# tst_preds = weights[0] * preds[0] + weights[1] * preds[1] + weights[2] * preds[2] + weights[3] * preds[3] + weights[4] * preds[4] + weights[5] * preds[5]\n# tst_preds = weights[0] * preds[0] + weights[1] * preds[1] + weights[2] * preds[2] + weights[3] * preds[3] + weights[4] * preds[4] + weights[5] * preds[5] + weights[6] * preds[6]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"test['label'] = np.argmax(tst_preds, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"test.to_csv('submission.csv', index=False)\ntest","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EOF"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}