{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Simple average ensemble of two highest-score infernece notebooks\n\nChanges made to the original notebooks:\n- For ResNext, added TTA \n\n<br>\n<br>\n<br>\n\n# Part 1: EfficientNet\n\nOriginal notebook: https://www.kaggle.com/mekhdigakhramanian/pytorch-efficientnet-baseline-inference-tta"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"package_path = '../input/pytorch-image-models/pytorch-image-models-master'\nimport sys; sys.path.append(package_path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from datetime import datetime\nfrom glob import glob\nfrom scipy.ndimage.interpolation import zoom\nfrom skimage import io\nfrom sklearn import metrics\nfrom sklearn.metrics import log_loss\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nfrom torch import nn\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torchvision import transforms\nfrom tqdm import tqdm\nimport copy\nimport cv2\nimport joblib\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nimport pydicom\nimport random\nimport sklearn\nimport time\nimport timm # from efficientnet_pytorch import EfficientNet\nimport torch\nimport torchvision\nimport warnings","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 10,\n    'seed': 719,\n    'model_arch': 'tf_efficientnet_b3_ns',\n    'img_size': 512,\n    'epochs': 32,\n    'train_bs': 28,\n    'valid_bs': 32,\n    'lr': 1e-4,\n    'num_workers': 4,\n    'accum_iter': 1, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'tta': 3,\n    'used_epochs': [6,7,8,9],\n    'weights': [1,1,1,1]\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Helper Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    return im_rgb\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(\n        self, df, data_root, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        \n        img  = get_img(path)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define Train\\Validation Image Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion,\n    HueSaturationValue, IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur,\n    IAAPiecewiseAffine, RandomResizedCrop, IAASharpen, IAAEmboss,\n    RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout,\n    ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_inference_transforms():\n    return Compose([\n        RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n        Transpose(p=0.5),\n        HorizontalFlip(p=0.5),\n        VerticalFlip(p=0.5),\n        HueSaturationValue(\n            hue_shift_limit=0.2,\n            sat_shift_limit=0.2,\n            val_shift_limit=0.2,\n            p=0.5\n        ),\n        RandomBrightnessContrast(\n            brightness_limit=(-0.1,0.1),\n            contrast_limit=(-0.1, 0.1),\n            p=0.5\n        ),\n        Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225],\n            max_pixel_value=255.0,\n            p=1.0\n        ),\n        ToTensorV2(p=1.0)\n    ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Main Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n    image_preds_all = []\n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        image_preds = model(imgs)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n    seed_everything(CFG['seed'])\n    # We do stratified validation split in each fold to make each fold's train\n    # and validation set looks like the whole train set in target distributions.\n    folds = (\n        StratifiedKFold(n_splits=CFG['fold_num'])\n        .split(np.arange(train.shape[0]), train.label.values)\n    )\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n\n        if fold > 0:\n            break \n        print('Inference fold {} started'.format(fold))\n        valid_ = train.loc[val_idx,:].reset_index(drop=True)\n        valid_ds = CassavaDataset(\n            valid_,\n            '../input/cassava-leaf-disease-classification/train_images/',\n            transforms=get_inference_transforms(),\n            output_label=False\n        )\n\n        test = pd.DataFrame()\n        test['image_id'] = list(\n            os.listdir('../input/cassava-leaf-disease-classification/test_images/')\n        )\n        test_ds = CassavaDataset(\n            test,\n            '../input/cassava-leaf-disease-classification/test_images/',\n            transforms=get_inference_transforms(),\n            output_label=False\n        )\n\n        val_loader = torch.utils.data.DataLoader(\n            valid_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n\n        tst_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n\n        device = torch.device(CFG['device'])\n        model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique()).to(device)\n\n        val_preds = []\n        tst_preds = []\n\n        for i, epoch in enumerate(CFG['used_epochs']):    \n            model.load_state_dict(\n                torch.load(\n                    '../input/fork-pytorch-efficientnet-baseline-train-amp-a/{}_fold_{}_{}'\n                    .format(CFG['model_arch'],fold, epoch)\n                )\n            )\n            with torch.no_grad():\n                for _ in range(CFG['tta']):\n                    val_preds += [\n                        CFG['weights'][i]\n                        /sum(CFG['weights'])\n                        /CFG['tta']\n                        *inference_one_epoch(model, val_loader, device)\n                    ]\n                    tst_preds += [\n                        CFG['weights'][i]\n                        /sum(CFG['weights'])\n                        /CFG['tta']\n                        *inference_one_epoch(model, tst_loader, device)\n                    ]\n\n        val_preds = np.mean(val_preds, axis=0) \n        tst_preds = np.mean(tst_preds, axis=0) \n\n        print('fold {} validation loss = {:.5f}'.format(\n            fold,\n            log_loss(valid_.label.values, val_preds)\n        ))\n        print('fold {} validation accuracy = {:.5f}'.format(\n            fold,\n            (valid_.label.values==np.argmax(val_preds, axis=1)).mean()\n        ))\n\n        del model\n        torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Clean Up Variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"variable_list = %who_ls\nfor _ in variable_list:\n    if _ is not \"tst_preds\":\n        del globals()[_]\n\n%who_ls","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Part 2: ResNext\n\nOriginal notebook: https://www.kaggle.com/piantic/no-tta-cassava-resnext50-32x4d-inference-lb0-903\n\n### Directory settings"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\nOUTPUT_DIR = './'\nMODEL_DIR = '../input/cassava-resnext50-32x4d-weights/'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n    \nTRAIN_PATH = '../input/cassava-leaf-disease-classification/train_images'\nTEST_PATH = '../input/cassava-leaf-disease-classification/test_images'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### CFG"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    debug=False\n    num_workers=8\n    model_name='resnext50_32x4d'\n    size=512\n    batch_size=32\n    seed=2020\n    target_size=5\n    target_col='label'\n    n_fold=5\n    trn_fold=[0, 1, 2, 3, 4]\n    inference=True\n    tta=8","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Library"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n\nfrom albumentations.pytorch import ToTensorV2\nfrom collections import defaultdict, Counter\nfrom contextlib import contextmanager\nfrom functools import partial\nfrom pathlib import Path\nfrom PIL import Image\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch.nn.parameter import Parameter\nfrom torch.optim import Adam, SGD\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm.auto import tqdm\nimport albumentations as A\nimport cv2\nimport math\nimport numpy as np\nimport os\nimport pandas as pd\nimport random\nimport scipy as sp\nimport shutil\nimport time\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as models\nimport warnings\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\ndef init_logger(log_file=OUTPUT_DIR+'inference.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_transforms(*, data):\n    if data == 'valid':\n        return A.Compose([\n            A.Resize(CFG.size, CFG.size),\n            # A.RandomResizedCrop(CFG.size, CFG.size),\n            A.Transpose(p=0.5),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2()\n        ])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomResNext(nn.Module):\n    def __init__(self, model_name='resnext50_32x4d', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_state(model_path):\n    model = CustomResNext(CFG.model_name, pretrained=False)\n    try:  # single GPU model_file\n        model.load_state_dict(torch.load(model_path)['model'], strict=True)\n        state_dict = torch.load(model_path)['model']\n    except:  # multi GPU model_file\n        state_dict = torch.load(model_path)['model']\n        state_dict = {\n            k[7:]\n            if k.startswith('module.')\n            else k: state_dict[k]\n            for k in state_dict.keys()\n        }\n\n    return state_dict\n\n\ndef inference(model, states, test_loader, device):\n    tta_probs = []\n    for e in range(CFG.tta):\n        model.to(device)\n        tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n        probs = []\n        for i, (images) in tk0:\n            images = images.to(device)\n            avg_preds = []\n            for state in states:\n                model.load_state_dict(state)\n                model.eval()\n                with torch.no_grad():\n                    y_preds = model(images)\n                avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n            avg_preds = np.mean(avg_preds, axis=0)\n            probs.append(avg_preds)\n        probs = np.concatenate(probs)\n        tta_probs.append(probs)\n    tta_probs = np.mean(tta_probs, axis=0)\n    return tta_probs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = CustomResNext(CFG.model_name, pretrained=False)\nstates = [load_state(MODEL_DIR+f'{CFG.model_name}_fold{fold}.pth') for fold in CFG.trn_fold]\ntest_dataset = TestDataset(test, transform=get_transforms(data='valid'))\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size=CFG.batch_size,\n    shuffle=False, \n    num_workers=CFG.num_workers,\n    pin_memory=True\n)\npredictions = inference(model, states, test_loader, device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Final Combine"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = test[[\"image_id\"]]\nsubmission[\"label\"] = (tst_preds * 0.5 + predictions * 0.5).argmax(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}