{"cells":[{"cell_type":"markdown","metadata":{},"source":"# About this notebook  \n\nThis notebook based on [Cassava / resnext50_32x4d starter [inference]](https://www.kaggle.com/yasufuminakama/cassava-resnext50-32x4d-starter-inference).\n\nDescribe customizations at [GitHub](https://github.com/IMOKURI/Cassava-Leaf-Disease-Classification) about this notebook.\n"},{"cell_type":"markdown","metadata":{},"source":"# Directory settings"},{"cell_type":"code","execution_count":1,"metadata":{},"outputs":[],"source":"# ====================================================\n# Directory settings\n# ====================================================\nimport os\n\nOUTPUT_DIR = \"./\"\nMODEL_DIR = \"../input/cassava-model/\"\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\nTRAIN_PATH = \"../input/cassava-leaf-disease-classification/train_images\"\nTEST_PATH = \"../input/cassava-leaf-disease-classification/test_images\""},{"cell_type":"markdown","metadata":{},"source":"# CFG"},{"cell_type":"code","execution_count":2,"metadata":{},"outputs":[],"source":"# ====================================================\n# CFG\n# ====================================================\nclass CFG:\n    debug = False\n    num_workers = 4\n    models = [\n        # \"tf_efficientnet_b3_ns\",\n        \"tf_efficientnet_b4_ns\",\n        \"vit_base_patch16_384\",\n        # \"deit_base_patch16_384\",\n        \"seresnext50_32x4d\",\n    ]\n    size = {\n        \"tf_efficientnet_b3_ns\": 512,\n        \"tf_efficientnet_b4_ns\": 512,\n        \"vit_base_patch16_384\": 384,\n        \"deit_base_patch16_384\": 384,\n        \"seresnext50_32x4d\": 512,\n    }\n    batch_size = 64\n    seed = 7097\n    target_size = 5\n    target_col = \"label\"\n    n_fold = 5\n    trn_fold = {  # [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n        \"tf_efficientnet_b3_ns\": {\n            \"best\": [0, 1, 2, 3, 4],\n            \"final\": [],\n        },\n        \"tf_efficientnet_b4_ns\": {\n            \"best\": [0, 1, 2, 3, 4],\n            \"final\": [],\n        },\n        \"vit_base_patch16_384\": {\"best\": [0, 1, 2, 3, 4], \"final\": []},\n        \"deit_base_patch16_384\": {\"best\": [0, 1, 2, 3, 4], \"final\": []},\n        \"seresnext50_32x4d\": {\"best\": [5, 6, 7, 8, 9], \"final\": []},\n    }\n    data_parallel = {\n        \"tf_efficientnet_b3_ns\": False,\n        \"tf_efficientnet_b4_ns\": True,  # True,\n        \"vit_base_patch16_384\": False,\n        \"deit_base_patch16_384\": False,\n        \"seresnext50_32x4d\": False,\n    }\n    transform = {\n        # \"tf_efficientnet_b3_ns\": None,\n        \"tf_efficientnet_b4_ns\": \"rotate\",\n        \"vit_base_patch16_384\": \"rotate\",\n        # \"deit_base_patch16_384\": None,\n        \"seresnext50_32x4d\": \"rotate\",\n    }\n    weight = {\n        # \"tf_efficientnet_b3_ns\": None,\n        \"tf_efficientnet_b4_ns\": 1,\n        \"vit_base_patch16_384\": 1,\n        # \"deit_base_patch16_384\": None,\n        \"seresnext50_32x4d\": 1,\n    }\n    tta = 10  # 1: no TTA, >1: TTA\n    no_tta_weight = tta - 1\n    train = False\n    inference = True"},{"cell_type":"code","execution_count":3,"metadata":{},"outputs":[],"source":"tta_weight_sum = CFG.no_tta_weight + (CFG.tta - 1)\nweight_sum = sum([CFG.weight[model] for model in CFG.models]) * tta_weight_sum"},{"cell_type":"markdown","metadata":{},"source":"# Library"},{"cell_type":"code","execution_count":4,"metadata":{},"outputs":[],"source":"# ====================================================\n# Library\n# ====================================================\nimport sys\n\nsys.path.append(\"../input/pytorch-image-models/pytorch-image-models-master\")\n\nimport math\nimport os\nimport random\nimport shutil\nimport time\nimport warnings\nfrom collections import Counter, defaultdict\nfrom contextlib import contextmanager\nfrom functools import partial\nfrom pathlib import Path\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as models\nfrom albumentations import (\n    CenterCrop,\n    CoarseDropout,\n    Compose,\n    Cutout,\n    HorizontalFlip,\n    HueSaturationValue,\n    IAAAdditiveGaussianNoise,\n    ImageOnlyTransform,\n    Normalize,\n    OneOf,\n    RandomBrightness,\n    RandomBrightnessContrast,\n    RandomContrast,\n    RandomCrop,\n    RandomResizedCrop,\n    Resize,\n    Rotate,\n    ShiftScaleRotate,\n    Transpose,\n    VerticalFlip,\n)\nfrom albumentations.pytorch import ToTensorV2\nfrom PIL import Image\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch.nn.parameter import Parameter\nfrom torch.optim import SGD, Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, CosineAnnealingWarmRestarts, ReduceLROnPlateau\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm.auto import tqdm\n\nwarnings.filterwarnings(\"ignore\")\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"},{"cell_type":"markdown","metadata":{},"source":"# Utils"},{"cell_type":"code","execution_count":5,"metadata":{},"outputs":[],"source":"# ====================================================\n# Utils\n# ====================================================\ndef get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f\"[{name}] start\")\n    yield\n    LOGGER.info(f\"[{name}] done in {time.time() - t0:.0f} s.\")\n\n\ndef init_logger(log_file=OUTPUT_DIR + \"inference.log\"):\n    from logging import INFO, FileHandler, Formatter, StreamHandler, getLogger\n\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\n\nLOGGER = init_logger()\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\n\nseed_torch(seed=CFG.seed)"},{"cell_type":"markdown","metadata":{},"source":"# Data Loading"},{"cell_type":"code","execution_count":6,"metadata":{},"outputs":[],"source":"test = pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")\ntest.head()"},{"cell_type":"markdown","metadata":{},"source":"# Dataset"},{"cell_type":"code","execution_count":7,"metadata":{},"outputs":[],"source":"# ====================================================\n# Dataset\n# ====================================================\nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df[\"image_id\"].values\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f\"{TEST_PATH}/{file_name}\"\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented[\"image\"]\n        return image"},{"cell_type":"markdown","metadata":{},"source":"# Transforms"},{"cell_type":"code","execution_count":8,"metadata":{},"outputs":[],"source":"# ====================================================\n# Transforms\n# ====================================================\ndef get_transforms(*, data, size):\n\n    if data == \"train\":\n        return Compose(\n            [\n                # Resize(size, size),\n                RandomResizedCrop(size, size),\n                Transpose(p=0.5),\n                HorizontalFlip(p=0.5),\n                VerticalFlip(p=0.5),\n                ShiftScaleRotate(p=0.5),\n                HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n                RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n                CoarseDropout(p=0.5),\n                Cutout(p=0.5),\n                Normalize(\n                    mean=[0.485, 0.456, 0.406],\n                    std=[0.229, 0.224, 0.225],\n                ),\n                ToTensorV2(),\n            ]\n        )\n\n    if data == \"valid\":\n        return Compose(\n            [\n                Resize(size, size),\n                CenterCrop(size, size),\n                Normalize(\n                    mean=[0.485, 0.456, 0.406],\n                    std=[0.229, 0.224, 0.225],\n                ),\n                ToTensorV2(),\n            ]\n        )\n\n    if data == \"simple\":\n        return Compose(\n            [\n                # Resize(size, size),\n                RandomResizedCrop(size, size),\n                # Transpose(p=0.5),\n                # HorizontalFlip(p=0.5),\n                # VerticalFlip(p=0.5),\n                # ShiftScaleRotate(p=0.5),\n                # HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n                # RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n                # CoarseDropout(p=0.5),\n                # Cutout(p=0.5),\n                Normalize(\n                    mean=[0.485, 0.456, 0.406],\n                    std=[0.229, 0.224, 0.225],\n                ),\n                ToTensorV2(),\n            ]\n        )\n\n    if data == \"rotate\":\n        return Compose(\n            [\n                # Resize(size, size),\n                RandomResizedCrop(size, size),\n                Transpose(p=0.5),\n                HorizontalFlip(p=0.5),\n                VerticalFlip(p=0.5),\n                # ShiftScaleRotate(p=0.5),\n                # HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n                # RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n                # CoarseDropout(p=0.5),\n                # Cutout(p=0.5),\n                Normalize(\n                    mean=[0.485, 0.456, 0.406],\n                    std=[0.229, 0.224, 0.225],\n                ),\n                ToTensorV2(),\n            ]\n        )"},{"cell_type":"markdown","metadata":{},"source":"# MODEL"},{"cell_type":"code","execution_count":9,"metadata":{},"outputs":[],"source":"# ====================================================\n# MODEL\n# ====================================================\nclass CassvaImgClassifier(nn.Module):\n    def __init__(self, model_name=\"resnext50_32x4d\", pretrained=False):\n        super().__init__()\n\n        if model_name == \"deit_base_patch16_384\":\n            # self.model = torch.hub.load(\"facebookresearch/deit:main\", model_name, pretrained=pretrained)\n            self.model = torch.hub.load(\"../input/fair-deit\", model_name, pretrained=pretrained, source=\"local\")\n            n_features = self.model.head.in_features\n            self.model.head = nn.Linear(n_features, CFG.target_size)\n\n        else:\n            self.model = timm.create_model(model_name, pretrained=pretrained)\n\n            if \"resnext50_32x4d\" in model_name:\n                n_features = self.model.fc.in_features\n                self.model.fc = nn.Linear(n_features, CFG.target_size)\n\n            elif model_name.startswith(\"tf_efficientnet\"):\n                n_features = self.model.classifier.in_features\n                self.model.classifier = nn.Linear(n_features, CFG.target_size)\n\n            elif model_name.startswith(\"vit_\"):\n                n_features = self.model.head.in_features\n                self.model.head = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x"},{"cell_type":"markdown","metadata":{},"source":"# Helper functions"},{"cell_type":"code","execution_count":10,"metadata":{},"outputs":[],"source":"# ====================================================\n# Helper functions\n# ====================================================\ndef inference(model, states, test_loader, device, data_parallel):\n    model.to(device)\n\n    # Use multi GPU\n    if device == torch.device(\"cuda\") and data_parallel:\n        model = torch.nn.DataParallel(model)  # make parallel\n        # torch.backends.cudnn.benchmark=True\n\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state[\"model\"])\n            model.eval()\n            with torch.no_grad():\n                y_preds = model(images)\n            avg_preds.append(y_preds.softmax(1).to(\"cpu\").numpy())\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs"},{"cell_type":"markdown","metadata":{},"source":"# inference"},{"cell_type":"code","execution_count":11,"metadata":{},"outputs":[],"source":"# ====================================================\n# inference\n# ====================================================\npredictions = None\nfor model_name in CFG.models:\n    for i in range(CFG.tta):\n        model = CassvaImgClassifier(model_name, pretrained=False)\n        states = []\n        for saved_model in [\"best\", \"final\"]:\n            if CFG.trn_fold[model_name][saved_model] != []:\n                LOGGER.info(\n                    f\"========== Model: {model_name}, TTA: {i}, Saved: {saved_model}, Fold: {CFG.trn_fold[model_name][saved_model]} ==========\"\n                )\n                states += [\n                    torch.load(MODEL_DIR + f\"{model_name}_fold{fold}_{saved_model}.pth\")\n                    for fold in CFG.trn_fold[model_name][saved_model]\n                ]\n\n        if i == 0:  # no TTA\n            test_dataset = TestDataset(test, transform=get_transforms(data=\"valid\", size=CFG.size[model_name]))\n            tta_weight = CFG.no_tta_weight\n        else:\n            test_dataset = TestDataset(\n                test, transform=get_transforms(data=CFG.transform[model_name], size=CFG.size[model_name])\n            )\n            tta_weight = 1\n\n        test_loader = DataLoader(\n            test_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.num_workers, pin_memory=True\n        )\n\n        inf = inference(model, states, test_loader, device, CFG.data_parallel[model_name])\n        LOGGER.info(f\"Inference example: {inf[0]}\")\n\n        if predictions is None:\n            predictions = inf[np.newaxis] * CFG.weight[model_name] * tta_weight\n        else:\n            predictions = np.append(predictions, inf[np.newaxis] * CFG.weight[model_name] * tta_weight, axis=0)\n\nsub = np.sum(predictions, axis=0) / weight_sum\nLOGGER.info(f\"========== Overall ==========\")\nLOGGER.info(f\"Submission example: {sub[0]}\")\n\n\n# submission\ntest[\"label\"] = sub.argmax(1)\ntest[[\"image_id\", \"label\"]].to_csv(OUTPUT_DIR + \"submission.csv\", index=False)\ntest.head()"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.9"}},"nbformat":4,"nbformat_minor":4}