{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Directory settings"},{"metadata":{"trusted":true},"cell_type":"code","source":"debug = True\n# debug = False\n\nMODEL_DIR = '../input/34t-efficientnet-b5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Directory settings\n# ====================================================\nimport os\nimport glob\n\nOUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\nTRAIN_PATH = '../input/cassava-leaf-disease-classification/train_images'\nTEST_PATH = '../input/cassava-leaf-disease-classification/test_images'\n\nassert len(glob.glob(f'{MODEL_DIR}/*.yml'))==1\nconfig_path = glob.glob(f'{MODEL_DIR}/*.yml')[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Library"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n\nimport os\nimport math\nimport time\nimport random\nimport glob\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose, CenterCrop\n    )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport timm\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CFG"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # ====================================================\n# # CFG\n# # ====================================================\nimport yaml\n\nwith open(config_path) as f:\n    config = yaml.load(f)\n\nINFO = config['info']\nTAG = config['tag']\nCFG = config['cfg']\n\nCFG['train'] = False\nCFG['inference'] = True\ninference_batch_size = 8\n\n\n# if not os.path.exists('__notebook__.ipynb'):\n#     CFG['debug'] = True\n\n# class CFG:\n#     debug=False\n#     num_workers=4\n#     model_name='tf_efficientnet_b4_ns'\n#     size=256\n#     batch_size=32\n#     seed=42\n#     target_size=5\n#     target_col='label'\n#     n_fold=5\n#     trn_fold=[0, 1, 2, 3, 4]\n#     train=False\n#     inference=True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n\ndef init_logger(log_file=OUTPUT_DIR+'inference.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = init_logger()\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG['seed'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_result(result_df):\n    preds = result_df['preds'].values\n    labels = result_df['label'].values\n    score = get_score(labels, preds)\n    LOGGER.info(f'Score: {score:<.5f}')\n    \n    return score\n\n\ndef get_aug_name(compose):\n    aug_list = []\n    for aug in compose:\n        aug_list.append(aug.__class__.__name__)\n        \n    return aug_list\n\n\ndef get_aug_score(aug_preds, labels, tta_list):\n    for i in range(aug_preds.shape[1]):\n        aug_pred = aug_preds[:,i,:]\n        aug_list = get_aug_name(tta_list[i])\n        score = get_score(labels, aug_pred.argmax(1))\n        print(score, aug_list)\n        LOGGER.info(f\"========== aug: {aug_list} result ==========\")\n        LOGGER.info(f'Score: {score:<.5f}')\n        \n        \n        \ndef get_aug_csv(aug_preds, oof_df, tta_list):\n    for i in range(aug_preds.shape[1]):\n        base_df = oof_df.copy()[['image_id', 'label', 'fold']]\n        aug_pred = aug_preds[:,i,:]\n        base_df[[str(c) for c in range(5)]] = aug_pred\n        base_df['preds'] = aug_pred.argmax(1)\n        aug_pred = aug_preds[:,i,:]\n        aug_list = get_aug_name(tta_list[i])\n        csv_name = '-'.join(aug_list)\n        base_df.to_csv(f'{OUTPUT_DIR}{csv_name}.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\nclass TrainDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.labels = df['label'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TRAIN_PATH}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).long()\n        return image, label\n    \n\nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image\n    \n    \nclass TTADataset(Dataset):\n    def __init__(self, df, image_path, ttas):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.labels = df['label'].values\n        self.image_path = image_path\n        self.ttas = ttas\n\n    def __len__(self) -> int:\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{self.image_path}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        imglist=[tta(image=image)['image'] for tta in self.ttas]  # update\n\n        image=torch.stack(imglist)\n        label = torch.tensor(self.labels[idx]).long()\n        \n        return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _get_augmentations(aug_list):\n    process = []\n    for aug in aug_list:\n        if aug ==  'Resize':\n            process.append(Resize(CFG['size'], CFG['size']))\n        elif aug == 'RandomResizedCrop':\n            process.append(RandomResizedCrop(CFG['size'], CFG['size']))\n        elif aug == 'CenterCrop':\n            process.append(CenterCrop(CFG['size'], CFG['size']))\n        elif aug == 'Transpose':\n            process.append(Transpose(p=0.5))\n        elif aug == 'HorizontalFlip':\n            process.append(HorizontalFlip(p=0.5))\n        elif aug == 'VerticalFlip':\n            process.append(VerticalFlip(p=0.5))\n        elif aug == 'ShiftScaleRotate':\n            process.append(ShiftScaleRotate(p=0.5))\n        elif aug == 'Normalize':\n            process.append(Normalize(\n                            mean=[0.485, 0.456, 0.406],\n                            std=[0.229, 0.224, 0.225],\n                        ))\n        else:\n            raise ValueError(f'{aug} is not suitable')\n\n    process.append(ToTensorV2())\n\n    return process","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Transforms\n# ====================================================\ndef get_transforms(*, aug_list):\n    \n    return Compose(\n        _get_augmentations(aug_list)\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"norm_mean = [0.485, 0.456, 0.406]\nnorm_std = [0.229, 0.224, 0.225]\n\noneof_augs = [\n    CenterCrop(CFG['size'], CFG['size']), \n    Resize(CFG['size'], CFG['size'])\n]\n\nttas = [[\n    Compose([\n        oneof_aug,\n        Normalize(mean=norm_mean, std=norm_std, p=1.),\n        ToTensorV2()\n    ]),\n    Compose([\n        oneof_aug,\n        Transpose(p=1),\n        Normalize(mean=norm_mean, std=norm_std, p=1.),\n        ToTensorV2()\n    ]),\n    Compose([\n        oneof_aug,\n        HorizontalFlip(p=1),\n        Normalize(mean=norm_mean, std=norm_std, p=1.),\n        ToTensorV2()\n    ]),\n    Compose([\n        oneof_aug,\n        VerticalFlip(p=1),\n        Normalize(mean=norm_mean, std=norm_std, p=1.),\n        ToTensorV2()\n    ])\n] for oneof_aug in oneof_augs]\n\n# 平滑化\nttas = sum(ttas, [])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ttas","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MODEL"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# MODEL\n# ====================================================\nclass CustomModel(nn.Module):\n    def __init__(self, model_name, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        if hasattr(self.model, 'classifier'):\n            n_features = self.model.classifier.in_features\n            self.model.classifier = nn.Linear(n_features, CFG['target_size'])\n        elif hasattr(self.model, 'fc'):\n            n_features = self.model.fc.in_features\n            self.model.fc = nn.Linear(n_features, CFG['target_size'])\n        elif hasattr(self.model, 'head'):\n            n_features = self.model.head.in_features\n            self.model.head = nn.Linear(n_features, CFG['target_size'])\n\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\ndef inference(model, states, test_loader, device):\n    model.to(device)\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images, _) in tk0:\n        images = images.to(device)\n        batch_size, n_crops, c, h, w = images.size()\n        images = images.view(-1, c, h, w)\n        \n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state['model'])\n            model.eval()\n            with torch.no_grad():\n                y_preds = model(images).softmax(1)\n                y_preds = y_preds.view(batch_size, n_crops,-1)\n            avg_preds.append(y_preds.to('cpu').numpy())\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n        del images, _, y_preds, avg_preds\n        torch.cuda.empty_cache()\n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# inference\n# ====================================================\nmodel = CustomModel(TAG['model_name'], pretrained=False)\nmodel_paths = glob.glob(f'{MODEL_DIR}/*.pth')\nmodel_paths.sort()\nstates = [torch.load(path) for path in model_paths]\n# test_dataset = TestDataset(test, transform=get_transforms(aug_list=['Resize', 'Normalize']))\ntest_dataset = TTADataset(test, TEST_PATH, ttas=ttas)\ntest_loader = DataLoader(test_dataset, batch_size=inference_batch_size, shuffle=False, \n                         num_workers=2, pin_memory=True)\npredictions = inference(model, states, test_loader, device)\n\n# stack tta\nprediction = predictions.mean(1)\n\n\n# submission\ntest['label'] = prediction.argmax(1)    \ntest[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# debug"},{"metadata":{"trusted":true},"cell_type":"code","source":"def valid_inference(model, state, test_loader, device):\n    model.to(device)\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images, labels) in tk0:\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size, n_crops, c, h, w = images.size()\n        images = images.view(-1, c, h, w)\n        model.load_state_dict(state['model'])\n        model.eval()\n        with torch.no_grad():\n            y_preds = model(images).softmax(1)\n            y_preds = y_preds.view(batch_size, n_crops,-1)\n        avg_preds = y_preds.to('cpu').numpy()\n        probs.append(avg_preds)\n        del images, labels, y_preds, avg_preds\n        torch.cuda.empty_cache()\n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## oofごとのスコア"},{"metadata":{"trusted":true},"cell_type":"code","source":"if debug:\n#     train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv').head(100)\n    train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\n    folds = train.copy()\n    Fold = StratifiedKFold(n_splits=CFG['n_fold'], shuffle=True, random_state=CFG['seed'])\n    for n, (train_index, val_index) in enumerate(Fold.split(folds, folds[CFG['target_col']])):\n        folds.loc[val_index, 'fold'] = int(n)\n    folds['fold'] = folds['fold'].astype(int)\n    \n    model_paths = glob.glob(f'{MODEL_DIR}/*.pth')\n    model_paths.sort()\n    states = [torch.load(path) for path in model_paths]\n    \n    oof_df = pd.DataFrame()\n    oof_aug_preds = []\n    for fold, state in enumerate(states):\n        \n        # ====================================================\n        # loader\n        # ====================================================\n        val_idx = folds[folds['fold'] == fold].index\n        valid_folds = folds.loc[val_idx].reset_index(drop=True)\n        valid_dataset = TTADataset(valid_folds, TRAIN_PATH, ttas=ttas)\n        valid_loader = DataLoader(valid_dataset, \n                                  batch_size=inference_batch_size, \n                                  shuffle=False, \n                                  num_workers=CFG['num_workers'], pin_memory=True)\n        valid_preds = valid_inference(model, state, valid_loader, device)\n        valid_pred = valid_preds.mean(1)\n        valid_folds[[str(c) for c in range(5)]] = valid_pred\n        valid_folds['preds'] = valid_pred.argmax(1)\n        oof_df = pd.concat([oof_df, valid_folds])\n        oof_aug_preds.append(valid_preds)\n        LOGGER.info(f\"========== fold: {fold} result ==========\")\n        _ = get_result(valid_folds)\n    # total score\n    LOGGER.info(f\"========== CV result ==========\")\n    score = get_result(oof_df)\n    score_rem3 = get_result(oof_df.query('fold!=3'))\n    \n    oof_aug_preds = np.concatenate(oof_aug_preds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## ttaごとのscore"},{"metadata":{"trusted":true},"cell_type":"code","source":"if debug:\n    LOGGER.info(f\"========== augmentation result ==========\")\n    get_aug_score(oof_aug_preds, oof_df['label'], ttas)\n    get_aug_csv(oof_aug_preds, oof_df, ttas)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}