{"cells":[{"metadata":{},"cell_type":"markdown","source":"# config"},{"metadata":{},"cell_type":"markdown","source":"## GPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"gpu_info = !nvidia-smi\ngpu_info = '\\n'.join(gpu_info)\nprint(gpu_info)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CFG"},{"metadata":{"trusted":true},"cell_type":"code","source":"CONFIG_NAME = 'stacking12.yml'\ndebug = False\n\nSTAGE2_DIR = '../input/train-stacking-2dcnn-ver3/output'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# CFG\n# ====================================================\nimport yaml\n\nCONFIG_PATH = f'{STAGE2_DIR}/{CONFIG_NAME}'\nwith open(CONFIG_PATH) as f:\n    config = yaml.load(f)\n\nINFO = config['info']\nTAG = config['tag']\nCFG = config['cfg']\n\nOUTPUT_DIR = './'\nDATA_PATH = '../input/cassava-leaf-disease-classification'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # ====================================================\n# # Directory settings\n# # ====================================================\n# import os\n# import glob\n\n# OUTPUT_DIR = './'\n# NORMAL_MODEL_DIRS = ['../input/06t-efficientnet-b4-ns-512',\n#                      '../input/12t-efficientnet-b5-cutout',\n#                      '../input/14t-seresnext50']\n# TTA_MODEL_DIRS = ['../input/20t-efficientnet-b3-cutmix-tta',\n#                   '../input/22t-efficientnet-b4-cutmix-tta']\n\n# MODEL_DIRS = NORMAL_MODEL_DIRS + TTA_MODEL_DIRS\n\n# MODEL_WEIGHTS = [0.13412473, 0.18325853, 0.18344057, 0.30333854, 0.19583763]\n\n# if not os.path.exists(OUTPUT_DIR):\n#     os.makedirs(OUTPUT_DIR)\n\n# TRAIN_PATH = '../input/cassava-leaf-disease-classification/train_images'\n# TEST_PATH = '../input/cassava-leaf-disease-classification/test_images'\n\n# normal_config_paths = []\n# for model_dir in NORMAL_MODEL_DIRS:\n#     assert len(glob.glob(f'{model_dir}/*.yml'))==1\n#     normal_config_paths.append(glob.glob(f'{model_dir}/*.yml')[0])\n    \n# tta_config_paths = []\n# for model_dir in TTA_MODEL_DIRS:\n#     assert len(glob.glob(f'{model_dir}/*.yml'))==1\n#     tta_config_paths.append(glob.glob(f'{model_dir}/*.yml')[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Library"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n\nimport datetime\nimport os\nimport math\nimport time\nfrom typing import Any, List, Optional\nimport random\nimport glob\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\nimport yaml\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose, CenterCrop\n    )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport timm\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\nif CFG['debug']:\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nelse:\n    device = torch.device('cuda')\n\nstart_time = datetime.datetime.now()\nstart_time_str = start_time.strftime('%m%d%H%M')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Directory settings"},{"metadata":{},"cell_type":"markdown","source":"## Data Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(f'{DATA_PATH}/train.csv')\ntest = pd.read_csv(f'{DATA_PATH}/sample_submission.csv')\nlabel_map = pd.read_json(f'{DATA_PATH}/label_num_to_disease_map.json', \n                         orient='index')\n\nif CFG['debug']:\n    train = train.sample(n=1000, random_state=CFG['seed']).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Outputs"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport glob\n\nmodel_dirs = []\nfor stage1 in CFG['stage1_models']:\n    num = str(stage1).rjust(2, '0')\n    output_dir_ = glob.glob(f'../input/{num}*/')\n    assert len(output_dir_) == 1, output_dir_\n    model_dirs.append(output_dir_[0])\nmodel_dirs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"normal_configs = []\ntta_configs = []\nnormal_model_dirs = []\ntta_model_dirs = []\n\nfor model_dir in model_dirs:\n    assert len(glob.glob(f'{model_dir}/*.yml'))==1\n    config_path = glob.glob(f'{model_dir}/*.yml')[0]\n    with open(config_path) as f:\n        config = yaml.load(f)\n    if 'valid_augmentation' in config['tag'].keys():\n        tta_model_dirs.append(model_dir)\n        tta_configs.append(config)\n    else:\n        normal_model_dirs.append(model_dir)\n        normal_configs.append(config)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)\n\n\ndef remove_glob(pathname, recursive=True):\n    for p in glob.glob(pathname, recursive=recursive):\n        if os.path.isfile(p):\n            os.remove(p)\n\n            \n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n\n# def init_logger(log_file=OUTPUT_DIR+'inference.log'):\n#     from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n#     logger = getLogger(__name__)\n#     logger.setLevel(INFO)\n#     handler1 = StreamHandler()\n#     handler1.setFormatter(Formatter(\"%(message)s\"))\n#     handler2 = FileHandler(filename=log_file)\n#     handler2.setFormatter(Formatter(\"%(message)s\"))\n#     logger.addHandler(handler1)\n#     logger.addHandler(handler2)\n#     return logger\n\n#LOGGER = init_logger()\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n# seed_torch(seed=CFG['seed'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# stage1"},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_PATH = '../input/cassava-leaf-disease-classification/train_images'\nTEST_PATH = '../input/cassava-leaf-disease-classification/test_images'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image\n    \n    \nclass TTADataset(Dataset):\n    def __init__(self, df, image_path, ttas):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.labels = df['label'].values\n        self.image_path = image_path\n        self.ttas = ttas\n\n    def __len__(self) -> int:\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{self.image_path}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        imglist=[tta(image=image)['image'] for tta in self.ttas]  # update\n\n        image=torch.stack(imglist)\n        label = torch.tensor(self.labels[idx]).long()\n        \n        return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _get_augmentations(aug_list, cfg):\n    process = []\n    for aug in aug_list:\n        if aug ==  'Resize':\n            process.append(Resize(cfg['size'], cfg['size']))\n        elif aug == 'RandomResizedCrop':\n            process.append(RandomResizedCrop(cfg['size'], cfg['size']))\n        elif aug == 'CenterCrop':\n            process.append(CenterCrop(CFG['size'], CFG['size']))\n        elif aug == 'Transpose':\n            process.append(Transpose(p=0.5))\n        elif aug == 'HorizontalFlip':\n            process.append(HorizontalFlip(p=0.5))\n        elif aug == 'VerticalFlip':\n            process.append(VerticalFlip(p=0.5))\n        elif aug == 'ShiftScaleRotate':\n            process.append(ShiftScaleRotate(p=0.5))\n        elif aug == 'Normalize':\n            process.append(Normalize(\n                            mean=[0.485, 0.456, 0.406],\n                            std=[0.229, 0.224, 0.225],\n                        ))\n        else:\n            raise ValueError(f'{aug} is not suitable')\n\n    process.append(ToTensorV2())\n\n    return process\n\n\n# Transforms\n# ====================================================\ndef get_transforms(*, aug_list, cfg):\n    \n    return Compose(\n        _get_augmentations(aug_list, cfg)\n    )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"[TODO] 違うTTAに対応する"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_ttas(cfg):\n    norm_mean = [0.485, 0.456, 0.406]\n    norm_std = [0.229, 0.224, 0.225]\n\n    oneof_augs = [\n        CenterCrop(cfg['size'], cfg['size']), \n        Resize(cfg['size'], cfg['size'])\n    ]\n\n    ttas = [[\n        Compose([\n            oneof_aug,\n            Normalize(mean=norm_mean, std=norm_std, p=1.),\n            ToTensorV2()\n        ]),\n        Compose([\n            oneof_aug,\n            Transpose(p=1),\n            Normalize(mean=norm_mean, std=norm_std, p=1.),\n            ToTensorV2()\n        ])\n    ] for oneof_aug in oneof_augs]\n\n    # 平滑化\n    ttas = sum(ttas, [])\n    \n    return ttas","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# MODEL\n# ====================================================\nclass CustomModel(nn.Module):\n    def __init__(self, model_name, target_size, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        if hasattr(self.model, 'classifier'):\n            n_features = self.model.classifier.in_features\n            self.model.classifier = nn.Linear(n_features, target_size)\n        elif hasattr(self.model, 'fc'):\n            n_features = self.model.fc.in_features\n            self.model.fc = nn.Linear(n_features, target_size)\n        elif hasattr(self.model, 'head'):\n            n_features = self.model.head.in_features\n            self.model.head = nn.Linear(n_features, target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## helper function"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\n# def inference_normal(model, states, test_loader, device):\n#     model.to(device)\n#     tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n#     probs = []\n#     for i, (images) in tk0:\n#         images = images.to(device)\n#         avg_preds = []\n#         for state in states:\n#             model.load_state_dict(state['model'])\n#             model.eval()\n#             with torch.no_grad():\n#                 y_preds = model(images)\n#             avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n#         avg_preds = np.mean(avg_preds, axis=0)\n#         probs.append(avg_preds)\n#     probs = np.concatenate(probs)\n#     return probs\n\n\ndef inference_tta(model, states, tta_loader, device):\n    model.to(device)\n    tk0 = tqdm(enumerate(tta_loader), total=len(tta_loader))\n    probs = []\n    for i, (images, _) in tk0:\n        images = images.to(device)\n        batch_size, n_crops, c, h, w = images.size()\n        images = images.view(-1, c, h, w)\n        \n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state['model'])\n            model.eval()\n            with torch.no_grad():\n                y_preds = model(images).softmax(1)\n                y_preds = y_preds.view(batch_size, n_crops,-1)\n            avg_preds.append(y_preds.to('cpu').numpy())\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n        del images, _, y_preds, avg_preds\n        torch.cuda.empty_cache()\n    probs = np.concatenate(probs)\n#     return probs.mean(1)\n    return probs\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"# def main_normal(config, model_dir):\n#     # ====================================================\n#     # inference\n#     # ====================================================\n    \n#     INFO = config['info']\n#     TAG = config['tag']\n#     CFG = config['cfg']\n#     CFG['train'] = False\n#     CFG['inference'] = True\n#     inference_batch_size = 64\n    \n#     seed_torch(seed=CFG['seed'])\n\n#     model = CustomModel(TAG['model_name'], CFG['target_size'], pretrained=False)\n#     states = [torch.load(path) for path in glob.glob(f'{model_dir}/*.pth')]\n#     test_dataset = TestDataset(test, transform=get_transforms(aug_list=['Resize', 'Normalize'], cfg=CFG))\n#     test_loader = DataLoader(test_dataset, batch_size=inference_batch_size, shuffle=False, \n#                              num_workers=CFG['num_workers'], pin_memory=True)\n#     predictions = inference_normal(model, states, test_loader, device)\n    \n#     return predictions\n\n\n\ndef main_tta(config, model_dir):\n    # ====================================================\n    # inference\n    # ====================================================\n    \n    INFO = config['info']\n    TAG = config['tag']\n    CFG = config['cfg']\n    CFG['train'] = False\n    CFG['inference'] = True\n    inference_batch_size = 8\n    \n    seed_torch(seed=CFG['seed'])\n\n    model = CustomModel(TAG['model_name'], CFG['target_size'], pretrained=False)\n    states = [torch.load(path) for path in glob.glob(f'{model_dir}/*.pth')]\n    ttas = get_ttas(CFG)\n    tta_dataset = TTADataset(test, TEST_PATH, ttas=ttas)\n    tta_loader = DataLoader(tta_dataset, batch_size=inference_batch_size, shuffle=False, \n                             num_workers=2, pin_memory=True)\n    predictions = inference_tta(model, states, tta_loader, device)\n    \n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_num = len(test)\nmodel_num = len(model_dirs)\ntarget_num = CFG['target_size']\nchannel_num = 4\n\n# [Models, N, Labels]\n# stage1_predictions = np.zeros((model_num, data_num, target_num), dtype=np.float)\n# [Models, N, Channel, Labels]\nstage1_predictions = np.zeros((model_num, data_num, channel_num, target_num), dtype=np.float)\n# for config, model_dir in zip(normal_configs, normal_model_dirs):\n#     stage1_predictions[model_dirs.index(model_dir)] = main_normal(config, model_dir)\nfor config, model_dir in zip(tta_configs, tta_model_dirs):\n    stage1_predictions[model_dirs.index(model_dir)] = main_tta(config, model_dir)\n    \n    \n# # [Models, N, Labels] -> [N, Models, Labels]\n# stage1_predictions = stage1_predictions.transpose(1, 0, 2)\n# # add Channel dim\n# stage1_predictions = stage1_predictions.reshape(data_num, model_num, target_num, 1)\n\n# [N, Models, Labels, Channel] -> [N, Channel, Models, Labels]\n# stage1_predictions = stage1_predictions.transpose(0, 3, 1, 2)\n\n\n# [Models, N, Channel, Labels] -> [N, Channel, Models, Labels]\nstage1_predictions = stage1_predictions.transpose(1, 2, 0, 3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# stage2"},{"metadata":{"trusted":true},"cell_type":"code","source":"class StackingDataset(Dataset):\n    def __init__(self, X: np.ndarray, y: Optional[np.ndarray] = None):\n        self.X = X\n        self.y = y\n\n    def __len__(self):\n        return self.X.shape[0]\n\n    def __getitem__(self, idx):\n        if self.y is None:\n            return torch.tensor(self.X[idx], dtype=torch.float)\n        else:\n            return (\n                torch.tensor(self.X[idx], dtype=torch.float),\n                torch.tensor(self.y[idx], dtype=torch.long),\n            )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CNNStacking(nn.Module):\n    def __init__(self, n_labels):\n        super(CNNStacking, self).__init__()\n\n        self.sq = nn.Sequential(\n            nn.Conv2d(in_channels=4, out_channels=8, kernel_size=(3, 1), bias=False),\n            nn.ReLU(),\n            nn.Conv2d(in_channels=8, out_channels=16, kernel_size=(3, 1), bias=False),\n            nn.ReLU(),\n#             nn.Conv2d(in_channels=16, out_channels=32, kernel_size=(2, 1), bias=False),\n#             nn.ReLU(),\n            # nn.Conv2d(in_channels=32, out_channels=64, kernel_size=(3, 1), bias=False),\n            # nn.ReLU(),\n            nn.Flatten(),\n#             nn.Linear(in_features=32* n_labels, out_features=16 * n_labels),\n#             nn.ReLU(),\n            nn.Linear(in_features=16* n_labels, out_features=4 * n_labels),\n            nn.ReLU(),\n            nn.Linear(in_features=4 * n_labels, out_features=n_labels),\n        )\n\n    def forward(self, x):\n        return self.sq(x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## helper function"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\ndef inference(model, states, test_loader, device):\n    model.to(device)\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (features) in tk0:\n        features = features.to(device)\n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state['model'])\n#             model.load_state_dict(state)\n            model.eval()\n            with torch.no_grad():\n                y_preds = model(features)\n            avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# inference\nmodel = CNNStacking(CFG['target_size'])\nstates = [torch.load(STAGE2_DIR+f'/fold{fold}_best.pth') for fold in CFG['trn_fold']]\ntest_dataset = StackingDataset(stage1_predictions)\ntest_loader = DataLoader(test_dataset, batch_size=CFG['batch_size'], shuffle=False, \n                         num_workers=CFG['num_workers'], pin_memory=True)\npredictions = inference(model, states, test_loader, device)\n# submission\ntest['label'] = predictions.argmax(1)\ntest[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}