{"cells":[{"metadata":{},"cell_type":"markdown","source":"## 実行する場合は、アップコメントお願いします。"},{"metadata":{},"cell_type":"markdown","source":"# 概要\n* このノートでは、アンサンブルによるスコアアップを試みています。\n* データセットは非公開にしました。\n\n# ソースカーネル\n* このノートは以下の素晴らしいカーネルを参考にして書きましたので、 **check and upvote** を忘れずにお願いします。\n* [[No TTA] Cassava Resnext50_32x4d Inference lb0.903](https://www.kaggle.com/piantic/no-tta-cassava-resnext50-32x4d-inference-lb0-903/output)\n* [Clean_Inference_Kernel_8xTTA_LB902](https://www.kaggle.com/underwearfitting/clean-inference-kernel-8xtta-lb902/data)\n* [Cassava-ensemble-(efnetb3-resnet50)](https://www.kaggle.com/shubham108/cassava-ensemble-efnetb3-resnet50)"},{"metadata":{"trusted":true},"cell_type":"code","source":"#for efficientnet(画像認識モデルの1つ)\nBATCH_SIZE = 1\nimage_size = 512\n# pytorchの画像モデルのコンフィグ\nenet_type = ['tf_efficientnet_b4_ns'] * 5\nmodel_path = ['../input/cassava-models-eff/baseline_cld_fold0_epoch8_tf_efficientnet_b4_ns_512.pth', \n              '../input/cassava-models-eff/baseline_cld_fold1_epoch9_tf_efficientnet_b4_ns_512.pth', \n              '../input/cassava-models-eff/baseline_cld_fold2_epoch9_tf_efficientnet_b4_ns_512.pth',\n              '../input/cassava-models-eff/baseline_cld_fold3_epoch5_tf_efficientnet_b4_ns_512.pth',\n              '../input/cassava-models-eff/baseline_cld_fold4_epoch11_tf_efficientnet_b4_ns_512.pth']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n\nimport os\nimport math\nimport time\nimport random\nimport shutil\nimport albumentations\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nfrom scipy.special import softmax\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Transform for efficientnet\n# 参考：https://pystyle.info/pytorch-list-of-transforms/#outline__5_2\ntransforms_valid = albumentations.Compose([\n    albumentations.CenterCrop(image_size, image_size, p=1),  # CenterCrop:画像の中心を切り抜く\n    albumentations.Resize(image_size, image_size),           # Resize:リサイズを行う\n    albumentations.Normalize()                               # Normalize:正規化を行う\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ディレクトリ設定"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Directory settings for Resnext\n# ====================================================\nimport os\n\nOUTPUT_DIR = './'\nMODEL_DIR = '../input/cassava-models-res/'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n    \nTRAIN_PATH = '../input/cassava-leaf-disease-classification/train_images'\nTEST_PATH = '../input/cassava-leaf-disease-classification/test_images'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/cassava-models-res/","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CFG"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# CFG for Resnext\n# ====================================================\nclass CFG:    # CFG->コンフィグ？\n    debug=False\n    num_workers=8\n    model_name='resnext50_32x4d'\n    size=512\n    batch_size=32\n    seed=2020\n    target_size=5\n    target_col='label'\n    n_fold=5                   # 分割数\n    trn_fold=[0, 1, 2, 3, 4]\n    inference=True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ユーティリティー"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Utils for Resnext\n# ====================================================\n# 精度を取得\ndef get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)\n\n# 参考：https://docs.python.org/ja/3/library/contextlib.html#module-contextlib\n# 実行時間算出用関数\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')  # 実行時間算出\n\n# ログ出力の設定\ndef init_logger(log_file=OUTPUT_DIR+'inference.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\n#LOGGER = init_logger()\n\n# ランダムの固定化\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# データローディング"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\ntest['filepath'] = test.image_id.apply(lambda x: os.path.join('../input/cassava-leaf-disease-classification/test_images', f'{x}'))\n#test.head()\ntest","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# データセット"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Dataset for Resnext\n# ====================================================\nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    # イメージの取得\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Dataset for efficientnet\n# ====================================================\nclass CLDDataset(Dataset):\n    def __init__(self, df, mode, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.mode = mode\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        row = self.df.loc[index]\n        image = cv2.imread(row.filepath)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if self.transform is not None:\n            res = self.transform(image=image)\n            image = res['image']\n        \n        image = image.astype(np.float32)\n        image = image.transpose(2,0,1)\n        if self.mode == 'test':\n            return torch.tensor(image).float()\n        else:\n            return torch.tensor(image).float(), torch.tensor(row.label).float()\n\n#test = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\n#test_dataset = CLDDataset(test, 'test', transform=transforms_valid)\n#test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False,  num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for efficientnet\ntest_dataset_efficient = CLDDataset(test, 'test', transform=transforms_valid)\ntest_loader_efficient = torch.utils.data.DataLoader(test_dataset_efficient, batch_size=BATCH_SIZE, shuffle=False,  num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Transforms for Resnext\n# ====================================================\ndef get_transforms(*, data):\n    if data == 'valid':\n        return A.Compose([\n            A.Resize(CFG.size, CFG.size),\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MODELS"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# ResNext Model\n# ====================================================\nclass CustomResNext(nn.Module):\n    # resnextのモデルを作成し、最後に1層追加\n    def __init__(self, model_name='resnext50_32x4d', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# EfficientNet Model\n# ====================================================\nclass enet_v2(nn.Module):\n    # # EfficientNetのモデルを作成し、最後に1層追加\n    def __init__(self, backbone, out_dim, pretrained=False):\n        super(enet_v2, self).__init__()\n        self.enet = timm.create_model(backbone, pretrained=pretrained)\n        in_ch = self.enet.classifier.in_features\n        self.myfc = nn.Linear(in_ch, out_dim)\n        # 参考：https://pytorch.org/docs/stable/generated/torch.nn.Identity.html\n        # コピーを作る？？\n        self.enet.classifier = nn.Identity()\n\n    def forward(self, x):\n        x = self.enet(x)\n        x = self.myfc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Helper functions for Resnext\n# ====================================================\ndef load_state(model_path):\n    model = CustomResNext(CFG.model_name, pretrained=False)\n    try:  # single GPU model_file\n        # load_state_dict:モデルの読み出す\n        model.load_state_dict(torch.load(model_path)['model'], strict=True)\n        # torch.load: torch.save()で保存したオブジェクトをファイルから読み込みます。\n        state_dict = torch.load(model_path)['model']\n    except:  # multi GPU model_file\n        state_dict = torch.load(model_path)['model']\n        state_dict = {k[7:] if k.startswith('module.') else k: state_dict[k] for k in state_dict.keys()}\n\n    return state_dict\n\n\ndef inference(model, states, test_loader, device):\n    model.to(device)\n    # プログレスバー情報\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        # pytorchファイル数分ループ\n        for state in states:\n            # 読み出し\n            model.load_state_dict(state)\n            # eval()はdropoutやbatch normの on/offの切替\n            # 評価モードに設定\n            model.eval()\n            # 勾配計算を無効にしてメモリ消費を抑える\n            with torch.no_grad():\n                # ???\n                y_preds = model(images)\n            # ???\n            avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n        # 平均を算出\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    # 行→列に変換？\n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Helper functions for efficientnet\n# ====================================================\ndef inference_func(test_loader):\n    model.eval()\n    bar = tqdm(test_loader)\n\n    LOGITS = []\n    PREDS = []\n    \n    with torch.no_grad():\n        for batch_idx, images in enumerate(bar):\n            x = images.to(device)\n            logits = model(x)\n            LOGITS.append(logits.cpu())\n            PREDS += [torch.softmax(logits, 1).detach().cpu()]\n        PREDS = torch.cat(PREDS).cpu().numpy()\n        LOGITS = torch.cat(LOGITS).cpu().numpy()\n    return PREDS\n\ndef tta_inference_func(test_loader):\n    model.eval()\n    bar = tqdm(test_loader)\n    PREDS = []\n    LOGITS = []\n\n    with torch.no_grad():\n        for batch_idx, images in enumerate(bar):\n            x = images.to(device)\n            x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),\n            x.transpose(-1,-2),x.transpose(-1,-2).flip(-1),\n            x.transpose(-1,-2).flip(-2),x.transpose(-1,-2).flip(-1,-2)],0)\n            x = x.view(-1, 3, image_size, image_size)\n            logits = model(x)\n            logits = logits.view(BATCH_SIZE, 8, -1).mean(1)\n            PREDS += [torch.softmax(logits, 1).detach().cpu()]\n            LOGITS.append(logits.cpu())\n\n        PREDS = torch.cat(PREDS).cpu().numpy()\n        \n    return PREDS","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# inference and Submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# inference\n# ====================================================\n\n#for Resnext\n# モデルの生成\nmodel = CustomResNext(CFG.model_name, pretrained=False)\n#model = enet_v2(enet_type[i], out_dim=5)\n# 5種類のpytorchファイルを読み込む\nstates = [load_state(MODEL_DIR+f'{CFG.model_name}_fold{fold}.pth') for fold in CFG.trn_fold]\n# データセット作成\ntest_dataset = TestDataset(test, transform=get_transforms(data='valid'))\n# datasetsからバッチごとに取り出す\n# https://qiita.com/tomp/items/3bf6d040bbc89a171880\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, \n                         num_workers=CFG.num_workers, pin_memory=True)\npredictions = inference(model, states, test_loader, device)\n\n#for Efficientnet\ntest_preds = []\n# モデル数分ループ\nfor i in range(len(enet_type)):\n    model = enet_v2(enet_type[i], out_dim=5)\n    model = model.to(device)\n    model.load_state_dict(torch.load(model_path[i]))\n    test_preds += [tta_inference_func(test_loader_efficient)]\n\n# submission\npred = 0.5*predictions + 0.5*np.mean(test_preds, axis=0)\ntest['label'] = softmax(pred).argmax(1)\ntest[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}