{"cells":[{"metadata":{"papermill":{"duration":0.012113,"end_time":"2021-01-31T18:09:14.000657","exception":false,"start_time":"2021-01-31T18:09:13.988544","status":"completed"},"tags":[]},"cell_type":"raw","source":"# Ensemble Kaggle推論用\n- ver01: resnet50-04-2019, eb7-00-baseline, densenet201-04-2019data LB: 0.899\n- ver02: resnet50-04-2019, eb7-00-baseline                          LB: 0.898\n- ver03: resnet50-04-2019, eb7-00-baseline, resnet152-04-2019data   LB: 0.899 (～8時間)\n- ver04: densenet201-04-2019data, resnet152-04-2019data             LB: 0.900\n- ver05: densenet201-04-2019data, resnet152-04-2019data, eb7-00-baseline LB: 0.902\n- ver06: densenet201-04-2019data, resnet152-04-2019data, eb7-seed70 LB: 0.903\n- ver07: densenet201-04-2019data, eb7m-seed70 LB: 0.903\n- ver08: resnet152-04-2019data,   eb7m-seed70 LB: 0.901\n- ver09～ver13: AMPは遅くなったのでやめる。BATCH_SIZE変更(少し早くなった？)。、テストデータサイズ変更(32, 早くならず)\n- ver14: ver07と同じでバッチサイズが 16  LB: 0.900 \n- ver15: バッチサイズを ver07 に戻す。\n- ver16: densenet201-seed60, eb7m-seed70  LB: 0.901\n- ver17: eb7m-seed70  LB:0.895\n- ver19: efficientnet-b7sl_SEED70.best LB: 0.898\n- ver20: efficientnet-b7sl_SEED70.orig LB: 0.895\n- ver21: densenet201-04-2019data, efficientnet-b7sl_SEED70.best  LB: 0.900\n- ver22: densenet201-04-2019data, efficientnet-b7sl_SEED70.orig  LB: 0.898\n- ver23: densenet201-seed60, efficientnet-b7sl_SEED70.best LB: 0.900\n- ver28: densenet201-04-2019data, eb7m-seed70, tta5m  LB: 0.899\n- ver30: densenet201-04-2019data, eb7m-seed70, efficientnet-b7sl_SEED70.best  # timeout\n- ver33: densenet201-04-2019data, eb7m-seed70, efficientnet-b7sl_SEED70.best tta減らす  LB: 0.901\n- ver34: densenet201-04-2019data, eb7m-seed70, eb7mseed71 tta減らす  LB:0.899\n- ver35: ver07と同じで tta 減らす LB:0.901\n- ver37: ver07と同じで tta に Rotate+CenterCrop 追加 LB:0.901\n- ver40: ebmls-seed70, eb7mseed71 LB:0.897\n- ver41: ebmls-seed70 LB: 0.889 (削除)\n- ver42: eb7mseed71   LB: 0.895\n- ver47: densenet201-04-2019data, resnet152-04-2019data, efficientnet-b7sl_SEED70.best LB:0.902\n- ver48: densenet201-04-2019data, resnet152-04-2019data, efficientnet-b7sl_SEED70.best LB:0.902\n- ver49: densenet201-04-2019data, eb7m-seed70 (TTA削減:6→5)\n- ver50: densenet201-04-2019data, resnet152-04-2019data, eb7m-seed70, efficientnet-b7sl_SEED70.best, AMP\n- ver51: efntb7の4モデル"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport glob","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:09:18.019092Z","iopub.status.busy":"2021-01-31T18:09:18.018559Z","iopub.status.idle":"2021-01-31T18:09:18.069637Z","shell.execute_reply":"2021-01-31T18:09:18.070379Z"},"papermill":{"duration":0.068509,"end_time":"2021-01-31T18:09:18.070518","exception":false,"start_time":"2021-01-31T18:09:18.002009","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# ver01 pretrained_models = glob.glob(f'../input/resnet50-04-2019/*.pth') + glob.glob(f'../input/eb7-00-baseline/*.pth') + glob.glob(f'../input/densenet201-04-2019data/*.pth')\n# ver02 pretrained_models = glob.glob(f'../input/resnet50-04-2019/*.pth') + glob.glob(f'../input/eb7-00-baseline/*.pth')\n# ver03 pretrained_models = glob.glob(f'../input/resnet50-04-2019/*.pth') + glob.glob(f'../input/resnet152-04-2019data/*.pth') + glob.glob(f'../input/eb7-00-baseline/*.pth')\n# ver04 pretrained_models = glob.glob(f'../input/densenet201-04-2019data/*.pth') + glob.glob(f'../input/resnet152-04-2019data/*.pth')\n# ver05 pretrained_models = glob.glob(f'../input/densenet201-04-2019data/*.pth') + glob.glob(f'../input/resnet152-04-2019data/*.pth') + glob.glob(f'../input/eb7-00-baseline/*.pth')\n# ver06 pretrained_models = glob.glob(f'../input/densenet201-04-2019data/*.pth') + glob.glob(f'../input/resnet152-04-2019data/*.pth') + glob.glob(f'../input/eb7-seed70/*.pth')\n# ver07 pretrained_models = glob.glob(f'../input/densenet201-04-2019data/*.pth') + glob.glob(f'../input/eb7m-seed70/*.pth')\n# ver08 pretrained_models = glob.glob(f'../input/resnet152-04-2019data/*.pth') + glob.glob(f'../input/eb7m-seed70/*.pth')\n# ver16 pretrained_models = glob.glob(f'../input/densenet201-seed60/*.pth') + glob.glob(f'../input/eb7m-seed70/*.pth')\n# ver17 pretrained_models = glob.glob(f'../input/eb7m-seed70/*.pth')\n# ver19 pretrained_models = glob.glob(f'../input/eb7slseed70/efficientnet-b7sl_SEED70.best/*.pth')\n# ver20 pretrained_models = glob.glob(f'../input/eb7slseed70/efficientnet-b7sl_SEED70.orig/*.pth')\n# ver21 pretrained_models = glob.glob(f'../input/densenet201-04-2019data/*.pth') + glob.glob(f'../input/eb7slseed70/efficientnet-b7sl_SEED70.best/*.pth')\n# ver22 pretrained_models = glob.glob(f'../input/densenet201-04-2019data/*.pth') + glob.glob(f'../input/eb7slseed70/efficientnet-b7sl_SEED70.orig/*.pth')\n# ver23 pretrained_models = glob.glob(f'../input/densenet201-seed60/*.pth') + glob.glob(f'../input/eb7slseed70/efficientnet-b7sl_SEED70.best/*.pth')\n# ver28 pretrained_models = glob.glob(f'../input/densenet201-04-2019data/*.pth') + glob.glob(f'../input/eb7m-seed70/*.pth')\n# ver33 pretrained_models = glob.glob(f'../input/densenet201-04-2019data/*.pth') + glob.glob(f'../input/eb7m-seed70/*.pth') + glob.glob(f'../input/eb7slseed70/efficientnet-b7sl_SEED70.best/*.pth')\n# ver34 pretrained_models = glob.glob(f'../input/densenet201-04-2019data/*.pth') + glob.glob(f'../input/eb7m-seed70/*.pth') + glob.glob(f'../input/eb7mseed71/*.pth')\n# ver38 pretrained_models = glob.glob(f'../input/ebmls-seed70/*.pth') + glob.glob(f'../input/eb7mseed71/*.pth')\n# ver41 pretrained_models = glob.glob(f'../input/ebmls-seed70/*.pth')\n# ver42 pretrained_models = glob.glob(f'../input/eb7mseed71/*.pth')\n# ver47, ver48 pretrained_models = glob.glob(f'../input/densenet201-04-2019data/*.pth') + glob.glob(f'../input/resnet152-04-2019data/*.pth') + glob.glob(f'../input/eb7slseed70/efficientnet-b7sl_SEED70.best/*.pth')\n# ver50 pretrained_models = glob.glob(f'../input/densenet201-04-2019data/*.pth') + glob.glob(f'../input/resnet152-04-2019data/*.pth') + glob.glob(f'../input/eb7m-seed70/*.pth') + glob.glob(f'../input/eb7slseed70/efficientnet-b7sl_SEED70.best/*.pth')\n# ver51\n#pretrained_models = glob.glob(f'../input/efntb7/efficientnet-b7m_SEED70/*.pth') + glob.glob(f'../input/efntb7/efficientnet-b7m_SEED72/*.pth') + glob.glob(f'../input/efntb7/efficientnet-b7m_SEED73/*.pth') + glob.glob(f'../input/efntb7/efficientnet-b7sl_SEED71.pretrained/*.pth')\n# ver59\npretrained_models = glob.glob(f'../input/densenet201-04-2019data/*.pth') + glob.glob(f'../input/eb7m-seed7x/efficientnet-b7m_SEED70/*.pth') + glob.glob(f'../input/eb7m-seed7x/efficientnet-b7m_SEED72/*.pth') + glob.glob(f'../input/eb7m-seed7x/efficientnet-b7m_SEED73/*.pth') + glob.glob(f'../input/eb7m-seed7x/efficientnet-b7sl_SEED71.pretrained/*.pth')\n\n\nprint(f'{len(pretrained_models)} models found.')\nprint('\\n'.join(np.sort(pretrained_models)))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:09:14.033009Z","iopub.status.busy":"2021-01-31T18:09:14.032305Z","iopub.status.idle":"2021-01-31T18:09:17.532102Z","shell.execute_reply":"2021-01-31T18:09:17.5316Z"},"papermill":{"duration":3.520699,"end_time":"2021-01-31T18:09:17.532261","exception":false,"start_time":"2021-01-31T18:09:14.011562","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import pandas as pd\n\n#\nimport torch\nimport torch.nn as nn\nimport torch.utils.data as data\n#import timm\n\n# Importing Libraries for Image Augmentations\nimport torchvision\nfrom torchvision import models, transforms  # 学習済みモデル、画像変換\nimport albumentations as A\nfrom albumentations import Compose\nfrom albumentations.pytorch import ToTensorV2\n\n# Working with Files\nimport os\nfrom pathlib import Path\nimport random\nimport json\nimport time\nimport pickle\n\n\n# Fancy progress bar\nfrom tqdm import tqdm\n\n# Static Graphs\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Working with images\nimport cv2\n\n\n# 乱数シードの固定\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nSEED = 42\nseed_everything(seed=SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append(\"/kaggle/input/package/EfficientNet-PyTorch-1.0\")\nfrom efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:09:17.559546Z","iopub.status.busy":"2021-01-31T18:09:17.558851Z","iopub.status.idle":"2021-01-31T18:09:17.561709Z","shell.execute_reply":"2021-01-31T18:09:17.561306Z"},"papermill":{"duration":0.017744,"end_time":"2021-01-31T18:09:17.561806","exception":false,"start_time":"2021-01-31T18:09:17.544062","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"SIZE = 512        # image size\nnum_classes = 5","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:09:17.934102Z","iopub.status.busy":"2021-01-31T18:09:17.933202Z","iopub.status.idle":"2021-01-31T18:09:17.936821Z","shell.execute_reply":"2021-01-31T18:09:17.937356Z"},"papermill":{"duration":0.364771,"end_time":"2021-01-31T18:09:17.93751","exception":false,"start_time":"2021-01-31T18:09:17.572739","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"device = ('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'使用デバイス: {device}')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:09:17.969763Z","iopub.status.busy":"2021-01-31T18:09:17.969222Z","iopub.status.idle":"2021-01-31T18:09:17.980911Z","shell.execute_reply":"2021-01-31T18:09:17.98007Z"},"papermill":{"duration":0.030299,"end_time":"2021-01-31T18:09:17.981051","exception":false,"start_time":"2021-01-31T18:09:17.950752","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# テスト画像ファイル名(image_id)のリスト\nif os.getenv('KAGGLE_KERNEL_RUN_TYPE') == 'Interactive':\n    # Kaggle環境でテスト (train_images の32ファイルを使用して実行)\n    print(\"Test run in Kaggle environment.\")\n    BASE_DIR = \"../input/cassava-leaf-disease-classification\"\n    TEST_PATH = f'{BASE_DIR}/train_images'\n    test_files = os.listdir(f'{TEST_PATH}/')[:32]\nelif os.getenv('KAGGLE_KERNEL_RUN_TYPE') == 'Batch':\n    print(\"In Kaggle environment.\")\n    BASE_DIR = \"../input/cassava-leaf-disease-classification\"\n    TEST_PATH = f'{BASE_DIR}/test_images'\n    test_files = os.listdir(f'{TEST_PATH}/')\nelse:\n    # ローカル環境\n    print(\"In the local environment.\")\n    BASE_DIR = \"data\"\n    TEST_PATH = f'{BASE_DIR}/train_images'\n    test_files = os.listdir(f'{TEST_PATH}/')[:32]     # glob.glob() と違ってファイル名だけが入る\n    PRETRAINED_MODEL_PATH=\"1610594484mf/bak/00個別5fold計算\"\n\nprint(f\"Number of test  images: {len(test_files)}\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:09:18.106823Z","iopub.status.busy":"2021-01-31T18:09:18.106049Z","iopub.status.idle":"2021-01-31T18:09:18.108821Z","shell.execute_reply":"2021-01-31T18:09:18.108359Z"},"papermill":{"duration":0.025462,"end_time":"2021-01-31T18:09:18.1089","exception":false,"start_time":"2021-01-31T18:09:18.083438","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# 予測結果を格納する DataFrame\ndf_test = pd.DataFrame(test_files, columns=['image_id'])\ndf_test['label'] = 1","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:09:18.144556Z","iopub.status.busy":"2021-01-31T18:09:18.143988Z","iopub.status.idle":"2021-01-31T18:09:18.146737Z","shell.execute_reply":"2021-01-31T18:09:18.147202Z"},"papermill":{"duration":0.026078,"end_time":"2021-01-31T18:09:18.147307","exception":false,"start_time":"2021-01-31T18:09:18.121229","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if len(df_test) == 1:\n    df_test.loc[1] = df_test.loc[0]\n    print(df_test)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:09:18.184839Z","iopub.status.busy":"2021-01-31T18:09:18.184159Z","iopub.status.idle":"2021-01-31T18:09:18.187096Z","shell.execute_reply":"2021-01-31T18:09:18.186624Z"},"papermill":{"duration":0.027316,"end_time":"2021-01-31T18:09:18.187213","exception":false,"start_time":"2021-01-31T18:09:18.159897","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# tta5\nmean = [0.485, 0.456, 0.406]\nstd  = [0.229, 0.224, 0.225]\n\ntransform = {\n    'test': [\n        Compose([\n            A.CenterCrop(SIZE, SIZE),\n            A.Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.),\n        Compose([\n            A.HorizontalFlip(p=1),\n            A.CenterCrop(SIZE, SIZE),\n            A.Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.),\n        Compose([\n            A.RandomResizedCrop(SIZE, SIZE),\n            A.Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.),\n#        Compose([\n#            A.RandomResizedCrop(SIZE, SIZE),\n#            A.HorizontalFlip(p=0.5),\n#            A.Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),\n#            ToTensorV2(p=1.0),\n#        ], p=1.),\n#        Compose([\n#            A.RandomResizedCrop(SIZE, SIZE),\n#            A.VerticalFlip(p=1),\n#            A.Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),\n#            ToTensorV2(p=1.0),\n#        ], p=1.),\n#        Compose([\n#            A.Rotate(p=1),\n#            A.RandomResizedCrop(SIZE, SIZE),\n#            A.Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),\n#            ToTensorV2(p=1.0),\n#        ], p=1.),\n        Compose([\n            A.Rotate(p=1),\n            A.CenterCrop(SIZE, SIZE),\n            A.Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.),\n    ]\n}","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.013952,"end_time":"2021-01-31T18:09:18.279492","exception":false,"start_time":"2021-01-31T18:09:18.26554","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 最終層前で mixup を行うモデル\n- 学習済みモデルを読み込み、畳み込み層と最終層を分ける\n- 最終層の出力サイズを変える\n- 畳み込み層の後で mixup を行う"},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:09:18.324759Z","iopub.status.busy":"2021-01-31T18:09:18.321127Z","iopub.status.idle":"2021-01-31T18:09:18.327387Z","shell.execute_reply":"2021-01-31T18:09:18.326948Z"},"papermill":{"duration":0.033507,"end_time":"2021-01-31T18:09:18.327477","exception":false,"start_time":"2021-01-31T18:09:18.29397","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# 最終層前で mixup を行うモデル\n\nclass FinalLayerMixupModel(nn.Module):\n    def __init__(self, model, criterion, num_classes, alpha):\n        '''\n        model: 学習済みモデルを指定\n        '''\n        super(FinalLayerMixupModel, self).__init__()\n        self.convlayer = torch.nn.Sequential(*(list(model.children())[:-1]))\n        num_ftrs = model.fc.in_features\n        self.fc = nn.Linear(num_ftrs, num_classes)\n        self.criterion = criterion\n        self.alpha = alpha\n        \n    def forward(self, inputs, labels, phase):\n        # vailidation時\n        if phase == 'val':\n            x = self.convlayer(inputs)\n            x = x.squeeze()\n            outputs = self.fc(x)\n            loss = self.criterion(outputs, labels)\n            \n            return outputs, loss\n\n        # test(prediction)時\n        if phase == 'test':\n            x = self.convlayer(inputs)\n            x = x.squeeze()\n            outputs = self.fc(x)\n            \n            return outputs\n        \n        # train時\n        alpha = self.alpha\n        if alpha > 0:\n            lam = np.random.beta(alpha, alpha)\n        else:\n            lam = 1\n\n        #index = torch.randperm(len(labels)).to(device)\n        index = torch.randperm(len(labels))\n\n        x1 = inputs             # torch.Size([64, 3, 256, 256])\n        x2 = inputs[index]      # torch.Size([64, 3, 256, 256])\n\n        x1 = self.convlayer(x1) # torch.Size([64, 512, 1, 1])\n        x2 = self.convlayer(x2) # torch.Size([64, 512, 1, 1])\n        \n        # 畳み込み層の結果を mix\n        mixed_x = lam * x1 + (1 - lam) * x2  # torch.Size([64, 512, 1, 1])\n        mixed_x = mixed_x.squeeze()          # torch.Size([64, 512])\n        outputs = self.fc(mixed_x)           # torch.Size([64, 5])\n        \n        labels_a = labels\n        labels_b = labels[index]\n        \n        # loss の計算\n        pred = outputs\n        loss = lam * self.criterion(pred, labels_a) + (1 - lam) * self.criterion(pred, labels_b)\n        \n        return outputs, loss, labels_a, labels_b, lam","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 最終層前で mixup を行うモデル\n# - 学習済みモデルを読み込み、畳み込み層と最終層を分ける\n# - 最終層の出力サイズを変える\n# - 畳み込み層の後で mixup を行う\n\nclass FinalLayerMixupModelDenseNet(nn.Module):\n    def __init__(self, model, criterion, num_classes, alpha):\n        '''\n        model: 学習済みモデルを指定\n        '''\n        super(FinalLayerMixupModelDenseNet, self).__init__()\n        self.convlayer = model.features\n        self.AdaptiveAvgPool2d = nn.AdaptiveAvgPool2d(output_size=(1, 1))\n        num_ftrs = model.classifier.in_features\n        self.fc = nn.Linear(num_ftrs, num_classes)\n        self.criterion = criterion\n        self.alpha = alpha\n        \n    def forward(self, inputs, labels, phase):\n        # vailidation時\n        if phase == 'val':\n            x = self.convlayer(inputs)\n            x = self.AdaptiveAvgPool2d(x)\n            x = x.squeeze()\n            outputs = self.fc(x)\n            loss = self.criterion(outputs, labels)\n\n            return outputs, loss\n\n        # test(prediction)時\n        if phase == 'test':\n            x = self.convlayer(inputs)\n            x = self.AdaptiveAvgPool2d(x)\n            x = x.squeeze()\n            outputs = self.fc(x)\n\n            return outputs\n        \n        # train時\n        alpha = self.alpha\n        if alpha > 0:\n            lam = np.random.beta(alpha, alpha)\n        else:\n            lam = 1\n\n        #index = torch.randperm(len(labels)).to(device)\n        index = torch.randperm(len(labels))\n\n        x1 = inputs             # torch.Size([12, 3, 512, 512])\n        x2 = inputs[index]      # torch.Size([12, 3, 512, 512])\n\n        x1 = self.convlayer(x1) # torch.Size([12, 1920, 16, 16])\n        x2 = self.convlayer(x2) # torch.Size([12, 1920, 16, 16])\n\n        x1 = self.AdaptiveAvgPool2d(x1)  # torch.Size([12, 1920, 1, 1])\n        x2 = self.AdaptiveAvgPool2d(x2)  # torch.Size([12, 1920, 1, 1])\n\n        # 畳み込み層の結果を mix\n        mixed_x = lam * x1 + (1 - lam) * x2  # torch.Size([64, 1920, 1, 1])\n        mixed_x = mixed_x.squeeze()          # torch.Size([64, 1920])\n        outputs = self.fc(mixed_x)           # torch.Size([64, 5])\n\n        labels_a = labels\n        labels_b = labels[index]\n\n        # loss の計算\n        pred = outputs\n        loss = lam * self.criterion(pred, labels_a) + (1 - lam) * self.criterion(pred, labels_b)\n\n        return outputs, loss, labels_a, labels_b, lam","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FinalLayerMixupModelEN(nn.Module):\n    def __init__(self, model, criterion, num_classes, alpha):\n        super(FinalLayerMixupModelEN, self).__init__()\n            \n        num_ftrs = model._fc.in_features\n        model._fc = nn.Linear(num_ftrs, num_classes)   \n        \n        self.model = model\n        self.criterion = criterion\n        \n    def forward(self, inputs, labels, phase):\n        # vailidation時\n        if phase == 'val':\n            outputs = self.model(inputs)\n            loss = self.criterion(outputs, labels)\n\n            return outputs, loss\n\n        # test(prediction)時\n        if phase == 'test':\n            outputs = self.model(inputs)\n\n            return outputs\n        \n        # train時\n\n        print('ここにきてはいけない')\n        sys.exit()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.013399,"end_time":"2021-01-31T18:09:18.356422","exception":false,"start_time":"2021-01-31T18:09:18.343023","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 推論"},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:09:18.392948Z","iopub.status.busy":"2021-01-31T18:09:18.392209Z","iopub.status.idle":"2021-01-31T18:09:18.395055Z","shell.execute_reply":"2021-01-31T18:09:18.394649Z"},"papermill":{"duration":0.024812,"end_time":"2021-01-31T18:09:18.395175","exception":false,"start_time":"2021-01-31T18:09:18.370363","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# 推論用データセット\n\nclass TestDataset(data.Dataset):\n    def __init__(self, df, transform=None):\n        super().__init__()\n\n        self.image_ids = df.image_id.tolist()\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.image_ids)\n    \n    def load_image(self, image_id):\n        img = cv2.imread(f'{TEST_PATH}/{image_id}')  # (H, W, C) の numpy.ndarray\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)              # BGR => RGB に変換\n        return img\n    \n    def __getitem__(self, index):\n        image_id = self.image_ids[index]\n        \n        img = self.load_image(image_id)\n        \n        if self.transform:\n            img = self.transform(image=img)['image']\n        \n        return img, image_id","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:09:18.433357Z","iopub.status.busy":"2021-01-31T18:09:18.432576Z","iopub.status.idle":"2021-01-31T18:09:18.434856Z","shell.execute_reply":"2021-01-31T18:09:18.435516Z"},"papermill":{"duration":0.025918,"end_time":"2021-01-31T18:09:18.435655","exception":false,"start_time":"2021-01-31T18:09:18.409737","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# モデルで推論を行う関数\ndef predict_model (basename, net, dataloader):\n    '''\n    basename: 学習済みモデル名\n    net     : 学習済みモデル\n    '''\n\n    model_start_time = time.time()\n    \n    # ネットワークをGPUへ\n    net.to(device)\n    net.eval()    # 検証モード\n    torch.set_grad_enabled(False)\n    \n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \n    #corrects = 0    # 正解数\n    #count = 0       # 処理したデータ数\n    \n    #df_test[basename] = np.nan\n    \n    probability = []\n    \n    #scaler = torch.cuda.amp.GradScaler()\n    \n    for phase in ['test']:\n        progress = tqdm(dataloader[phase], desc=f\"{basename}: \")\n\n        # データローダーからミニバッチを取り出すループ\n        for inputs, image_ids in progress:\n            # GPUにデータを転送\n            inputs = inputs.to(device)\n\n            # forward計算\n            with torch.cuda.amp.autocast():\n                outputs = net(inputs, False, 'test')\n                _, preds = torch.max(outputs, 1)  # ラベルを予測\n\n                probability.append(torch.softmax(outputs, dim=1).cpu().numpy())\n\n            #for i, image_id in enumerate(image_ids):\n            #    df_test.loc[df_test.image_id == image_id, basename] = preds[i].item()\n\n    #df_test[basename] = df_test[basename].astype(np.int64)\n    \n    print(f'{basename} time: {time.time() - model_start_time:.2f}[sec]')\n\n    return np.concatenate(probability)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:09:18.474612Z","iopub.status.busy":"2021-01-31T18:09:18.474071Z","iopub.status.idle":"2021-01-31T18:10:26.596386Z","shell.execute_reply":"2021-01-31T18:10:26.597341Z"},"papermill":{"duration":68.1469,"end_time":"2021-01-31T18:10:26.597528","exception":false,"start_time":"2021-01-31T18:09:18.450628","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"probability = []\n\nstart_time = time.time()\n\nfor pretrained_model in pretrained_models:\n    basename = os.path.splitext(os.path.basename(pretrained_model))[0]\n\n    # model\n\n    criterion = nn.CrossEntropyLoss()\n\n    if 'resnet18' in basename:\n        MODEL_NAME = 'resnet18'\n        net = models.resnet18(pretrained=False)\n        net = FinalLayerMixupModel(net, criterion, num_classes, False)\n        BATCH_SIZE = 64\n    elif 'resnet50' in basename:\n        MODEL_NAME = 'resnet50'\n        net = models.resnet50(pretrained=False)\n        net = FinalLayerMixupModel(net, criterion, num_classes, False)\n        BATCH_SIZE = 32\n    elif 'resnet152' in basename:\n        MODEL_NAME = 'resnet152'\n        net = models.resnet152(pretrained=False)\n        net = FinalLayerMixupModel(net, criterion, num_classes, False)\n        BATCH_SIZE = 16\n    elif 'resnext101' in basename:\n        MODEL_NAME = 'resnext101'\n        net = models.resnext101_32x8d(pretrained=False)\n        net = FinalLayerMixupModel(net, criterion, num_classes, False)\n        BATCH_SIZE = 12    #  3063MiB\n    elif 'densenet201' in basename:\n        MODEL_NAME = 'densenet201'\n        net = models.densenet201(pretrained=False)\n        net = FinalLayerMixupModelDenseNet(net, criterion, num_classes, False)\n        BATCH_SIZE = 12\n    elif 'efficientnet-b7' in basename:\n        MODEL_NAME = 'efficientnet-b7'\n        net = EfficientNet.from_name(MODEL_NAME)\n        net = FinalLayerMixupModelEN(net, criterion, num_classes, False)\n        BATCH_SIZE = 10\n    else:\n        print(f'{basename} is not supported.')\n        sys.exit()\n    \n    print(f'{basename}: {MODEL_NAME}')    \n    \n    if MODEL_NAME == 'efficientnet-b7':\n        #net.model.load_state_dict(torch.load(pretrained_model))\n        if 'b7m_SEED' in pretrained_model:\n            net.model.load_state_dict(torch.load(pretrained_model))\n        else:\n            net.load_state_dict(torch.load(pretrained_model))\n    else:\n        net.load_state_dict(torch.load(pretrained_model))\n        \n    \n    for param in net.parameters():\n        param.requires_grad = False\n        \n    # TTA\n    for tid, transform_ in enumerate(transform['test']):\n        print(f'transform loop={tid}')\n        dataset = {\n            'test': TestDataset(df_test, transform=transform_),\n        }\n        dataloader = {\n            'test': torch.utils.data.DataLoader(dataset['test'], batch_size=BATCH_SIZE, shuffle=False, num_workers=8, pin_memory=True),\n        }\n\n        proba = predict_model(basename, net, dataloader)\n        probability.append(proba)\n    \n    #\n    \n    del net\n    # キャッシュクリア\n    torch.cuda.empty_cache()\n    \n    #break\n    \ndf_test['mean'] = np.array(probability).mean(axis=0).argmax(axis=1)\n\nprint(f'total time: {time.time() - start_time:.2f}[sec]')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:10:26.897145Z","iopub.status.busy":"2021-01-31T18:10:26.896191Z","iopub.status.idle":"2021-01-31T18:10:26.903702Z","shell.execute_reply":"2021-01-31T18:10:26.9032Z"},"papermill":{"duration":0.133874,"end_time":"2021-01-31T18:10:26.903807","exception":false,"start_time":"2021-01-31T18:10:26.769933","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if len(df_test) == 2 and df_test.loc[0, 'image_id'] == df_test.loc[1, 'image_id']:\n    df_test = pd.read_csv(f'{BASE_DIR}/sample_submission.csv')\nelse:\n    df_test['label'] = df_test['mean']","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:10:27.141773Z","iopub.status.busy":"2021-01-31T18:10:27.141061Z","iopub.status.idle":"2021-01-31T18:10:27.150289Z","shell.execute_reply":"2021-01-31T18:10:27.150732Z"},"papermill":{"duration":0.128529,"end_time":"2021-01-31T18:10:27.150848","exception":false,"start_time":"2021-01-31T18:10:27.022319","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"df_test","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T18:10:27.370491Z","iopub.status.busy":"2021-01-31T18:10:27.369864Z","iopub.status.idle":"2021-01-31T18:10:27.703348Z","shell.execute_reply":"2021-01-31T18:10:27.702412Z"},"papermill":{"duration":0.444395,"end_time":"2021-01-31T18:10:27.703461","exception":false,"start_time":"2021-01-31T18:10:27.259066","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# submission.csv に保存\ndf_test[['image_id', 'label']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}