{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2021-02-06T17:49:21.215239Z","iopub.status.busy":"2021-02-06T17:49:21.214402Z","iopub.status.idle":"2021-02-06T17:49:52.283024Z","shell.execute_reply":"2021-02-06T17:49:52.282058Z"},"papermill":{"duration":31.096601,"end_time":"2021-02-06T17:49:52.283183","exception":false,"start_time":"2021-02-06T17:49:21.186582","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"!pip install \"../input/pytorchlightning110/pytorch_lightning-1.1.0-py3-none-any.whl\"\n\nimport os\nimport sys\nsys.path.append(\"../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master\")\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nsys.path.append(\"../input/microsoftvision\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if os.getcwd() in [\"/kaggle/working\", \"/content\"]:  \n    sys.path.append(\"../input/cassava-code\")\nelse:\n    sys.path.append(\"../kaggle_Cassava/code\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2021-02-06T17:49:52.343302Z","iopub.status.busy":"2021-02-06T17:49:52.342386Z","iopub.status.idle":"2021-02-06T17:49:56.296478Z","shell.execute_reply":"2021-02-06T17:49:56.297177Z"},"papermill":{"duration":3.988233,"end_time":"2021-02-06T17:49:56.29738","exception":false,"start_time":"2021-02-06T17:49:52.309147","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import cv2\nimport glob\nimport json\nimport math\nimport yaml\nimport pickle\nimport random\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import accuracy_score, log_loss\nfrom tqdm.notebook import tqdm\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nimport timm\nimport pytorch_lightning as pl\nfrom efficientnet_pytorch import EfficientNet\nfrom deit_models import deit_base_patch16_224\nimport microsoftvision\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"timm version:\", timm.__version__)\nprint(\"pytorch_lightning version:\", pl.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"device:\", device)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:49:56.438739Z","iopub.status.busy":"2021-02-06T17:49:56.437993Z","iopub.status.idle":"2021-02-06T17:49:56.441251Z","shell.execute_reply":"2021-02-06T17:49:56.441731Z"},"papermill":{"duration":0.037552,"end_time":"2021-02-06T17:49:56.441867","exception":false,"start_time":"2021-02-06T17:49:56.404315","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def set_seed(seed: int=0) -> None:\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    torch.backends.cudnn.benchmark = False\n    torch.backends.cudnn.deterministic = True\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://pypi.org/project/microsoftvision/\nclass MicrosoftVisionResnet50(nn.Module):\n    def __init__(self, pretrained=False, n_classes=5):\n        super().__init__()\n        self.model = microsoftvision.models.resnet50(pretrained=pretrained)\n        self.fc = nn.Linear(2048, n_classes)\n        \n    def forward(self, x):\n        x = self.model(x)\n        x = self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:49:56.520449Z","iopub.status.busy":"2021-02-06T17:49:56.509167Z","iopub.status.idle":"2021-02-06T17:49:56.552564Z","shell.execute_reply":"2021-02-06T17:49:56.552019Z"},"papermill":{"duration":0.084405,"end_time":"2021-02-06T17:49:56.552693","exception":false,"start_time":"2021-02-06T17:49:56.468288","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class CassavaLite(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n        \n        if source == 'efficientnet-pytorch':\n            self.net = EfficientNet.from_name(arch)\n            self.net._fc = nn.Linear(self.net._fc.in_features, n_classes)\n                \n        elif source == 'timm':\n            self.net = timm.create_model(arch, pretrained=None)\n            if \"eff\" in arch:\n                self.net.classifier = nn.Linear(self.net.classifier.in_features, n_classes) \n            elif \"rexnet\" in arch:\n                self.net.head.fc = nn.Linear(self.net.head.fc.in_features, n_classes)\n            elif \"vit\" in arch:\n                self.net.head = nn.Linear(self.net.head.in_features, n_classes)\n            else:\n                self.net.fc = nn.Linear(self.net.fc.in_features, n_classes)\n    \n    def forward(self,x):\n        out = self.net(x)\n        return out\n\nclass CassavaLite2(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n   \n        self.net = timm.create_model(arch, pretrained=None)\n\n        if \"eff\" in arch:\n            self.net.classifier = nn.Linear(self.net.classifier.in_features, n_classes)\n            self.feat_net = nn.Sequential(*list(self.net.children())[:-2])  \n        elif \"rexnet\" in arch:\n            self.net.head.fc = nn.Linear(self.net.head.fc.in_features, n_classes)\n            self.feat_net = nn.Sequential(*list(self.net.children())[:-1])\n        elif \"vit\" in arch:\n            self.net.head = nn.Linear(self.net.head.in_features, n_classes)\n            self.feat_net = nn.Sequential(*list(self.net.children())[:-1])\n        else:\n            self.net.fc = nn.Linear(self.net.fc.in_features, n_classes)\n            self.feat_net = nn.Sequential(*list(self.net.children())[:-2])  # global_poolとfc層除く\n\n    def forward(self, x):\n        out = self.net(x)\n        return out\n    \nclass CassavaLite3(pl.LightningModule):\n    def __init__(self, CFG):\n        super().__init__()\n\n        self.net = timm.create_model(CFG.arch, pretrained=None)\n\n        if \"eff\" in CFG.arch:\n            self.net.classifier = nn.Linear(\n                self.net.classifier.in_features, CFG.n_classes\n            )\n            if CFG.gem_p > 0.0:\n                self.net = GeMNet(\n                    list(self.net.children())[:-2],\n                    GeM(p=CFG.gem_p),\n                    self.net.classifier.in_features,\n                    CFG.n_classes,\n                )\n            self.feat_net = nn.Sequential(*list(self.net.children())[:-2])\n            \n        elif \"rexnet\" in CFG.arch:\n            self.net.head.fc = nn.Linear(self.net.head.fc.in_features, CFG.n_classes)\n            if CFG.gem_p > 0.0:\n                self.net = GeMNet(\n                    list(self.net.children())[:-1],\n                    GeM(p=CFG.gem_p),\n                    self.net.head.fc.in_features,\n                    CFG.n_classes,\n                )\n            self.feat_net = nn.Sequential(*list(self.net.children())[:-1])\n\n        elif \"vit\" in CFG.arch:\n            self.net.head = nn.Linear(self.net.head.in_features, CFG.n_classes)\n            self.feat_net = nn.Sequential(*list(self.net.children())[:-1])\n\n        else:\n            self.net.fc = nn.Linear(self.net.fc.in_features, CFG.n_classes)\n            if CFG.gem_p > 0.0:\n                self.net = GeMNet(\n                    list(self.net.children())[:-2],\n                    GeM(p=CFG.gem_p),\n                    self.net.fc.in_features,\n                    CFG.n_classes,\n                )\n            self.feat_net = nn.Sequential(\n                *list(self.net.children())[:-2]\n            )  # global_poolとfc層除く\n\n    def forward(self, x):\n        out = self.net(x)\n        return out\n    \nclass CassavaLite4(pl.LightningModule):\n    def __init__(self, CFG):\n        super().__init__()\n\n        if \"microsoft\" in CFG.arch:\n            self.net = MicrosoftVisionResnet50(pretrained=False, n_classes=CFG.n_classes)\n            self.feat_net = nn.Sequential(*list(self.net.children())[:-1])\n            \n        else:\n            self.net = timm.create_model(CFG.arch, pretrained=None)\n\n            if \"eff\" in CFG.arch:\n                self.net.classifier = nn.Linear(\n                    self.net.classifier.in_features, CFG.n_classes\n                )\n                self.feat_net = nn.Sequential(*list(self.net.children())[:-2])\n\n            elif \"rexnet\" in CFG.arch:\n                self.net.head.fc = nn.Linear(self.net.head.fc.in_features, CFG.n_classes)\n                self.feat_net = nn.Sequential(*list(self.net.children())[:-1])\n\n            elif \"vit\" in CFG.arch:\n                self.net.head = nn.Linear(self.net.head.in_features, CFG.n_classes)\n                self.feat_net = nn.Sequential(*list(self.net.children())[:-1])\n\n            elif \"deit\" in CFG.arch:\n                self.net = deit_base_patch16_224(pretrained=False)  # pretrained=True にすると失敗する。。。\n                self.net.head = nn.Linear(self.net.head.in_features, CFG.n_classes)\n                self.feat_net = nn.Sequential(*list(self.net.children())[:-1])\n\n            else:\n                self.net.fc = nn.Linear(self.net.fc.in_features, CFG.n_classes)\n                self.feat_net = nn.Sequential(\n                    *list(self.net.children())[:-2]\n                )  # global_poolとfc層除く\n\n    def forward(self, x):\n        out = self.net(x)\n        return out","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:49:56.619283Z","iopub.status.busy":"2021-02-06T17:49:56.617459Z","iopub.status.idle":"2021-02-06T17:49:56.620048Z","shell.execute_reply":"2021-02-06T17:49:56.620555Z"},"papermill":{"duration":0.040613,"end_time":"2021-02-06T17:49:56.6207","exception":false,"start_time":"2021-02-06T17:49:56.580087","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, df:pd.DataFrame, imfolder:str, transformgroups):\n        self.df = df\n        self.imfolder = imfolder\n        self.transformgroups = transformgroups\n        \n    def __getitem__(self,index):\n        im_path = os.path.join(self.imfolder,self.df.iloc[index]['image_id'])\n        x = cv2.imread(im_path,cv2.IMREAD_COLOR)\n        x = cv2.cvtColor(x, cv2.COLOR_BGR2RGB)\n        ret = { }\n        for i, g in enumerate(self.transformgroups):\n            v = None\n            for t in g:\n                r = t(image=x)['image'].unsqueeze(0)\n                v = r if v is None else torch.cat([v, r], axis=0)\n            ret[f'x{i}'] = v.float()\n        return ret\n        \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:49:56.709753Z","iopub.status.busy":"2021-02-06T17:49:56.696504Z","iopub.status.idle":"2021-02-06T17:49:56.71257Z","shell.execute_reply":"2021-02-06T17:49:56.712053Z"},"papermill":{"duration":0.066659,"end_time":"2021-02-06T17:49:56.712705","exception":false,"start_time":"2021-02-06T17:49:56.646046","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def get_models(model_paths, lite_type, CFG=None):\n    models = []\n    for model_path in model_paths:\n        if lite_type == \"CassavaLite\":\n            model = CassavaLite().load_from_checkpoint(model_path).to(device)\n        elif lite_type == \"CassavaLite2\":\n            model = CassavaLite2().load_from_checkpoint(model_path).to(device)\n        elif lite_type == \"CassavaLite3\":\n            model = CassavaLite3(CFG).load_from_checkpoint(model_path, CFG=CFG).to(device)\n        elif lite_type == \"CassavaLite4\":\n            model = CassavaLite4(CFG).load_from_checkpoint(model_path, CFG=CFG).to(device)\n        model.eval()\n        models.append(model)\n        print(f'{model_path} is loaded')\n    return models\n\ndef get_dataloader(img_size_desc):\n    transforms = [ ]\n    for item in img_size_desc:\n        # model = item[0]\n        tta_nums = item[1]\n        img_size = item[2]\n        if img_size < 500:\n            t = A.Compose([\n                A.RandomResizedCrop(img_size, img_size, \n                #    p=0.5\n                ),\n                A.Normalize(\n                    mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0,\n                ),\n                ToTensorV2(p=1.0),\n            ],\n            p=1.0)\n        else:\n            t = A.Compose([\n                A.CenterCrop(img_size, img_size, \n                #    p=0.5\n                ),\n                A.Normalize(\n                    mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0,\n                ),\n                ToTensorV2(p=1.0),\n            ], p=1.0)\n        transforms.append([ t for _ in range(tta_nums) ])\n    test_dataset = CassavaDataset(test_df, img_dir, transforms)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False, drop_last=False)\n    return test_loader","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.025913,"end_time":"2021-02-06T17:49:56.76423","exception":false,"start_time":"2021-02-06T17:49:56.738317","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# main"},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:49:56.826098Z","iopub.status.busy":"2021-02-06T17:49:56.825415Z","iopub.status.idle":"2021-02-06T17:49:56.83894Z","shell.execute_reply":"2021-02-06T17:49:56.838074Z"},"papermill":{"duration":0.049213,"end_time":"2021-02-06T17:49:56.83905","exception":false,"start_time":"2021-02-06T17:49:56.789837","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"n_classes = 5\nnum_workers = 2\n\npath = \"../input/cassava-leaf-disease-classification/\"\n\nif DEBUG:\n    test_df = pd.read_csv(f\"{path}/train.csv\")\n    test_df = test_df.iloc[:100, :]\n    img_dir = f'{path}train_images/'\n    COMMIT  = False\nelse:\n    test_df = pd.read_csv(f\"{path}/sample_submission.csv\")\n    img_dir = f'{path}test_images/'\n    COMMIT  = len(glob.glob(f'{path}test_images/*.jpg')) == 1\n\nwith open(f'{path}/label_num_to_disease_map.json', 'r') as f:\n    name_mapping = json.load(f)\nname_mapping = {int(k): v for k, v in name_mapping.items()}","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:49:56.900649Z","iopub.status.busy":"2021-02-06T17:49:56.899635Z","iopub.status.idle":"2021-02-06T17:49:56.902672Z","shell.execute_reply":"2021-02-06T17:49:56.902124Z"},"papermill":{"duration":0.036863,"end_time":"2021-02-06T17:49:56.902777","exception":false,"start_time":"2021-02-06T17:49:56.865914","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"n_tta = 5\nn_tta_many = 8\n#batch_size = 16\nbatch_size = 1\ntest_preds_list = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def randomTTA(x):\n    if np.random.rand() > 0.5:\n        x = torch.flip(x, [-1])\n    if np.random.rand() > 0.5:\n        x = torch.flip(x, [-2])\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    def __init__(self, **kwargs):\n        self.seeds = [42]\n        self.n_classes = n_classes\n        self.arch = kwargs[\"arch\"]\n        self.gem_p = kwargs[\"gem_p\"]\n        self.height = kwargs[\"height\"]\n\ndef GetYaml(yamlpath):\n    with open(yamlpath, 'r') as f:\n        cdict = yaml.load(f)\n    cfg = Config(**cdict)\n    return cfg","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:49:57.027353Z","iopub.status.busy":"2021-02-06T17:49:57.026557Z","iopub.status.idle":"2021-02-06T17:50:25.753085Z","shell.execute_reply":"2021-02-06T17:50:25.752468Z"},"papermill":{"duration":28.763919,"end_time":"2021-02-06T17:50:25.753209","exception":false,"start_time":"2021-02-06T17:49:56.98929","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"%%time\n\n# tf_efficientnet_b4_ns\n# oof 0.894\ncfg1 = GetYaml(\"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_tf_efficientnet_b4_ns_BiTemperedLoss/hparams.yaml\")\nmdl1 = sorted(\n    glob.glob(\n        \"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_tf_efficientnet_b4_ns_BiTemperedLoss/model_seed_*.ckpt\"\n    )\n)\nmodels1 = get_models(mdl1, \"CassavaLite3\", CFG=cfg1)\n\n\n# byol + seresnext50_32x4d\n# oof -.---\ncfg2 = GetYaml(\"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_byol_seresnext50_32x4d_cutmix_labelsmooth_half/cfg.yaml\")\nmdl2 = sorted(\n    glob.glob(\n        \"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_byol_seresnext50_32x4d_cutmix_labelsmooth_half/model_seed_*.ckpt\"\n    )\n)\nmodels2 = get_models(mdl2, \"CassavaLite4\", CFG=cfg2)\n\n\n# vit_b16_224_fold10\n# oof -.---\ncfg3 = GetYaml(\"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_vit_b16_224_fold10/kaggle_upload_vit_b16_224_fold10/cfg.yaml\")\nmdl3 = sorted(\n    glob.glob(\n        \"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_vit_b16_224_fold10/kaggle_upload_vit_b16_224_fold10/model_seed_*.ckpt\"\n    )\n)\nmodels3 = get_models(mdl3, \"CassavaLite4\", CFG=cfg3)\n\n\n## seresnext50_32x4d_fmix\n## oof -.---\n#cfg_se50 = Config(**{\"arch\":\"seresnext50_32x4d\", \"gem_p\":0, \"height\":512})\n#mdl_se50 = sorted(\n#    glob.glob(\n#        \"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_seresnext50_32x4d_fmix/model_seed_*.ckpt\"\n#    )\n#)\n#models_se50 = get_models(mdl_se50, \"CassavaLite4\", CFG=cfg_se50)\n\n\n## resnest101e\n## oof \n#cfg_r101e = Config(**{\"arch\":\"resnest101e\", \"gem_p\":0, \"height\":512})\n#mdl_r101e = sorted(\n#    glob.glob(\n#        \"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_resnest101e_bi_tempered_loss/model_seed_*.ckpt\"\n#    )\n#)\n#models_r101e = get_models(mdl_r101e, \"CassavaLite3\", CFG=cfg_r101e)\n\n\n## resnest101e_cleanlab\n## oof \n#cfg_r101e_c = GetYaml(\"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_resnest101e_cleanlab/cfg.yaml\")\n#mdl_r101e_c = sorted(\n#    glob.glob(\n#        \"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_resnest101e_cleanlab/model_seed_*.ckpt\"\n#    )\n#)\n#models_r101e_c = get_models(mdl_r101e_c, \"CassavaLite4\", CFG=cfg_r101e_c)\n\n\n## resnest101e_cleanlab_noise_cutmix\n## oof -.---\n#cfg_r101e_cnc = GetYaml(\"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_resnest101e_cleanlab_noise_cutmix/cfg.yaml\")\n#mdl_r101e_cnc = sorted(\n#    glob.glob(\n#        \"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_resnest101e_cleanlab_noise_cutmix/model_seed_*.ckpt\"\n#    )\n#)\n#models_r101e_cnc = get_models(mdl_r101e_cnc, \"CassavaLite4\", CFG=cfg_r101e_cnc)\n\n\n## tf_efficientnet_b4_ns_fold3\n## oof -.---\n#cfg_b4_f3 = GetYaml(\"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_tf_efficientnet_b4_ns_fold3/cfg.yaml\")\n#mdl_b4_f3 = sorted(\n#    glob.glob(\n#        \"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_tf_efficientnet_b4_ns_fold3/model_seed_*.ckpt\"\n#    )\n#)\n#models_b4_f3 = get_models(mdl_b4_f3, \"CassavaLite4\", CFG=cfg_b4_f3)\n\n\n## tf_efficientnet_b4_ns_fold10\n## oof -.---\n#cfg_b4_f10 = GetYaml(\"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_tf_efficientnet_b4_ns_fold10/cfg.yaml\")\n#mdl_b4_f10 = sorted(\n#    glob.glob(\n#        \"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_tf_efficientnet_b4_ns_fold10/model_seed_*.ckpt\"\n#    )\n#)\n#models_b4_f10 = get_models(mdl_b4_f10, \"CassavaLite4\", CFG=cfg_b4_f10)\n\n\n## tf_efficientnet_b4_ns_cleanlab_noise_cutmix_fmix_n_over\n## oof -.---\n#cfg_b4_cncfn = GetYaml(\"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_tf_efficientnet_b4_ns_cleanlab_noise_cutmix_fmix_n_over/cfg.yaml\")\n#mdl_b4_cncfn = sorted(\n#    glob.glob(\n#        \"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_tf_efficientnet_b4_ns_cleanlab_noise_cutmix_fmix_n_over/model_seed_*.ckpt\"\n#    )\n#)\n#models_b4_cncfn = get_models(mdl_b4_cncfn, \"CassavaLite4\", CFG=cfg_b4_cncfn)\n\n\n## vit-base-patch16-224\n## oof -.---\n#cfg_vp16 = Config(**{\"arch\":\"vit_base_patch16_224\", \"gem_p\":0, \"height\":224})\n#mdl_vp16 = sorted(\n#    glob.glob(\n#        \"../input/cassava-vit-base-patch16-224-fit/model_seed_*.ckpt\"\n#    )\n#)\n#models_vp16 = get_models(mdl_vp16, \"CassavaLite4\", CFG=cfg_vp16)\n\n\n## vit-base-patch32-384\n## oof -.---\n#cfg_vp32 = GetYaml(\"../input/cassava-vit-base-patch32-384-fit/cfg.yaml\")\n#mdl_vp32 = sorted(\n#    glob.glob(\n#        \"../input/cassava-vit-base-patch32-384-fit/model_seed_*.ckpt\"\n#    )\n#)\n#models_vp32 = get_models(mdl_vp32, \"CassavaLite4\", CFG=cfg_vp32)\n\n\n## deit-base-patch16-224\n## oof -.---\n#cfg_deit_224 = GetYaml(\"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_deit-base-patch16-224/cfg.yaml\")\n#mdl_deit_224 = sorted(\n#    glob.glob(\n#        \"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_deit-base-patch16-224/model_seed_*.ckpt\"\n#    )\n#)\n#models_deit_224 = get_models(mdl_deit_224, \"CassavaLite4\", CFG=cfg_deit_224)\n\n\n## microsoftvisionresnet50\n## oof -.---\n#cfg_mic = GetYaml(\"../input/cassava-microsoftvisionresnet50-ipynb/cfg.yaml\")\n#mdl_mic = sorted(\n#    glob.glob(\n#        \"../input/cassava-microsoftvisionresnet50-ipynb/model_seed_*.ckpt\"\n#    )\n#)\n#models_mic = get_models(mdl_mic, \"CassavaLite4\", CFG=cfg_mic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = [\n    (models1, n_tta, 512),\n    (models2, n_tta, cfg2.height),\n    (models3, n_tta_many, cfg3.height),\n]\n    #(models1, n_tta, 512),\n    #(models2, n_tta, cfg2.height),\n    #(models3, n_tta_many, cfg3.height),\n    #(models_se50, n_tta, cfg_se50.height)\n    #(models_r101e, n_tta, cfg_r101e.height),\n    #(models_r101e_c, n_tta, cfg_r101e_c.height),\n    #(models_r101e_cnc, n_tta, cfg_r101e_cnc.height),\n    #(models_b4_f3, n_tta, cfg_b4_f3.height),\n    #(models_b4_f10, n_tta, cfg_b4_f10.height),\n    #(models_b4_cncfn, n_tta, cfg_b4_cncfn.height),\n    #(models_vp16, n_tta_many, cfg_vp16.height),\n    #(models_vp32, n_tta_many, cfg_vp32.height),\n    #(models_deit_224, n_tta_many, cfg_deit_224.height),\n    #(models_mic, n_tta, cfg_mic.height),\n    \nmodels_num = len(models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loader = get_dataloader(models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds_list = []\nset_seed(seed=42)\nwith torch.no_grad():\n    preds = [ [ ] for _ in range(len(models)) ]\n    actfn = nn.Softmax(dim=1)\n    for batch in loader:\n        for i, mdesc in enumerate(models):\n            mdlgrp = mdesc[0]\n            numtta = mdesc[1]\n            img = batch['x{}'.format(i)][0].to(device)\n            b = img.shape[0]\n            x = img\n            y = torch.zeros([b, 5], device=device)\n            for j in range(b):\n                x[j] = randomTTA(img[j])\n            nummdl = len(mdlgrp)\n            for mdl in mdlgrp:\n                z = mdl(x)\n                y = y + (actfn(z) / nummdl)\n            y = y.detach().cpu().numpy()\n            y = np.mean(y, axis=0)\n            preds[i] += [ y ]\n    for i in range(models_num):\n        preds[i] = np.concatenate(preds[i], axis=0).reshape(-1, 5)\n        test_preds_list.append(preds[i])","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.150074,"end_time":"2021-02-06T17:53:11.773401","exception":false,"start_time":"2021-02-06T17:53:11.623327","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### --- SiNpcw Module ---"},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:53:12.087287Z","iopub.status.busy":"2021-02-06T17:53:12.086285Z","iopub.status.idle":"2021-02-06T17:53:12.088709Z","shell.execute_reply":"2021-02-06T17:53:12.089306Z"},"papermill":{"duration":0.163616,"end_time":"2021-02-06T17:53:12.089453","exception":false,"start_time":"2021-02-06T17:53:11.925837","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# image size\nSIZE = 512\n\n# class\nNUM_CLASSES = 5","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:53:12.405515Z","iopub.status.busy":"2021-02-06T17:53:12.404824Z","iopub.status.idle":"2021-02-06T17:53:12.409349Z","shell.execute_reply":"2021-02-06T17:53:12.408844Z"},"papermill":{"duration":0.170246,"end_time":"2021-02-06T17:53:12.409457","exception":false,"start_time":"2021-02-06T17:53:12.239211","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def GetPath(pth):\n    return os.path.join('../input/cassavapth/', pth)\n\nmodeldefs = [\n    # Gr1.\n    [\n        # B4+FINE+UDA+TTA3 (LB900), CV: k0=8981, k1ep13=9030, k2ep13=8927, k3ep12=8983, k4ep14=8904\n        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019613/22019613k0.pth') },\n        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019613/22019613k1.pth') },\n        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019613/22019613k2.pth') },\n        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019613/22019613k3.pth') },\n        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019613/22019613k4.pth') },\n    ],\n    # Gr2.\n    [\n        # resnest101e (LB896), CV: k0=9005, k1=9012, k2=8960, k3=8967, k4=8920\n        { 'name' : 'timm-resnest101e', 'pth' : GetPath('22019632/22019632k0.pth') },\n        { 'name' : 'timm-resnest101e', 'pth' : GetPath('22019632/22019632k1.pth') },\n        { 'name' : 'timm-resnest101e', 'pth' : GetPath('22019632/22019632k2.pth') },\n        { 'name' : 'timm-resnest101e', 'pth' : GetPath('22019632/22019632k3.pth') },\n        { 'name' : 'timm-resnest101e', 'pth' : GetPath('22019632/22019632k4.pth') },\n    ],\n#    [\n#        # B4 (LB---), CV: k0=8951, k1=8963, k2=8923, k3=9002, k4=8897\n#        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019629/22019629k0.pth') },\n#        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019629/22019629k1.pth') },\n#        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019629/22019629k2.pth') },\n#        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019629/22019629k3.pth') },\n#        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019629/22019629k4.pth') },\n#    ],\n#    [\n#        # se_resnext101 (LB896), CV: k0e=8965, k1=8986, k2=8913, k3=9007, k4=8876\n#        { 'name' : 'se_resnext101', 'pth' : GetPath('22019631/22019631k0.pth') },\n#        { 'name' : 'se_resnext101', 'pth' : GetPath('22019631/22019631k1.pth') },\n#        { 'name' : 'se_resnext101', 'pth' : GetPath('22019631/22019631k2.pth') },\n#        { 'name' : 'se_resnext101', 'pth' : GetPath('22019631/22019631k3.pth') },\n#        { 'name' : 'se_resnext101', 'pth' : GetPath('22019631/22019631k4.pth') },\n#    ],\n#    [\n#        # se_resnet101 (LB894), CV: k0=8958, k1e=8953, k2=8869, k3=8997, k4=8885\n#        { 'name' : 'se_resnet101', 'pth' : GetPath('22019630/22019630k0.pth') },\n#        { 'name' : 'se_resnet101', 'pth' : GetPath('22019630/22019630k1.pth') },\n#        { 'name' : 'se_resnet101', 'pth' : GetPath('22019630/22019630k2.pth') },\n#        { 'name' : 'se_resnet101', 'pth' : GetPath('22019630/22019630k3.pth') },\n#        { 'name' : 'se_resnet101', 'pth' : GetPath('22019630/22019630k4.pth') },\n#    ],\n#    [\n#        # regnety_032 (LB896), CV: k0=8974, k1=8984, k2=8906, k3=9009, k4=8918\n#        { 'name' : 'timm-regnety_032', 'pth' : GetPath('22019638/22019638k0.pth') },\n#        { 'name' : 'timm-regnety_032', 'pth' : GetPath('22019638/22019638k1.pth') },\n#        { 'name' : 'timm-regnety_032', 'pth' : GetPath('22019638/22019638k2.pth') },\n#        { 'name' : 'timm-regnety_032', 'pth' : GetPath('22019638/22019638k3.pth') },\n#        { 'name' : 'timm-regnety_032', 'pth' : GetPath('22019638/22019638k4.pth') },\n#    ],\n#    [\n#        # B5 (LB892), CV: k0=8993, k1=8965, k2=8962, k3=8986, k4=8927\n#        { 'name' : 'efficientnet-b5', 'pth' : GetPath('22019639/22019639k0.pth') },\n#        { 'name' : 'efficientnet-b5', 'pth' : GetPath('22019639/22019639k1.pth') },\n#        { 'name' : 'efficientnet-b5', 'pth' : GetPath('22019639/22019639k2.pth') },\n#        { 'name' : 'efficientnet-b5', 'pth' : GetPath('22019639/22019639k3.pth') },\n#        { 'name' : 'efficientnet-b5', 'pth' : GetPath('22019639/22019639k4.pth') },\n#    ],\n#    [\n#        # resnest200e (LB---) k0=9009, k1=9002, k2=8937, k3=9021, k4=8904\n#        { 'name' : 'timm-resnest200e', 'pth' : GetPath('22019640/22019640k0.pth') },\n#        { 'name' : 'timm-resnest200e', 'pth' : GetPath('22019640/22019640k1.pth') },\n#        { 'name' : 'timm-resnest200e', 'pth' : GetPath('22019640/22019640k2.pth') },\n#        { 'name' : 'timm-resnest200e', 'pth' : GetPath('22019640/22019640k3.pth') },\n#        { 'name' : 'timm-resnest200e', 'pth' : GetPath('22019640/22019640k4.pth') },\n#    ],\n#    [\n#        # resnest101e (LB---), CV: k0=8991, k1=9000, k2=8925, k3=8965, k4=8862 (oof=0.89480)\n#        { 'name' : 'timm-resnest101e', 'pth' : GetPath('22019717/22019717k0.pth') },\n#        { 'name' : 'timm-resnest101e', 'pth' : GetPath('22019717/22019717k1.pth') },\n#        { 'name' : 'timm-resnest101e', 'pth' : GetPath('22019717/22019717k2.pth') },\n#        { 'name' : 'timm-resnest101e', 'pth' : GetPath('22019717/22019717k3.pth') },\n#        { 'name' : 'timm-resnest101e', 'pth' : GetPath('22019717/22019717k4.pth') },\n#    ],\n#    [\n#        # efficientnet-b4 (LB---), CV: k0=8993, k1=8972, k2=8939, k=3=9021, k4=8941 (oof=0.89728)\n#        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019720/22019720k0.pth') },\n#        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019720/22019720k1.pth') },\n#        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019720/22019720k2.pth') },\n#        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019720/22019720k3.pth') },\n#        { 'name' : 'efficientnet-b4', 'pth' : GetPath('22019720/22019720k4.pth') },\n#    ],\n#    [\n#        # efficientnet-b4 (LB---), CV: k0=8993, k1=8972, k2=8939, k=3=9021, k4=8941 (oof=0.89629)\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"22019725/22019725k0.pth\")},\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"22019725/22019725k1.pth\")},\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"22019725/22019725k2.pth\")},\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"22019725/22019725k3.pth\")},\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"22019725/22019725k4.pth\")},\n#    ],\n#    [\n#        # combine_set1 (oof=0.89886)\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"22019725/22019725k0.pth\")},\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"22019613/22019613k1.pth\")},\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"22019725/22019725k2.pth\")},\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"22019720/22019720k3.pth\")},\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"22019720/22019720k4.pth\")},\n#    ],\n#    [\n#        # efficientnet-b4 (oof=0.89961)\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"ex02C/ex02Ck0.pth\")},\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"ex02C/ex02Ck1.pth\")},\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"ex02C/ex02Ck2.pth\")},\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"ex02C/ex02Ck3.pth\")},\n#        {\"name\": \"efficientnet-b4\", \"pth\": GetPath(\"ex02C/ex02Ck4.pth\")},\n#    ],\n]\n\nmodelttas = [\n    3,\n    3,\n]\n\nTTA_ROUND = np.max(modelttas)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:53:12.735483Z","iopub.status.busy":"2021-02-06T17:53:12.733613Z","iopub.status.idle":"2021-02-06T17:53:12.736244Z","shell.execute_reply":"2021-02-06T17:53:12.736769Z"},"papermill":{"duration":0.171441,"end_time":"2021-02-06T17:53:12.736895","exception":false,"start_time":"2021-02-06T17:53:12.565454","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def GetModel(name, param):\n    num_classes = NUM_CLASSES\n    if name in [ 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152', 'resnext50', 'resnext101', 'wide_resnet50', 'wide_resnet101' ]:\n        if name == 'resnext50' or name == 'resnext101':\n            name = name + '_32x4d'\n        elif name == 'wide_resnet50' or name == 'wide_resnet101':\n            name = name + '_2'\n        model = getattr(torchvision.models, name)(pretrained=None)\n        model.fc = nn.Linear(model.fc.in_features, num_classes)\n    elif name in [ 'senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152', 'se_resnext50', 'se_resnext101', 'se_resnext50_32x4d', 'se_resnext101_32x4d' ]:\n        if name == 'se_resnext50' or name == 'se_resnext101':\n            name = name + '_32x4d'\n        model = getattr(pretrainedmodels, name)(pretrained=None)\n        model.avg_pool = nn.AdaptiveAvgPool2d((1, 1))\n        model.last_linear = nn.Linear(model.last_linear.in_features, num_classes)\n    elif name.startswith('efficientnet-b'):\n        model = EfficientNet.from_name(name)\n        model._fc = nn.Linear(model._fc.in_features, num_classes)\n    elif name.startswith('timm-'):\n        model = timm.create_model(model_name=name[len('timm-'):], num_classes=num_classes, in_chans=3, pretrained=False)\n    else:\n        raise NameError()\n    state = torch.load(param, map_location=device)\n    model.load_state_dict(state, strict=True)\n    model.eval()\n    print('model ({}) is loaded'.format(name))\n    return model","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:53:13.055192Z","iopub.status.busy":"2021-02-06T17:53:13.054453Z","iopub.status.idle":"2021-02-06T17:53:13.058468Z","shell.execute_reply":"2021-02-06T17:53:13.057957Z"},"papermill":{"duration":0.1685,"end_time":"2021-02-06T17:53:13.058576","exception":false,"start_time":"2021-02-06T17:53:12.890076","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def GetAugment(size):\n    return A.Compose([\n        A.Resize(size, size),\n        A.Normalize()\n    ], p=1.0)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:53:13.529148Z","iopub.status.busy":"2021-02-06T17:53:13.528127Z","iopub.status.idle":"2021-02-06T17:53:13.533343Z","shell.execute_reply":"2021-02-06T17:53:13.53443Z"},"papermill":{"duration":0.274308,"end_time":"2021-02-06T17:53:13.534621","exception":false,"start_time":"2021-02-06T17:53:13.260313","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def TTA(img, ops):\n    # input: NxCxHxW\n    if ops == 0:\n        pass\n    elif ops == 1:\n        img = torch.flip(img, [-1])\n    elif ops == 2:\n        img = torch.flip(img, [-2])\n    elif ops == 3:\n        img = torch.flip(img, [-1, -2])\n    elif ops == 4:\n        img = torch.rot90(img, 1, [2, 3])\n    elif ops == 5:\n        img = torch.rot90(img, 3, [2, 3])\n    else:\n        pass\n    return img","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:53:13.972171Z","iopub.status.busy":"2021-02-06T17:53:13.970005Z","iopub.status.idle":"2021-02-06T17:53:13.972986Z","shell.execute_reply":"2021-02-06T17:53:13.973622Z"},"papermill":{"duration":0.209191,"end_time":"2021-02-06T17:53:13.973823","exception":false,"start_time":"2021-02-06T17:53:13.764632","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def GetDataLoader(batch=8, num_workers=2):\n    files = test_df.iloc[:, 0].values\n    dataset = InferDataset(files, augops=GetAugment(SIZE))\n    return torch.utils.data.DataLoader(\n        dataset,\n        batch_size=batch,\n        shuffle=False,\n        drop_last=False,\n        num_workers=num_workers\n    )\n\nclass InferDataset(Dataset):\n    def __init__(self, files, augops):\n        self.files = files\n        self.augops = augops\n\n    def __len__(self):\n        return len(self.files)\n\n    def __getitem__(self, idx):\n        img = cv2.imread(os.path.join(img_dir, self.files[idx]))\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        out = self.augops(force_apply=False, image=img)['image']\n        out = out.transpose(2, 0, 1)\n        return torch.from_numpy(out)\n\nloader = GetDataLoader(batch=8)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:53:14.309368Z","iopub.status.busy":"2021-02-06T17:53:14.308194Z","iopub.status.idle":"2021-02-06T17:53:34.644556Z","shell.execute_reply":"2021-02-06T17:53:34.644056Z"},"papermill":{"duration":20.504744,"end_time":"2021-02-06T17:53:34.644705","exception":false,"start_time":"2021-02-06T17:53:14.139961","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"modelgroups = [ ]\nfor i, groupdef in enumerate(modeldefs):\n    print(f'--- Group{i} ---')\n    models = [ ]\n    for mdef in groupdef:\n        mdl = GetModel(mdef['name'], mdef['pth']).to(device)\n        models.append(mdl)\n    modelgroups.append(models)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:53:34.972401Z","iopub.status.busy":"2021-02-06T17:53:34.967499Z","iopub.status.idle":"2021-02-06T17:53:36.298379Z","shell.execute_reply":"2021-02-06T17:53:36.297251Z"},"papermill":{"duration":1.495351,"end_time":"2021-02-06T17:53:36.298518","exception":false,"start_time":"2021-02-06T17:53:34.803167","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"with torch.no_grad():\n    gnums = len(modelgroups)\n    preds = [ [] for _ in range(gnums) ] * gnums\n    actfn = nn.Softmax(dim=1)\n    for _, itr in enumerate(loader):\n        x = itr\n        b = x.shape[0]\n        x = x.to(device)\n        y = torch.zeros([gnums, b, NUM_CLASSES], device=device)\n        for tta in range(TTA_ROUND):\n            xi = TTA(x, tta)  # tta画像\n            #print(tta, xi.shape)\n            for g, mgi in enumerate(zip(modelgroups, modelttas)):  # モデルごとに処理するループ\n                mgrp, mtta = mgi\n                #print(\"mtta:\", mtta)  # そのモデルのtta回数\n                if tta < mtta:\n                    ratio = 1.0 / (len(mgrp) * mtta)\n                    for model in mgrp:\n                        y[g, :, :] = y[g, :, :] + actfn(model(xi)) * ratio\n                        #if tta == 0:\n                        #    print(\"tta == 0\", actfn(model(xi)))  # ttaなしでのy\n                        #if tta == 1:\n                        #    print(\"tta == 1\", actfn(model(xi)))  # hflipでのy\n                        #if tta == 2:\n                        #    print(\"tta == 2\", actfn(model(xi)))  # vflipでのy\n        y = y.detach().cpu().numpy()\n        for g in range(gnums):\n            preds[g] += [ y[g, :, :] ]\nfor g in range(gnums):\n    preds[g] = np.concatenate(preds[g], axis=0).reshape(-1, 5)\n    test_preds_list.append(preds[g])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:53:36.635415Z","iopub.status.busy":"2021-02-06T17:53:36.634596Z","iopub.status.idle":"2021-02-06T17:53:36.638724Z","shell.execute_reply":"2021-02-06T17:53:36.638191Z"},"papermill":{"duration":0.174968,"end_time":"2021-02-06T17:53:36.638836","exception":false,"start_time":"2021-02-06T17:53:36.463868","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if COMMIT:\n    for tp in test_preds_list:\n        display(pd.DataFrame(tp, columns=name_mapping.values()))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.16071,"end_time":"2021-02-06T17:53:36.960214","exception":false,"start_time":"2021-02-06T17:53:36.799504","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## ensemble"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"### LBの値をアンサンブル重みにする場合\n##\n### oof=0.9050801514230967\n### LB=0.905\n###ens_weights = [0.8904051969902322, 0.8941440388839557, 0.8943309809786418, 0.8874608589989251, 0.8963873440201897]\n##\n### oof=0.9063\n### LB=タイムアウト\n###ens_weights = [0.8904051969902322, 0.893302799457868, 0.8941440388839557, 0.8943309809786418, 0.8874608589989251, 0.8847501986259756]\n#\n## oof=0.9057811842781698\n## LB=\n#ens_weights = [0.8904051969902322, 0.893302799457868, 0.8947983362153573, 0.897275318969949, 0.8997990372482124]\n#\n#\n#test_preds_ensemble = np.zeros((test_df.shape[0], n_classes))\n#for i, pre in enumerate(test_preds_list):\n#    test_preds_ensemble += (ens_weights[i] / sum(ens_weights)) * pre\n#\n#display(pd.DataFrame(test_preds_ensemble, columns=name_mapping.values()))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:53:37.303405Z","iopub.status.busy":"2021-02-06T17:53:37.302325Z","iopub.status.idle":"2021-02-06T17:53:37.306088Z","shell.execute_reply":"2021-02-06T17:53:37.306679Z"},"papermill":{"duration":0.183922,"end_time":"2021-02-06T17:53:37.306814","exception":false,"start_time":"2021-02-06T17:53:37.122892","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# oofのcfmにもとづいた重み\n# oof=0.9053605645651259\n# LB=0.905\nens_weights = np.array(\n    [\n        [0.21396141, 0.20027513, 0.20551536, 0.19993096, 0.19595167],  # tf_efficientnet_b4_ns\n        [0.20459705, 0.19993122, 0.20834644, 0.20013494, 0.20400593],  # byol_seresnext50_32x4d\n        [0.1753689 , 0.1870916 , 0.18139876, 0.1981736 , 0.18853328],  # vit_b16_224_fold10\n        [0.2122588 , 0.20382896, 0.20069204, 0.20110776, 0.20379398],  # 22019613_efficientnet-b4.pkl\n        [0.19381385, 0.20887309, 0.20404739, 0.20065273, 0.20771513],  # 22019632_timm-resnest101e.pkl\n    ]\n)\ntest_preds_ensemble = np.zeros((test_df.shape[0], n_classes))\nfor i,pre in enumerate(test_preds_list):\n    test_preds_ensemble += ens_weights[i] * pre\ndisplay(pd.DataFrame(test_preds_ensemble, columns=name_mapping.values()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"## oofのcfmにもとづいた重み  20210212_objective2\n## oof=0.9055\n## LB=\n#ens_weights = np.array(\n#    [\n#        [0.21269394, 0.20043598, 0.20435825, 0.2000628, 0.19597244],   # sub_tf_efficientnet_b4_ns\n#        [0.20338505, 0.20009179, 0.20717339, 0.20026692, 0.20402756],  # sub_byol_seresnext50_32x4d\n#        [0.17433004, 0.18724185, 0.18037744, 0.19830429, 0.18855326],  # sub_vit_b16_224_fold10\n#        [0.20056417, 0.2077788, 0.20300282, 0.20056524, 0.20222576],   # 22019631_se_resnext101.pkl\n#        [0.2090268, 0.20445158, 0.2050881, 0.20080075, 0.20922099],    # ex02C_efficientnet-b4.pkl\n#    ]\n#)\n#test_preds_ensemble = np.zeros((test_df.shape[0], n_classes))\n#for i,pre in enumerate(test_preds_list):\n#    test_preds_ensemble += ens_weights[i] * pre\n#display(pd.DataFrame(test_preds_ensemble, columns=name_mapping.values()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"## oofのcfmにもとづいた重み  20210212_objective3\n## oof_tta=0.9048\n## LB=0.904\n#ens_weights = np.array(\n#    [\n#        [0.2059718, 0.19812068, 0.20305518, 0.1998686, 0.19315833],    # tf_efficientnet_b4_ns_tta\n#        [0.19159524, 0.20038492, 0.20388652, 0.19919596, 0.20556015],  # resnest101e_cleanlab_noise_cutmix_tta\n#        [0.18495991, 0.19551681, 0.18819495, 0.20066639, 0.1988425],   # vit_b16_224_fold10_tta\n#        [0.20873652, 0.20298879, 0.19910631, 0.20047867, 0.19832575],  # 22019613_tta3_efficientnet\n#        [0.20873652, 0.20298879, 0.20575704, 0.19979039, 0.20411327],  # ex02C_tta5_efficientnet\n#    ]\n#)\n#test_preds_ensemble = np.zeros((test_df.shape[0], n_classes))\n#for i,pre in enumerate(test_preds_list):\n#    test_preds_ensemble += ens_weights[i] * pre\n#display(pd.DataFrame(test_preds_ensemble, columns=name_mapping.values()))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"## oofのcfmにもとづいた重み  20210212_objective5\n## oof_tta=0.90578\n## LB=\n#ens_weights = np.array(\n#    [\n#        [0.21317501, 0.19585202, 0.20264682, 0.19972413, 0.1907759],   # tf_efficientnet_b4_ns_tta\n#        [0.19790783, 0.19932735, 0.20212986, 0.19942631, 0.2042922],   # resnest101e_cleanlab_noise_cutmix_tta\n#        [0.1931015, 0.20426009, 0.20119934, 0.20044516, 0.20222864],   # 22019632_tta3_timm\n#        [0.20949958, 0.19977578, 0.20337055, 0.20046083, 0.20367313],  # ex02C_tta5_efficientnet\n#        [0.18631609, 0.20078475, 0.19065343, 0.19994357, 0.19903013],  # vit_base_patch16_384_TTA\n#    ]\n#)\n#test_preds_ensemble = np.zeros((test_df.shape[0], n_classes))\n#for i,pre in enumerate(test_preds_list):\n#    test_preds_ensemble += ens_weights[i] * pre\n#display(pd.DataFrame(test_preds_ensemble, columns=name_mapping.values()))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"## oofのcfmにもとづいた重み  20210212_objective6\n## oof_tta=0.90596\n## LB=\n#ens_weights = np.array(\n#    [\n#        [0.19648426, 0.19977427, 0.20473756, 0.19918661, 0.20534792],  # resnest101e_cleanlab_noise_cutmix_tta\n#        [0.21122767, 0.19751693, 0.20390274, 0.19985922, 0.19295891],  # tf_efficientnet_b4_ns_tta\n#        [0.18627729, 0.19525959, 0.1829281, 0.20114187, 0.19574644],   # vit_base_patch32_384\n#        [0.19364899, 0.20564334, 0.20306793, 0.20003128, 0.20235391],  # 22019632_timm\n#        [0.21236178, 0.20180587, 0.20536366, 0.19978101, 0.20359281],  # ex02C_tta3_efficientnet\n#    ]\n#)\n#test_preds_ensemble = np.zeros((test_df.shape[0], n_classes))\n#for i,pre in enumerate(test_preds_list):\n#    test_preds_ensemble += ens_weights[i] * pre\n#display(pd.DataFrame(test_preds_ensemble, columns=name_mapping.values()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"## oofのcfmにもとづいた重み  20210212_objective7\n## oof_tta=0.90578\n## LB=\n#ens_weights = np.array(\n#    [\n#        [0.20922478, 0.19727891, 0.19879455, 0.19959025, 0.20360584],  # resnest101e_tta\n#        [0.20283412, 0.20147392, 0.20721189, 0.19943385, 0.19096467],  # tf_efficientnet_b4_ns_fold3_tta\n#        [0.18588497, 0.19580499, 0.18819495, 0.20062245, 0.19935758],  # vit_b16_224_fold10_tta\n#        [0.1961656, 0.20340136, 0.2013925, 0.20034094, 0.20153352],    # 22019720_tta3_efficientnet\n#        [0.20589053, 0.20204082, 0.20440611, 0.20001251, 0.20453839],  # ex02C_efficientnet\n#    ]\n#)\n#test_preds_ensemble = np.zeros((test_df.shape[0], n_classes))\n#for i,pre in enumerate(test_preds_list):\n#    test_preds_ensemble += ens_weights[i] * pre\n#display(pd.DataFrame(test_preds_ensemble, columns=name_mapping.values()))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"## optunaで最適化した重み\n## oof = 0.9065289526569145  20210212_objective7\n## LB=0.902\n#ens_weights = np.array(   \n#    [\n#        [0.15752952526994954,0.22514403632959823,0.18918843864327994,0.2479556449888305,0.24244176464181127], # 'resnest101e_tta'\n#        [0.23845076912265822,0.1925789461846951,0.1591652375899216,0.22242568481767636,0.2139059538229616], # 'tf_efficientnet_b4_ns_fold3_tta'\n#        [0.21851624337733444,0.16902723642770212,0.206556619321817,0.21702286601089407,0.19426374746503489], # 'vit_b16_224_fold10_tta'\n#        [0.21952535985009275,0.19421036061074562,0.19408755305253653,0.2073030212336284,0.1825570805293047], # '22019720_tta3_efficientnet-b4\n#        [0.1999030377835539,0.20443143909871822,0.22043869750054165,0.1686958731252343,0.1847032538931056],  # ex02C_efficientnet-b4.pkl\n#    ]\n#)\n#test_preds_ensemble = np.zeros((test_df.shape[0], n_classes))\n#for i,pre in enumerate(test_preds_list):\n#    test_preds_ensemble += ens_weights[i] * pre\n#display(pd.DataFrame(test_preds_ensemble, columns=name_mapping.values()))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-06T17:53:37.6479Z","iopub.status.busy":"2021-02-06T17:53:37.646925Z","iopub.status.idle":"2021-02-06T17:53:37.871437Z","shell.execute_reply":"2021-02-06T17:53:37.872065Z"},"papermill":{"duration":0.401426,"end_time":"2021-02-06T17:53:37.872217","exception":false,"start_time":"2021-02-06T17:53:37.470791","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"test_df['label'] = test_preds_ensemble.argmax(1)\n#test_df.to_csv('submission.csv', index=False)\ndisplay(test_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## stacking"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append(\"../input/cassava-code\")\nfrom lightning_cassava_stacking import (\n    train_stacking,\n    pred_stacking,\n    StackingConfig,\n)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# ノイズとおぼしきサンプルtrainから除くか\nfrom params import base_data\n\n#df = pd.read_csv(f\"{path}/train.csv\")\n#noise_image_id = base_data.noise_image_id\n#noise_idx = list(df[df[\"image_id\"].isin(noise_image_id)].index)\n\nnoise_idx = None\n\n#df = pd.read_csv(f\"{path}/train.csv\")\n#df[\"noise_image_id\"] = df.iloc[noise_idx][\"image_id\"]\n#not_noise_image_id = df[df[\"noise_image_id\"] == False].index.values\n#print(\"len(noise_image_id):\", len(noise_image_id))\n#print(\"len(not_noise_image_id):\", len(not_noise_image_id))\n#\n#\n#def del_df_noise_image_id(preds, y, not_noise_image_id):\n#    preds = [p[not_noise_image_id] for p in preds]\n#    y = y[not_noise_image_id]\n#    return preds, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cnmn2d params\nStackingCFG = StackingConfig()\nStackingCFG.device = device\nStackingCFG.num_workers = num_workers\nStackingCFG.n_classes = n_classes\nStackingCFG.arch = \"cnmn2d\"\n\n# 20210212_objective13\nStackingCFG.cnn2d_params = {'kwargs_head': {'drop_rate': 0.9, \n                                            'n_features_list': [-1, 5], \n                                            'use_bn': False, \n                                            'use_tail_as_out': True, \n                                            'use_wn': False},\n                            'n_channels_list': [1, 8],\n                            'n_classes': n_classes, \n                            'n_models': 5, \n                            'use_bias': False,\n                           }\nStackingCFG.weight_decay = 1e-05\nStackingCFG.smoothing = 0.2\nStackingCFG.t1 = 0.8\nStackingCFG.t2 = 1.3\nStackingCFG.gauss_scale = 0.24\nStackingCFG.cutmix_p = 0.0\nStackingCFG.alpha = 0.2\nStackingCFG.seeds = list(range(2))\ns_type = 1\ngauss_scale = 0.0\nis_noise_only_test = False\nis_del_df_noise_image_id = False\npseudo_th = 0.0\n\nif gauss_scale > 0.0:\n    set_seed(seed=42)\n              \n#StackingCFG.max_epochs = 5\n\nprint(StackingCFG.__dict__)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"%%time\n# cnmn2d semi supervised learning\nStackingCFG.out_dir = StackingCFG.arch\nos.makedirs(StackingCFG.out_dir, exist_ok=True)\n\n\n# load oof\npreds = []\n\n# --------------------------------------------------------------------------------------------\n# ---- anonamename ----\ntta_dir = \"../input/cassava-efficientnetwithpytorchlightning/kaggle_upload_oof_tta\"\n\npkl = f\"{tta_dir}/tf_efficientnet_b4_ns_tta.pkl\"\npred = pickle.load(open(pkl, \"rb\"))\npreds.append(pred.values)\n\npkl = f\"{tta_dir}/byol_seresnext50_32x4d_cutmix_labelsmooth_half_tta.pkl\"\npred = pickle.load(open(pkl, \"rb\"))\npreds.append(pred.values)\n\npkl = f\"{tta_dir}/vit_b16_224_fold10_tta.pkl\"\npred = pickle.load(open(pkl, \"rb\"))\npreds.append(pred.values)\n\n#pkl = f\"{tta_dir}/resnest101e_cleanlab_noise_cutmix_tta.pkl\"\n#pred = pickle.load(open(pkl, \"rb\"))\n#preds.append(pred.values)\n\n#pkl = f\"{tta_dir}/resnest101e_tta.pkl\"\n#pred = pickle.load(open(pkl, \"rb\"))\n#preds.append(pred.values)\n\n#pkl = f\"{tta_dir}/tf_efficientnet_b4_ns_fold3_tta.pkl\"\n#pred = pickle.load(open(pkl, \"rb\"))\n#preds.append(pred.values)\n\n#pkl = f\"{tta_dir}/vit_base_patch32_384.fit_tta.pkl\"\n#pred = pickle.load(open(pkl, \"rb\"))\n#preds.append(pred.values)\n\n# ---- SiNpcw ----\nm_dir = \"../input/cassavapkl\"\n\n#pkl = f\"{m_dir}/22019613_efficientnet-b4.pkl\"\npkl = f\"{m_dir}/22019613_tta3_efficientnet-b4.pkl\"\npred = pickle.load(open(pkl, \"rb\"))\npreds.append(pred.values)\n\n#pkl = f\"{m_dir}/22019632_timm-resnest101e.pkl\"\npkl = f\"{m_dir}/22019632_tta3_timm-resnest101e.pkl\"\npred = pickle.load(open(pkl, \"rb\"))\npreds.append(pred.values)\n\n#pkl = f\"{m_dir}/22019631_tta3_se_resnext101.pkl\"\n#pred = pickle.load(open(pkl, \"rb\"))\n#preds.append(pred.values)\n\n#pkl = f\"{m_dir}/22019717_tta3_timm-resnest101e.pkl\"\n#pred = pickle.load(open(pkl, \"rb\"))\n#preds.append(pred.values)\n\n#pkl = f\"{m_dir}/22019720_tta3_efficientnet-b4.pkl\"\n#pred = pickle.load(open(pkl, \"rb\"))\n#preds.append(pred.values)\n\n#pkl = f\"{m_dir}/ex02C_efficientnet-b4.pkl\"\n#pkl = f\"{m_dir}/ex02C_tta3_efficientnet-b4.pkl\"\n#pkl = f\"{m_dir}/ex02C_tta5_efficientnet-b4.pkl\"\n#pred = pickle.load(open(pkl, \"rb\"))\n#preds.append(pred.values)\n# --------------------------------------------------------------------------------------------\n\n\ndf = pd.read_csv(f\"{path}/train.csv\")\ny = df[\"label\"].values\n\n\n# 過去コンペのデータは除く\npreds = [p[:21397] for p in preds]\n\n\nif is_del_df_noise_image_id:\n    # --- LB0.905の5モデルの内3モデルすべて同じ予測で間違ってるレコード削除 ---    \n    preds, y = del_df_noise_image_id(preds, y, not_noise_image_id)\n\n    \nif pseudo_th > 0.0 and len(test_df) > 1:\n    # --- スコア pseudo_th 以上のサンプルだけ pseudo label に採用する ---\n    test_df_orig = test_df.copy()\n    test_preds_list_orig = test_preds_list[:]\n    test_df['logit'] = test_preds_ensemble.max(1)\n    test_df = test_df[test_df['logit'] > pseudo_th]\n    test_y = test_df['label'].values\n    test_preds_list = [t[test_df['label'].index.values] for t in test_preds_list]\nelse:\n    test_y = test_df['label'].values\n    \n\nif s_type == 0:\n    # --- pseudo label使わない ---\n    x = np.array(preds).transpose(1,2,0)\n    x = x.reshape(len(x), 1, StackingCFG.n_classes, len(preds))\n    \n    oof, oof_loss = train_stacking(x, y, StackingCFG, is_check_model=False, \n                                   noise_idx=noise_idx)\n\nelif s_type == 1:\n    # --- pseudo labelそのまま突っ込む場合 ---\n    preds = [np.vstack((preds[ii], test_preds_list[ii])) for ii in range(len(preds))]\n    y = np.concatenate([y, test_y])\n\n    x = np.array(preds).transpose(1,2,0)\n    x = x.reshape(len(x), 1, StackingCFG.n_classes, len(preds))\n    \n    if gauss_scale > 0.0:\n        # oofとtestにガウスノイズ加算\n        # https://www.kaggle.com/c/stanford-covid-vaccine/discussion/189709\n        x += np.random.normal(0.0, scale=gauss_scale, size=x.shape)\n    \n    oof, oof_loss = train_stacking(x, y, StackingCFG, is_check_model=False, \n                                   noise_idx=noise_idx)\n\nelif s_type == 2:\n    # --- pseudo labelのデータvalidationには入れない ---\n    test_x = np.array(test_preds_list).transpose(1,2,0)\n    test_x = test_x.reshape(len(test_x), 1, StackingCFG.n_classes, len(preds))\n    \n    x = np.array(preds).transpose(1,2,0)\n    x = x.reshape(len(x), 1, StackingCFG.n_classes, len(preds))\n    \n    if gauss_scale > 0.0:\n        if is_noise_only_test:\n            # testにガウスノイズ加算\n            test_x += np.random.normal(0.0, scale=gauss_scale, size=test_x.shape)\n        else:\n            # oofとtestにガウスノイズ加算\n            test_x += np.random.normal(0.0, scale=gauss_scale, size=test_x.shape)\n            x += np.random.normal(0.0, scale=gauss_scale, size=x.shape)\n            \n    oof, oof_loss = train_stacking(x, y, StackingCFG, is_check_model=False, \n                                   add_train_x=test_x, add_train_y=test_y, \n                                   noise_idx=noise_idx)\n    \n\nif pseudo_th > 0.0 and len(test_df) > 1:\n    # 抜いたtestデータ戻す\n    test_df = test_df_orig\n    test_preds_list = test_preds_list_orig\n    \n    \n## predict check\n#pred_cnn2d = pred_stacking(x, StackingCFG)\n#accuracy_score(y, pred_cnn2d.argmax(1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cnmn2d predict\nx = np.array(test_preds_list).transpose(1,2,0)\nx = x.reshape(len(x), 1, StackingCFG.n_classes, len(test_preds_list))\nprint(x.shape)\n\nif gauss_scale > 0.0:\n    x += np.random.normal(0.0, scale=gauss_scale, size=x.shape)\n\npred_cnn2d = pred_stacking(x, StackingCFG)\ndisplay(pd.DataFrame(pred_cnn2d, columns=name_mapping.values()))\n\ntest_df['label'] = pred_cnn2d.argmax(1)\ntest_df[[\"image_id\", \"label\"]].to_csv(f'submission.csv', index=False)\ndisplay(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#import os, shutil\n#if os.path.exists(\"lightning_logs/\"):\n#    shutil.rmtree(\"lightning_logs/\")\n#    os.mkdir(\"lightning_logs/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}