{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Sheep"},{"metadata":{"_kg_hide-output":true,"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# environment init\n!pip install ../input/panda-essential/pretrained-models.pytorch-master/pretrained-models.pytorch-master/\n!pip install ../input/landmark-additional-packages/EfficientNet-PyTorch/EfficientNet-PyTorch-master/\n!pip install ../input/../input/landmark-additional-packages/timm-0.3.4-py3-none-any.whl\n\n!mkdir /root/.cache/torch\n!mkdir /root/.cache/torch/hub/\n!mkdir /root/.cache/torch/hub/checkpoints\n!cp -r ../input/landmark-additional-packages/rwightman_gen-efficientnet-pytorch_master/rwightman_gen-efficientnet-pytorch_master /root/.cache/torch/hub/\n!cp -r ../input/landmark-additional-packages/tf_efficientnet_* /root/.cache/torch/hub/checkpoints\n!cp -r ../input/landmark-additional-packages/se* /root/.cache/torch/hub/checkpoints\n!cp -r ../input/landmark-additional-packages/resnet200d* /root/.cache/torch/hub/checkpoints\n!cp -r ../input/landmark-additional-packages/dense* /root/.cache/torch/hub/checkpoints","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /root/.cache/torch/hub/checkpoints","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir leaf\n!cp -r ../input/leaf-search-7fa0ec2/leaf-search/* leaf\n!cp ../input/leaf-search-7fa0ec2/leaf-search/notebook_infer.py .","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nfrom subprocess import call\nimport numpy as np\nimport pandas as pd\nget_score = lambda x: float(x.split('/')[-1][-10:-4])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cmd = ('python notebook_infer.py --model={} ' + \n        '--config={} ' +\n       '--df=../input/cassava-leaf-disease-classification/sample_submission.csv --output=test')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model list\n```python\n'../input/leaf-0210-four-models/0129_dense_dpp_cutmix_744_n1_sin_aug_cut_1_fn/0129_dense_dpp_cutmix_744_n1_sin_aug_cut_1_fn'\n'../input/leaf-0129-eff-se50-three-model/0129_eff_dpp_cutmix_eff_16_ext_v1_f0_n1_sin_aug_cut_1_744_beta6_f/0129_eff_dpp_cutmix_eff_16_ext_v1_f0_n1_sin_aug_cut_1_744_beta6_f1.yaml'\n'../input/leaf-0210-four-models/0209_upload_eb7_eb7_fn/0209_upload_eb7_eb7_f0.yaml'\n'../input/leaf-0210-four-models/0209_upload_r200d_light_beta6_800_16_r200d_800_16_beta6_fn_16/0209_upload_r200d_light_beta6_800_16_r200d_800_16_beta6_fn_16'\n'../input/leaf-0210-four-models/0209_upload_se50_full_se50_744_sin_full_fn/0209_upload_se50_full_se50_744_sin_full_fn'\n'../input/leaf-0129-eff-se50-three-model/upload_clean_ext_no_ext_sin_clean_600_drop_snapmix_32_ext_v1_fn_n1_sin_aug_cut_0.25.yaml/upload_clean_ext_no_ext_sin_clean_600_drop_snapmix_32_ext_v1_fn_n1_sin_aug_cut_0.25.yaml'\n\n```"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_1 = '../input/leaf-0210-four-models/0129_dense_dpp_cutmix_744_n1_sin_aug_cut_1_fn/0129_dense_dpp_cutmix_744_n1_sin_aug_cut_1_fn'\nmodel_2 = '../input/leaf-0129-eff-se50-three-model/0129_eff_dpp_cutmix_eff_16_ext_v1_f0_n1_sin_aug_cut_1_744_beta6_f1/0129_eff_dpp_cutmix_eff_16_ext_v1_f0_n1_sin_aug_cut_1_744_beta6_f1.yaml'\nmodel_3 = '../input/leaf-0210-four-models/0209_upload_eb7_eb7_fn/0209_upload_eb7_eb7_f0.yaml'\nmodel_4 = '../input/leaf-0210-four-models/0209_upload_r200d_light_beta6_800_16_r200d_800_16_beta6_fn_16/0209_upload_r200d_light_beta6_800_16_r200d_800_16_beta6_fn_16'\nmodel_5 = '../input/leaf-0210-four-models/0209_upload_se50_full_se50_744_sin_full_fn/0209_upload_se50_full_se50_744_sin_full_fn'\nmodel_6 = '../input/leaf-0129-eff-se50-three-model/upload_clean_ext_no_ext_sin_clean_600_drop_snapmix_32_ext_v1_fn_n1_sin_aug_cut_0.25.yaml/upload_clean_ext_no_ext_sin_clean_600_drop_snapmix_32_ext_v1_fn_n1_sin_aug_cut_0.25.yaml'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preds"},{"metadata":{"trusted":true},"cell_type":"code","source":"sheep_models = []\nfor m in [model_1, model_2, model_3, model_4, model_5, model_6]:\n    preds = []\n    for f in range(5):\n        mdl = glob.glob(f'{m}/f{f}/*.pth')[0]\n        cfg = glob.glob(f'{m}/f{f}/*.json')[0]\n        call(cmd.format(mdl, cfg), shell=True)\n        preds.append(np.load('test.npy'))\n    pred = np.zeros_like(preds[0])\n    for x in preds:\n        pred += x\n    pred = pred / 5\n    sheep_models.append(pred)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Combine Preds"},{"metadata":{"trusted":true},"cell_type":"code","source":"sheep_models","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# sin"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport pandas as pd\nimport numpy as np\nimport time\nimport cv2\nimport PIL.Image\nfrom sklearn.metrics import accuracy_score\nfrom tqdm import tqdm\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nimport albumentations\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pylab import rcParams\nimport joblib\nimport lightgbm as lgb\nimport timm\nimport glob\ndevice = torch.device('cuda') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CLDDataset(Dataset):\n    def __init__(self, df, mode, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.mode = mode\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        row = self.df.loc[index]\n        image = cv2.imread(row.filepath)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if self.transform is not None:\n            res = self.transform(image=image)\n            image = res['image']\n        \n        image = image.astype(np.float32)\n        image = image.transpose(2,0,1)\n        if self.mode == 'test':\n            return torch.tensor(image).float()\n        else:\n            return torch.tensor(image).float(), torch.tensor(row.label).float()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nclass CustomTimmModel(nn.Module):\n    def __init__(self, backbone, out_dim, pretrained=True):\n        super().__init__()\n        self.model = timm.create_model(backbone, pretrained=pretrained)\n        if 'efficientnet' in backbone or 'densenet' in backbone:\n            in_ch = self.model.classifier.in_features\n            self.model.classifier = nn.Identity() \n        elif 'resnext' in backbone or 'resnet' in backbone or 'resnest' in backbone:\n            in_ch = self.model.fc.in_features\n            self.model.fc = nn.Identity() \n        elif 'vit' in backbone:\n            in_ch = self.model.head.in_features\n            self.model.head = nn.Identity()\n        elif 'csp' in backbone:\n            in_ch = self.model.head.fc.in_features\n            self.model.head.fc = nn.Identity()\n            \n        self.myfc = nn.Linear(in_ch, out_dim)\n    def forward(self, x):\n        x = self.model(x)\n        x = self.myfc(x)\n        return x\n\n\n\nclass CLDResNext(nn.Module):\n\n    def __init__(self, backbone, out_dim, pretrained=True):\n        super().__init__()\n        self.model = timm.create_model(backbone, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, out_dim)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n\nclass CLDVit(nn.Module):\n    def __init__(self, backbone, out_dim, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(backbone, pretrained=pretrained)\n        n_features = self.model.head.in_features\n        self.model.head = nn.Linear(n_features, out_dim)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n\n\n\ndef inference_func(test_loader):\n    model.eval()\n    print('Predicting...')\n    LOGITS = []\n    PREDS = []\n    \n    with torch.no_grad():\n        for images in test_loader:\n            x = images.to(device)\n            logits = model(x)\n            LOGITS.append(logits.cpu())\n            PREDS += [torch.softmax(logits, 1).detach().cpu()]\n        PREDS = torch.cat(PREDS).cpu().numpy()\n        LOGITS = torch.cat(LOGITS).cpu().numpy()\n    return PREDS\n\n\n\ntest = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\ntest['filepath'] = test.image_id.apply(lambda x: os.path.join('../input/cassava-leaf-disease-classification/test_images', f'{x}'))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sin_models = []","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Vit"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nimage_size = 384\nmodel_type = ['vit_base_patch16_384'] * 5\nmodel_path = ['../input/cass-vit-base-384/fold0.pth', \n              '../input/cass-vit-base-384/fold1.pth', \n              '../input/cass-vit-base-384/fold2.pth',\n              '../input/cass-vit-base-384/fold3.pth',\n              '../input/cass-vit-base-384/fold4.pth']\n\ntransforms_valid = albumentations.Compose([\n    albumentations.CenterCrop(image_size, image_size),\n    albumentations.Normalize()\n])\n\n\ntest_dataset = CLDDataset(test, 'test', transform=transforms_valid)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False,  num_workers=4)\n\ntest_preds = []\nfor i in range(len(model_type)):\n    model = CLDVit(model_type[i], out_dim=5, pretrained=False)\n    model = model.to(device)\n    model.load_state_dict(torch.load(model_path[i], map_location='cuda:0'))\n    test_preds += [inference_func(test_loader)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sin_models.append(test_preds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## B4ns_512"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nimage_size = 512\nmodel_type = ['tf_efficientnet_b4_ns'] * 5\nmodel_path = ['../input/b4ns-512/fold0.pth', \n              '../input/b4ns-512/fold1.pth', \n              '../input/b4ns-512/fold2.pth',\n              '../input/b4ns-512/fold3.pth',\n              '../input/b4ns-512/fold4.pth']\n\ntransforms_valid = albumentations.Compose([\n    albumentations.Resize(image_size, image_size),\n    albumentations.Normalize()\n])\n\n\ntest_dataset = CLDDataset(test, 'test', transform=transforms_valid)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False,  num_workers=4)\n\ntest_preds = []\nfor i in range(len(model_type)):\n    model = CustomTimmModel(model_type[i], out_dim=5, pretrained=False)\n    model = model.to(device)\n    model.load_state_dict(torch.load(model_path[i], map_location='cuda:0'))\n    test_preds += [inference_func(test_loader)]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sin_models.append(test_preds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Resnest50"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 24\nimage_size = 700\nmodel_type = ['resnest50d_1s4x24d'] * 5\nmodel_path = ['../input/resnest50-pl-cv898/fold0.pth', \n              '../input/resnest50-pl-cv898/fold1.pth', \n              '../input/resnest50-pl-cv898/fold2.pth',\n              '../input/resnest50-pl-cv898/fold3.pth',\n              '../input/resnest50-pl-cv898/fold4.pth',]\n\ntransforms_valid = albumentations.Compose([\n    albumentations.Resize(image_size, image_size),\n    albumentations.Normalize()\n])\n\n\ntest_dataset = CLDDataset(test, 'test', transform=transforms_valid)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False,  num_workers=4)\n\ntest_preds = []\nfor i in range(len(model_type)):\n    model = CustomTimmModel(model_type[i], out_dim=5, pretrained=False)\n    model = model.to(device)\n    model.load_state_dict(torch.load(model_path[i], map_location='cuda:0'))\n    test_preds += [inference_func(test_loader)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sin_models.append(test_preds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Darknet"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nimage_size = 648\nmodel_type = ['cspdarknet53'] * 5\nmodel_path = ['../input/cspdarknet53/fold0.pth', \n              '../input/cspdarknet53/fold1.pth', \n              '../input/cspdarknet53/fold2.pth',\n              '../input/cspdarknet53/fold3.pth',\n              '../input/cspdarknet53/fold4.pth',]\n\ntransforms_valid = albumentations.Compose([\n    albumentations.Resize(image_size, image_size),\n    albumentations.Normalize()\n])\n\n\ntest_dataset = CLDDataset(test, 'test', transform=transforms_valid)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False,  num_workers=4)\n\ntest_preds = []\nfor i in range(len(model_type)):\n    model = CustomTimmModel(model_type[i], out_dim=5, pretrained=False)\n    model = model.to(device)\n    model.load_state_dict(torch.load(model_path[i], map_location='cuda:0'))\n    test_preds += [inference_func(test_loader)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sin_models.append(test_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sin_models = [np.mean(i, axis=0) for i in sin_models]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Holy Chen"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaClassifier(nn.Module):\n    def __init__(self, model_arch, num_classes, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        ### EffNet\n        if model_arch == 'tf_efficientnet_b4_ns' or model_arch == 'tf_efficientnet_b5_ns':\n            num_features = self.model.classifier.in_features\n            self.model.classifier = nn.Linear(num_features, num_classes)\n           \n        ### Vit\n        if model_arch == 'vit_base_patch16_384':\n            num_features = self.model.head.in_features\n            self.model.head = nn.Linear(num_features, num_classes)\n\n        ### ResNet\n        if model_arch == 'resnext50d_32x4d' or model_arch == 'resnet200d_320' or model_arch == 'seresnet152d_320':\n            num_features = self.model.fc.in_features\n            self.model.fc = nn.Linear(num_features, num_classes)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"chen_models = []","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Vit"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nimage_size = 384\nmodel_type = ['vit_base_patch16_384'] * 5\nmodel_path = ['../input/cldcvitb16sin/models/vit_base_patch16_384_fold0_best.ckpt', \n              '../input/cldcvitb16sin/models/vit_base_patch16_384_fold1_best.ckpt', \n              '../input/cldcvitb16sin/models/vit_base_patch16_384_fold2_best.ckpt',\n              '../input/cldcvitb16sin/models/vit_base_patch16_384_fold3_best.ckpt',\n              '../input/cldcvitb16sin/models/vit_base_patch16_384_fold4_best.ckpt',]\n\ntransforms_valid = albumentations.Compose([\n    albumentations.CenterCrop(image_size, image_size),\n    albumentations.Normalize()\n])\n\n\ntest_dataset = CLDDataset(test, 'test', transform=transforms_valid)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False,  num_workers=4)\n\ntest_preds = []\nfor i in range(len(model_type)):\n    model = CassavaClassifier(model_type[i], 5, pretrained=False)\n    model = model.to(device)\n    model.load_state_dict(torch.load(model_path[i], map_location='cuda:0'))\n    test_preds += [inference_func(test_loader)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"chen_models.append(test_preds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## resnet200d-320"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nimage_size = 512\nmodel_type = ['resnet200d_320'] * 5\nmodel_path = ['../input/cldcresnet200d-320sin/models/resnet200d_320_fold0_best.ckpt', \n              '../input/cldcresnet200d-320sin/models/resnet200d_320_fold1_best.ckpt', \n              '../input/cldcresnet200d-320sin/models/resnet200d_320_fold2_best.ckpt',\n              '../input/cldcresnet200d-320sin/models/resnet200d_320_fold3_best.ckpt',\n              '../input/cldcresnet200d-320sin/models/resnet200d_320_fold4_best.ckpt',]\n\ntransforms_valid = albumentations.Compose([\n    albumentations.CenterCrop(image_size, image_size),\n    albumentations.Normalize()\n])\n\n\ntest_dataset = CLDDataset(test, 'test', transform=transforms_valid)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False,  num_workers=4)\n\ntest_preds = []\nfor i in range(len(model_type)):\n    model = CassavaClassifier(model_type[i], 5, pretrained=False)\n    model = model.to(device)\n    model.load_state_dict(torch.load(model_path[i], map_location='cuda:0'))\n    test_preds += [inference_func(test_loader)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"chen_models.append(test_preds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## tf_efficientnet_b4_ns"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nimage_size = 512\nmodel_type = ['tf_efficientnet_b4_ns'] * 5\nmodel_path = ['../input/cldcb4sin/models/tf_efficientnet_b4_ns_fold0_best.ckpt', \n              '../input/cldcb4sin/models/tf_efficientnet_b4_ns_fold1_best.ckpt', \n              '../input/cldcb4sin/models/tf_efficientnet_b4_ns_fold2_best.ckpt',\n              '../input/cldcb4sin/models/tf_efficientnet_b4_ns_fold3_best.ckpt',\n              '../input/cldcb4sin/models/tf_efficientnet_b4_ns_fold4_best.ckpt',]\n\ntransforms_valid = albumentations.Compose([\n    albumentations.CenterCrop(image_size, image_size),\n    albumentations.Normalize()\n])\n\n\ntest_dataset = CLDDataset(test, 'test', transform=transforms_valid)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False,  num_workers=4)\n\ntest_preds = []\nfor i in range(len(model_type)):\n    model = CassavaClassifier(model_type[i], 5, pretrained=False)\n    model = model.to(device)\n    model.load_state_dict(torch.load(model_path[i], map_location='cuda:0'))\n    test_preds += [inference_func(test_loader)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"chen_models.append(test_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"chen_models = [np.mean(i, axis=0) for i in chen_models]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Stacking"},{"metadata":{"trusted":true},"cell_type":"code","source":"sin_models.extend(sheep_models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sin_models.extend(chen_models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_overall = np.hstack(np.array(sin_models).squeeze())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = []\nfor model in glob.glob('../input/13model-stacking/*.pkl'):\n    gbm = joblib.load(model)\n    try:\n        preds += [gbm.predict(preds_overall)]\n    except:\n        preds += [gbm.predict(preds_overall.reshape(1,-1))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.label = np.argmax(np.mean(preds, axis=0), axis=1)\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}