{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h2 style='background:#FFFFFF; border:0; color:black'><center>If you fork it, please give an upvote!<center><h2>","metadata":{}},{"cell_type":"markdown","source":"# About\n* In this notebook, we try to improve the score by ensemble.\n* I made datasets private.\n\n# Source Kernels\n* This notebook was written by refering these great kernels below, so please don't forget to **check and upvote** them.\n* [[No TTA] Cassava Resnext50_32x4d Inference lb0.903](https://www.kaggle.com/piantic/no-tta-cassava-resnext50-32x4d-inference-lb0-903/output)\n* [Clean_Inference_Kernel_8xTTA_LB902](https://www.kaggle.com/underwearfitting/clean-inference-kernel-8xtta-lb902/data)\n* [Cassava-ensemble-(efnetb3-resnet50)](https://www.kaggle.com/shubham108/cassava-ensemble-efnetb3-resnet50)","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Import all relevant library\n# ====================================================\n\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nimport time\nimport random\nimport albumentations\nfrom contextlib import contextmanager\nfrom scipy.special import softmax\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import accuracy_score\nfrom tqdm.auto import tqdm\nimport cv2\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nimport timm\nimport os\nimport warnings","metadata":{"execution":{"iopub.status.busy":"2023-04-10T07:23:20.510238Z","iopub.execute_input":"2023-04-10T07:23:20.510569Z","iopub.status.idle":"2023-04-10T07:23:20.520100Z","shell.execute_reply.started":"2023-04-10T07:23:20.510538Z","shell.execute_reply":"2023-04-10T07:23:20.519223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Define some variables and some settings\n# ====================================================\n\nBATCH_SIZE = 1\nimage_size = 512\nenet_type = ['tf_efficientnet_b8'] * 5\nmodel_path = ['../input/cassava-models/tf_efficientnet_b8_fold_0_7',\n              '../input/cassava-models/tf_efficientnet_b8_fold_1_7',\n              '../input/cassava-models/tf_efficientnet_b8_fold_2_7',\n              '../input/cassava-models/tf_efficientnet_b8_fold_3_7',\n              '../input/cassava-models/tf_efficientnet_b8_fold_4_7']\n\n\n# Directory settings for Resnext\nOUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\nTRAIN_PATH = '../input/cassava-leaf-disease-classification/train_images'\nTEST_PATH = '../input/cassava-leaf-disease-classification/test_images'\n\n# Defines a validation data transformation pipeline\ntransforms_valid = albumentations.Compose([\n    albumentations.CenterCrop(image_size, image_size, p=1),\n    albumentations.Resize(image_size, image_size),\n    albumentations.Normalize()\n])\n\n\n# Sets the random seed\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n# About warnings and GPU availble\nwarnings.filterwarnings('ignore')\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-10T07:23:20.523220Z","iopub.execute_input":"2023-04-10T07:23:20.523524Z","iopub.status.idle":"2023-04-10T07:23:20.535386Z","shell.execute_reply.started":"2023-04-10T07:23:20.523481Z","shell.execute_reply":"2023-04-10T07:23:20.534426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For Resnext\n\n# ====================================================\n# Dataset for Resnext\n# Loading images and their corresponding labels\n# ====================================================\nclass TestDataset(Dataset):    \n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image\n\n\n# ====================================================\n# Define a configuration object for a specific image classification task using the Resnext model\n# ====================================================\nclass CFG:\n    debug=False\n    num_workers=8\n    model_name='resnext50_32x4d'\n    size=512\n    batch_size=32\n    seed=2020\n    target_size=5\n    target_col='label'\n    n_fold=5\n    trn_fold=[0, 1, 2, 3, 4]\n    inference=True\n\n# ====================================================\n# Helper functions for Resnext\n# Loads the state of a pre-trained model from a given model path\n# ====================================================\ndef load_state(model,model_path):\n    try:  # single GPU model_file\n        model.load_state_dict(torch.load(model_path)['model'], strict=True)\n        state_dict = torch.load(model_path)['model']\n    except:  # multi GPU model_file\n        state_dict = torch.load(model_path)['model']\n        state_dict = {k[7:] if k.startswith('module.') else k: state_dict[k] for k in state_dict.keys()}\n\n    return state_dict\n\n# ====================================================\n#  Performs inference on test data using the provided model and model states for Resnext\n# ====================================================\ndef inference(model, states, test_loader, device):\n    model.to(device)\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state)\n            model.eval()\n            with torch.no_grad():\n                y_preds = model(images)\n            avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs\n\n\n\n# ====================================================\n# Utils for Resnext\n# Computes the accuracy score\n# About logs\n# ====================================================\n\ndef get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n\ndef init_logger(log_file=OUTPUT_DIR+'inference.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = init_logger()\n\n\n# ====================================================\n# Transforms for Resnext\n# ====================================================\ndef get_transforms(*, data):\n    if data == 'valid':\n        return A.Compose([\n            A.Resize(CFG.size, CFG.size),\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n    \n# ====================================================\n# ResNext Model\n# ====================================================\nclass CustomResNext(nn.Module):\n    def __init__(self, model_name='resnext50_32x4d', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n    ","metadata":{"execution":{"iopub.status.busy":"2023-04-10T07:23:20.537066Z","iopub.execute_input":"2023-04-10T07:23:20.537427Z","iopub.status.idle":"2023-04-10T07:23:20.562603Z","shell.execute_reply.started":"2023-04-10T07:23:20.537392Z","shell.execute_reply":"2023-04-10T07:23:20.561937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For Efficientnet\n\n# ====================================================\n# Dataset for efficientnet\n# Loading images and their corresponding labels\n# ====================================================\nclass CLDDataset(Dataset):\n    def __init__(self, df, mode, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.mode = mode\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, index):\n        row = self.df.loc[index]\n        image = cv2.imread(row.filepath)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        if self.transform is not None:\n            res = self.transform(image=image)\n            image = res['image']\n\n        image = image.astype(np.float32)\n        image = image.transpose(2, 0, 1)\n        if self.mode == 'test':\n            return torch.tensor(image).float()\n        else:\n            return torch.tensor(image).float(), torch.tensor(row.label).float()\n\n\n# ====================================================\n# Defines the model architecture for the task\n# ====================================================\nclass CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        # self.model.classifier = nn.Linear(n_features, n_class)\n        # '''\n        self.model.classifier = nn.Sequential(\n            nn.Dropout(0.3),\n            #nn.Linear(n_features, hidden_size,bias=True), nn.ELU(),\n            nn.Linear(n_features, n_class, bias=True)\n        )\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \n\n# ====================================================\n#  Helper functions for Efficientnet\n#  Performs inference on test data using the provided model and model states for Efficientnet\n# ====================================================\ndef inference_func(test_loader):\n    model.eval()\n    bar = tqdm(test_loader)\n\n    LOGITS = []\n    PREDS = []\n\n    with torch.no_grad():\n        for batch_idx, images in enumerate(bar):\n            x = images.to(device)\n            logits = model(x)\n            LOGITS.append(logits.cpu())\n            PREDS += [torch.softmax(logits, 1).detach().cpu()]\n        PREDS = torch.cat(PREDS).cpu().numpy()\n        LOGITS = torch.cat(LOGITS).cpu().numpy()\n    return PREDS\n\n# ====================================================\n#performs test-time augmentation inference on test data using the provided model\n# ====================================================\ndef tta_inference_func(test_loader):\n    model.eval()\n    bar = tqdm(test_loader)\n    PREDS = []\n    LOGITS = []\n\n    with torch.no_grad():\n        for batch_idx, images in enumerate(bar):\n            x = images.to(device)\n            x = torch.stack([x, x.flip(-1), x.flip(-2), x.flip(-1, -2),\n                             x.transpose(-1, -2), x.transpose(-1, -2).flip(-1),\n                             x.transpose(-1, -2).flip(-2), x.transpose(-1, -2).flip(-1, -2)], 0)\n            x = x.view(-1, 3, image_size, image_size)\n            logits = model(x)\n            logits = logits.view(BATCH_SIZE, 8, -1).mean(1)\n            PREDS += [torch.softmax(logits, 1).detach().cpu()]\n            LOGITS.append(logits.cpu())\n        PREDS = torch.cat(PREDS).cpu().numpy()\n    return PREDS\n\n# ====================================================\n# Define a configuration object for a specific image classification task using the Efficientnet_b8 model\n# ====================================================\nclass CFG_b8:\n    debug=False\n    num_workers=1\n    model_name='tf_efficientnet_b8'\n    size=512\n    batch_size=32\n    seed=2020\n    target_size=5\n    target_col='label'\n    n_fold=5\n    trn_fold=[0, 1, 2, 3, 4]\n    inference=True\n    \n    \n","metadata":{"execution":{"iopub.status.busy":"2023-04-10T07:23:20.647050Z","iopub.execute_input":"2023-04-10T07:23:20.647352Z","iopub.status.idle":"2023-04-10T07:23:20.789184Z","shell.execute_reply.started":"2023-04-10T07:23:20.647324Z","shell.execute_reply":"2023-04-10T07:23:20.787962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# inference and Submit","metadata":{}},{"cell_type":"code","source":"# Set the random seed for reproducibility during the training and inference process\nseed_torch(seed=CFG_b8.seed)\ntest = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\ntest['filepath'] = test.image_id.apply(\n    lambda x: os.path.join('../input/cassava-leaf-disease-classification/test_images', f'{x}'))\n\n#RestNet\nMODEL_DIR = '../input/cassava-resnext50-32x4d-weights/'\n\nmodel_res = CustomResNext(CFG.model_name, pretrained=False)\n\nstates = [load_state(model_res,MODEL_DIR + f'{CFG.model_name}_fold{fold}.pth') for fold in CFG.trn_fold]\n\ntest_dataset = CLDDataset(test, 'test', transform=transforms_valid)\n\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False,\n                         num_workers=CFG.num_workers, pin_memory=True)\npredictions = inference(model_res, states, test_loader, device)\n\n\ntest_dataset_efficient = CLDDataset(test, 'test', transform=transforms_valid)\ntest_loader_efficient = torch.utils.data.DataLoader(test_dataset_efficient, batch_size=BATCH_SIZE, shuffle=False,\n                                                    num_workers=1)\n\n\n# for Efficientnet\ntest_preds = []\nfor i in range(len(enet_type)):\n    model = CassvaImgClassifier(enet_type[i], n_class=5)\n    model = model.to(device)\n    model.load_state_dict(torch.load(model_path[i]))\n    test_preds += [tta_inference_func(test_loader_efficient)]\n    \n# submission\npred = 0.8 * predictions + 0.2 * np.mean(test_preds, axis=0)\n# pred = np.mean(test_preds, axis=0)\ntest['label'] = softmax(pred).argmax(1)\ntest[['image_id', 'label']].to_csv(OUTPUT_DIR + '/submission.csv', index=False)\n\nprint(test.head())\nprint(test.head())\nsoftmax(np.mean(test_preds, axis=0)).argmax()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T07:23:20.791103Z","iopub.execute_input":"2023-04-10T07:23:20.791509Z","iopub.status.idle":"2023-04-10T07:23:49.929618Z","shell.execute_reply.started":"2023-04-10T07:23:20.791470Z","shell.execute_reply":"2023-04-10T07:23:49.928661Z"},"trusted":true},"execution_count":null,"outputs":[]}]}