{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Plant Pathology 2021: Inference of EfficientNet model","metadata":{}},{"cell_type":"markdown","source":"Here is an inference notebook based on [PyTorch EfficientNet](https://github.com/lukemelas/EfficientNet-PyTorch). The code for training models is [in this notebook](https://www.kaggle.com/vgarshin/plant-efficientnet-train-pytorch) or you may find code for local training [on GitHub](https://github.com/vgarshin/kaggle_plant).","metadata":{}},{"cell_type":"code","source":"%%time\n!pip install ../input/pytorch-model/EfficientNet-PyTorch-master -f ./ --no-index","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport json\nimport time\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.utils.data as data\nimport torchvision\nfrom torchvision import models, transforms\nfrom torch.utils.data.sampler import SequentialSampler\nfrom efficientnet_pytorch import model as enet\nfrom tqdm.notebook import tqdm\nfrom os import path as osp\nimport glob\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom PIL import Image\n\n\nKAGGLE = True\nif not KAGGLE: os.environ['CUDA_VISIBLE_DEVICES'] = '0' \nelse: pass\nDEVICE = torch.device('cuda')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST = False\nVER = 'v0'\nif KAGGLE:\n    DATA_PATH = '../input/plant-pathology-2021-fgvc8'\n    MDLS_PATH = f'../input/plant-efficientnet-train-pytorch/models_{VER}'\nelse:\n    DATA_PATH = './data'\n    MDLS_PATH = f'./models_{VER}'\nTTAS = [0, 1, 2]\nFOLDS = [0]\nIMGS_PATH = f'{DATA_PATH}/test_images' if TEST else f'{DATA_PATH}/train_images'\n\nstart_time = time.time()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ndf_train = pd.read_csv(\"/kaggle/input/plant-pathology-2021-fgvc8/train.csv\")\ndf_sub = pd.read_csv(\"/kaggle/input/plant-pathology-2021-fgvc8/sample_submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_label(df):\n    \"\"\"\n    Function for Label encoding.\n    \"\"\"\n    le = LabelEncoder()\n    df[\"labels_n\"] = le.fit_transform(df.labels.values)\n    return df\n\ndf_train = to_label(df_train)\ndf_labels_idx = df_train.loc[df_train.duplicated([\"labels\", \"labels_n\"])==False]\\\n                [[\"labels_n\", \"labels\"]].set_index(\"labels_n\").sort_index()\ndisplay(df_labels_idx)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(f'{MDLS_PATH}/params.json') as file:\n    params = json.load(file)\nLABELS_ = params['labels_']\nLABELS = params['labels']\nWORKERS = 2 if KAGGLE else params['workers']\nprint('loaded params:', params)\n\nwith open(f'{MDLS_PATH}/ths.json') as file:\n    ths = json.load(file)\nprint('thresholds:', ths)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.DataFrame(os.listdir(IMGS_PATH)) if TEST else pd.DataFrame(os.listdir(IMGS_PATH)[:100])\ndf_sub.columns = ['image']\ndf_sub['labels'] = 'healthy'\ndisplay(df_sub.head())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImageTransform():\n    \"\"\"\n    Class for image preprocessing.\n    \n    Attributes\n    ----------\n    resize : int\n        224\n    mean : (R, G, B)\n        Average value for each color channel\n    std : (R, G, B)\n        Standard deviation for each color channel\n    \"\"\"\n    \n    def __init__(self, resize, mean, std):\n        self.data_transform = {\n#             'train': A.Compose(albumentation_list),\n            'train': transforms.Compose([\n                transforms.Resize(resize),\n                transforms.RandomResizedCrop(resize, scale=(0.5, 1.0)),\n                transforms.RandomHorizontalFlip(),\n                transforms.RandomPerspective(),\n                transforms.ToTensor(),\n#                 transforms.RandomRotation(),\n                transforms.Normalize(mean, std)\n            ]),\n            'val': transforms.Compose([\n                transforms.Resize(resize),\n                transforms.CenterCrop(resize),\n                transforms.ToTensor(),\n                transforms.Normalize(mean, std)\n            ]),\n            'test': transforms.Compose([\n                transforms.Resize(resize),\n                transforms.CenterCrop(resize),\n                transforms.ToTensor(),\n                transforms.Normalize(mean, std)\n            ])\n        }\n    \n    def __call__(self, img, phase=\"train\"):\n        \"\"\"\n        Parameters\n        ----------\n        phase: 'train' or 'val' or 'test'\n            Specify the mode of preprocessing\n        \"\"\"\n        \n        return self.data_transform[phase](img)\n#         return self.data_transform[phase](image=img).get('image')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PlantDataset(data.Dataset):\n    \"\"\"\n    Class to create a Dataset\n    \n    Attributes\n    ----------\n    df_train : DataFrame\n        DataFrame containing the image labels.\n    file_list : list\n        A list containing the paths to the images\n    transform : object\n        Instance of the preprocessing class (ImageTransform)\n    phase : 'train' or 'val' or 'test'\n        Specify whether to use train, validation, or test\n    \"\"\"\n    def __init__(self, df_train, file_list, transform=None, phase='train'):\n        self.df_train = df_train\n        self.df_labels_idx = df_labels_idx\n        self.file_list = file_list\n        self.transform = transform\n        self.phase = phase\n   \n    def __len__(self):\n        \"\"\"\n        Returns the number of images.\n        \"\"\"\n        return len(self.file_list)\n    \n    def __getitem__(self, index):\n        \"\"\"\n        Get data in Tensor format and labels of preprocessed images.\n        \"\"\"\n        #print(index)\n        \n        # Load the index number image.\n        img_path = self.file_list[index]\n        img = Image.open(img_path)\n#         img = cv2.imread(img_path)\n#         img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        # Preprocessing images\n        img_transformed = self.transform(img, self.phase)\n        \n        # image name\n        image_name = img_path[-20:]\n        \n        # Extract the labels\n        if self.phase in [\"train\", \"val\"]:\n            label = df_train.loc[df_train[\"image\"]==image_name][\"labels_n\"].values[0]\n        elif self.phase in [\"test\"]:\n            label = -1\n        \n        return img_transformed, label, image_name","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def flip(img, axis=0):\n    if axis == 1:\n        return img[::-1, :, ]\n    elif axis == 2:\n        return img[:, ::-1, ]\n    elif axis == 3:\n        return img[::-1, ::-1, ]\n    else:\n        return img\n\n# class PlantDataset(data.Dataset):\n    \n#     def __init__(self, df, size, labels, transform=None, tta=0):\n#         self.df = df.reset_index(drop=True)\n#         self.size = size\n#         self.labels = labels\n#         self.transform = transform\n#         self.tta = tta\n    \n#     def __len__(self):\n#         return self.df.shape[0]\n    \n#     def __getitem__(self, index):\n#         row = self.df.iloc[index]\n#         img_name = row.image\n#         img_path = f'{IMGS_PATH}/{img_name}'\n#         img = cv2.imread(img_path)\n#         if not np.any(img):\n#             print('no img file read:', img_path)\n#         img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n#         img = cv2.resize(img, (self.size, self.size))\n#         img = img.astype(np.float32) / 255\n#         if self.transform is not None:\n#             img = self.transform(image=img)['image']\n#         if self.labels:\n#             img = img.transpose(2, 0, 1)\n#             label = np.zeros(len(self.labels)).astype(np.float32)\n#             for lbl in row.labels.split():\n#                 label[self.labels[lbl]] = 1\n#             return torch.tensor(img), torch.tensor(label)\n#         else:\n#             img = flip(img, axis=self.tta)\n#             img = img.transpose(2, 0, 1)\n#             return torch.tensor(img.copy())\n\nclass EffNet(nn.Module):\n    \n    def __init__(self, params, out_dim):\n        super(EffNet, self).__init__()\n        self.enet = enet.EfficientNet.from_name(params['backbone'])\n        nc = self.enet._fc.in_features\n        self.enet._fc = nn.Identity()\n        self.myfc = nn.Sequential(\n            nn.Dropout(params['dropout']),\n            nn.Linear(nc, int(nc / 4)),\n            nn.Dropout(params['dropout']),\n            nn.Linear(int(nc / 4), out_dim)\n        )\n        \n    def extract(self, x):\n        return self.enet(x)\n    \n    def forward(self, x):\n        x = self.extract(x)\n        x = self.myfc(x)\n        return x\n\nclass ResNext(nn.Module):\n    \n    def __init__(self, params, out_dim):\n        super(ResNext, self).__init__()\n        self.rsnxt = torchvision.models.resnext50_32x4d(pretrained=False)\n        nc = self.rsnxt.fc.in_features\n        self.rsnxt.fc = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(nc, int(nc / 4)),\n            nn.ReLU(),\n            nn.Dropout(params['dropout']),\n            nn.Linear(int(nc / 4), out_dim)\n        )\n        self.rsnxt = nn.DataParallel(self.rsnxt)\n        \n    def forward(self, x):\n        x = self.rsnxt(x)\n        return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = []\nfor n_fold in FOLDS:\n    if params['backbone'] == 'resnext':\n        model = ResNext(params=params, out_dim=len(LABELS_)) \n    else:\n        model = EffNet(params=params, out_dim=len(LABELS_)) \n    path = '{}/model_best_{}.pth'.format(MDLS_PATH, n_fold)\n    state_dict = torch.load(path, map_location=torch.device('cpu'))\n    model.load_state_dict(state_dict)\n    model.float()\n    model.eval()\n    model.cuda()\n    models.append(model)\n    print('loaded:', path)\ndel state_dict, model\ngc.collect();\n\nmodels[0].train()","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# datasets, loaders = [], []\n# for tta in TTAS:\n#     dataset = PlantDataset(\n#         df=df_sub,\n#         size=params['img_size'],\n#         labels=None,\n#         transform=None,\n#         tta=tta)\n#     datasets.append(dataset)\n#     loader = torch.utils.data.DataLoader(\n#         dataset, \n#         batch_size=params['batch_size'], \n#         sampler=SequentialSampler(dataset), \n#         num_workers=WORKERS)\n#     loaders.append(loader)\n    \n# data_loader_dict = {'train': }","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_IMAGE_PATH = '/kaggle/input/resized-plant2021/img_sz_256'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_datapath_list(phase=\"train\", val_size=0.25):\n    \"\"\"\n    Function to create a PATH to the data.\n    \n    Parameters\n    ----------\n    phase : 'train' or 'val' or 'test'\n        Specify whether to use Train data or test data.\n    val_size : float\n        Ratio of validation data to train data\n        \n    Returns\n    -------\n    path_lsit : list\n        A list containing the PATH to the data.\n    \"\"\"\n    \n    if phase in [\"train\", \"val\"]:\n        phase_path = \"train_images\"\n    elif phase in [\"test\"]:\n        phase_path = \"test_images\"\n    else:\n        print(f\"{phase} not in path\")\n    rootpath = \"/kaggle/input/plant-pathology-2021-fgvc8/\"\n#     rootpath = \"/kaggle/input/resized-plant2021/img_sz_256/\"\n    target_path = osp.join(TRAIN_IMAGE_PATH , '*.jpg') if  phase in ['train', 'val'] else osp.join(rootpath+phase_path+\"/*.jpg\")\n\n    path_list = []\n    \n    for path in glob.glob(target_path):\n        path_list.append(path)\n        \n    if phase in [\"train\", \"val\"]:\n        train, val = train_test_split(path_list, test_size=val_size, random_state=0, shuffle=True)\n        if phase == \"train\":\n            path_list = train\n        else:\n            path_list = val\n    \n    return path_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_list = make_datapath_list(phase=\"train\")\nprint(f\"train data length : {len(train_list)}\")\nval_list = make_datapath_list(phase=\"val\")\nprint(f\"validation data length : {len(val_list)}\")\ntest_list = make_datapath_list(phase=\"test\")\nprint(f\"test data length : {len(test_list)}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"size = 224\nmean = (0.485, 0.456, 0.406)\nstd = (0.229, 0.224, 0.225)\n\ntrain_dataset = PlantDataset(df_train, train_list, transform=ImageTransform(size, mean, std), phase='train')\nval_dataset = PlantDataset(df_train, val_list, transform=ImageTransform(size, mean, std), phase='val')\ntest_dataset = PlantDataset(df_train, test_list, transform=ImageTransform(size, mean, std), phase='test')\n\nindex = 0\n\nprint(\"【train dataset】\")\nprint(f\"img num : {train_dataset.__len__()}\")\nprint(f\"img : {train_dataset.__getitem__(index)[0].size()}\")\nprint(f\"label : {train_dataset.__getitem__(index)[1]}\")\nprint(f\"image name : {train_dataset.__getitem__(index)[2]}\")\n\nprint(\"\\n【validation dataset】\")\nprint(f\"img num : {val_dataset.__len__()}\")\nprint(f\"img : {val_dataset.__getitem__(index)[0].size()}\")\nprint(f\"label : {val_dataset.__getitem__(index)[1]}\")\nprint(f\"image name : {val_dataset.__getitem__(index)[2]}\")\n\nprint(\"\\n【test dataset】\")\nprint(f\"img num : {test_dataset.__len__()}\")\nprint(f\"img : {test_dataset.__getitem__(index)[0].size()}\")\nprint(f\"label : {test_dataset.__getitem__(index)[1]}\")\nprint(f\"image name : {test_dataset.__getitem__(index)[2]}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 128\n\n# Create DataLoader\ntrain_dataloader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nval_dataloader = data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\ntest_dataloader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\n# to Dictionary\ndataloaders_dict = {\"train\": train_dataloader, \"val\": val_dataloader, \"test\": test_dataloader}\n\n# Operation check\n#batch_iterator = iter(dataloaders_dict[\"train\"])\n#inputs, labels = next(batch_iterator)\n#print(inputs.size())  # torch.Size([3, 3, 224, 224]) : [batch_size, Channel, H, W]\n#print(labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name, param in net_vgg16.named_parameters():\n    print(name)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):\n    \"\"\"\n    Function for training the model.\n\n    Parameters\n    ----------\n    net: object\n    dataloaders_dict: dictionary\n    criterion: object\n    optimizer: object\n    num_epochs: int\n    \"\"\"\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    print(f\"Devices to be used : {device}\")\n    net.to(device)\n    torch.backends.cudnn.benchmark = True\n    # loop for epoch\n    for epoch in range(num_epochs):\n        print(f\"Epoch {epoch+1} / {num_epochs}\")\n        print(\"-------------------------------\")\n        for phase in [\"train\", \"val\"]:\n            if phase == \"train\":\n                net.train()\n            else:\n                net.eval()\n            epoch_loss = 0.0\n            epochcorrects = 0\n            #if (epoch == 0) and (phase == \"train\"):\n                #continue\n            for inputs, labels,  in tqdm(dataloaders_dict[phase]):\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n                optimizer.zero_grad()\n                with torch.set_gradenabled(phase == \"train\"):\n                    outputs = net(inputs)\n                    loss = criterion(outputs, labels)\n                    , preds = torch.max(outputs, 1)\n                    if phase == \"train\":\n                        loss.backward()\n                        optimizer.step()\n                    epoch_loss += loss.item() * inputs.size(0)\n                    epoch_corrects += torch.sum(preds == labels.data)\n            epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)\n            epoch_acc = epoch_corrects.double() / len(dataloaders_dict[phase].dataset)\n            print(f\"{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name, param in models[0].named_parameters():\n    print(name)","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Store the parameters to be learned by finetuning in the variable params_to_update.\nparams_to_update_1 = []\nparams_to_update_2 = []\nparams_to_update_3 = []\n\n# Specify the parameter name of the layer to be trained.\nupdate_param_names_1 = [\n    \"enet._blocks.24._bn0.weight\",\n\"enet._blocks.24._bn0.bias\",\n\"enet._blocks.24._depthwise_conv.weight\",\n\"enet._blocks.24._bn1.weight\",\n\"enet._blocks.24._bn1.bias\",\n\"enet._blocks.24._se_reduce.weight\",\n\"enet._blocks.24._se_reduce.bias\",\n\"enet._blocks.24._se_expand.weight\",\n\"enet._blocks.24._se_expand.bias\",\n\"enet._blocks.24._project_conv.weight\",\n\"enet._blocks.24._bn2.weight\",\n\"enet._blocks.24._bn2.bias\"\n]\nupdate_param_names_2 = [\n   \"enet._blocks.25._expand_conv.weight\",\n    \"enet._blocks.25._bn0.weight\",\n    \"enet._blocks.25._bn0.bias\",\n    \"enet._blocks.25._depthwise_conv.weight\",\n    \"enet._blocks.25._bn1.weight\",\n    \"enet._blocks.25._bn1.bias\",\n    \"enet._blocks.25._se_reduce.weight\",\n    \"enet._blocks.25._se_reduce.bias\",\n    \"enet._blocks.25._se_expand.weight\",\n    \"enet._blocks.25._se_expand.bias\",\n    \"enet._blocks.25._project_conv.weight\",\n    'enet._blocks.25._bn2.weight',\n    \"enet._blocks.25._bn2.bias\"\n]\nupdate_param_names_3 = [\"myfc.1.weight\", \"myfc.1.bias\", \"myfc.3.weight\",  \"myfc.3.bias\"]\n\nfor name, param in models[0].named_parameters():\n    if name in update_param_names_1:\n        param.requires_grad = False\n        params_to_update_1.append(param)\n        if param.requires_grad:\n            print(f\"Store in params_to_update_1 : {name}\")\n        else:\n            print(f\"Parameters not to be learned :  {name}\")\n    elif name in update_param_names_2:\n        param.requires_grad = False\n        params_to_update_2.append(param)\n        if param.requires_grad:\n            print(f\"Store in params_to_update_2 : {name}\")\n        else:\n            print(f\"Parameters not to be learned :  {name}\")\n    elif name in update_param_names_3:\n        param.requires_grad = True\n        params_to_update_3.append(param)\n        if param.requires_grad:\n            print(f\"Store in params_to_update_3 : {name}\")\n        else:\n            print(f\"Parameters not to be learned :  {name}\")\n    else:\n        param.requires_grad = False\n        print(f\"Parameters not to be learned :  {name}\")","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = optim.SGD([\n    {\"params\": params_to_update_1, \"lr\": 1e-3},\n    {\"params\": params_to_update_2, \"lr\": 1e-3},\n    {\"params\": params_to_update_3, \"lr\": 1e-3}\n], momentum=0.9)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_EPOCHS = 5\ntrain_model(models[0], dataloaders_dict, criterion, optimizer, NUM_EPOCHS)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_labels(row, labels, ths):\n    try:\n        row = [i for i, x in enumerate(row) if x > ths[str(i)]]\n        row = [labels[str(i)] for i in row]\n        row = 'healthy' if ('healthy' in row or len(row) == 0) else ' '.join(row)\n    except:\n        print(row)\n    return row\n\nlogits = []\nwith torch.no_grad():\n    for i, model in enumerate(models):\n        for j, loader in enumerate(loaders):\n            logits_tta = []\n            for img_data in loader:\n                img_data = img_data.to(DEVICE)\n                preds = np.squeeze(model(img_data).sigmoid().cpu().numpy())\n                logits_tta.append(preds)\n            print('model {} | loader {} -> done'.format(i, j))\n            logits.append(logits_tta)\nlogits = np.mean(logits, axis=0)\nlogits = np.squeeze(np.vstack(logits))\ndf_sub['labels'] = [get_labels(x, LABELS, ths) for x in list(logits)]\n\nelapsed_time = time.time() - start_time\nprint(f'time elapsed: {elapsed_time // 60:.0f} min {elapsed_time % 60:.0f} sec')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('value counts:')\nprint(df_sub.labels.value_counts())\ndf_sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}