{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview\n\nIn this notebook, I used Pytorch Lightning to solve it as a multi-label problem.\nI used the following [notebook](https://www.kaggle.com/demetrypascal/better-train-csv-format-keras-starter) as a reference.\n\nThe accuracy of the multi-label solution is about the same as that of the simple solution, and I think the accuracy can be improved by post-processing.\n\n[Inference Notebook](https://www.kaggle.com/pegasos/plant2021-multi-label-model-inference)","metadata":{}},{"cell_type":"markdown","source":"## Version Notes\n\n- V4  Model: Resnet50,           IMAGE_SIZE: 512, BS: 32, LB: 0.616\n- V6  Model: SE-ResNeXt50_32x4d, IMAGE_SIZE: 512, BS: 16, LB: 0.555\n- V8  Model: Resnet50, IMAGE_SIZE: 512, BS: 32, LB: 0.584\n  - Add processing to remove duplicates [Reference Discussion](https://www.kaggle.com/c/plant-pathology-2021-fgvc8/discussion/227829)\n- V11 Model: Resnet50, IMAGE_SIZE: 512, BS: 32, LB: 0.585\n  - More epoch, change lr_scheduler\n- V14 Model: Resnet50, IMAGE_SIZE: 512, BS: 32, LB: 0.572\n  - used torchmetrics(F1, weighted)\n- V15 Model: Resnet50, IMAGE_SIZE: 512, BS: 32, LB: 0.560\n  - Focal Loss(alpha=1, gamma=2)\n- V16 Model: Resnet50, IMAGE_SIZE: 512, BS: 32, LB: 0.580\n  - iterative-stratification(cross validators with stratification for multilabel data)\n- V17 Model: Resnet50, IMAGE_SIZE: 512, BS: 32, LB: 0.758\n  - epoch 60\n- V18 Model: EfficientNetB5 NS, IMAGE_SIZE: 512, BS: 32, LB: ???\n  - change model","metadata":{}},{"cell_type":"code","source":"!pip install -q torchmetrics\n!pip install -q iterative-stratification\n!pip install -q pytorch-lightning==1.2.8","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:12.085081Z","iopub.execute_input":"2021-05-24T15:24:12.085364Z","iopub.status.idle":"2021-05-24T15:24:28.885730Z","shell.execute_reply.started":"2021-05-24T15:24:12.085336Z","shell.execute_reply":"2021-05-24T15:24:28.884905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"package_paths = [\n    '../input/pytorch-image-library/pytorch-image-models-master/pytorch-image-models-master',\n]\nimport sys;\n\nfor pth in package_paths:\n    sys.path.append(pth)\n\nimport timm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-24T15:24:28.889083Z","iopub.execute_input":"2021-05-24T15:24:28.889358Z","iopub.status.idle":"2021-05-24T15:24:28.894581Z","shell.execute_reply.started":"2021-05-24T15:24:28.889330Z","shell.execute_reply":"2021-05-24T15:24:28.893585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport cv2\nimport timm\nimport torch\nimport torch.nn as nn\nimport albumentations as A\nimport pytorch_lightning as pl\nimport matplotlib.pyplot as plt\nimport torchmetrics\n\nfrom torch.utils.data import Dataset, DataLoader\nfrom albumentations.core.composition import Compose, OneOf\nfrom albumentations.pytorch import ToTensorV2\n\nfrom pytorch_lightning import Trainer, seed_everything\nfrom pytorch_lightning import Callback\nfrom pytorch_lightning.loggers import CSVLogger\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:28.895740Z","iopub.execute_input":"2021-05-24T15:24:28.896106Z","iopub.status.idle":"2021-05-24T15:24:31.706394Z","shell.execute_reply.started":"2021-05-24T15:24:28.896069Z","shell.execute_reply":"2021-05-24T15:24:31.705611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"PyTorch Lightning version: {pl.__version__}\")","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:31.709544Z","iopub.execute_input":"2021-05-24T15:24:31.709819Z","iopub.status.idle":"2021-05-24T15:24:31.721243Z","shell.execute_reply.started":"2021-05-24T15:24:31.709791Z","shell.execute_reply":"2021-05-24T15:24:31.720279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"DEBUG = False\n\nclass CFG:\n    seed = 42\n    model_name = 'tf_efficientnet_b5_ns'\n    pretrained = True\n    img_size = 512\n    num_classes = 6\n    lr = 1e-4\n    max_lr = 1e-3\n    pct_start = 0.3\n    div_factor = 1.0e+3\n    final_div_factor = 1.0e+3\n    num_epochs = 5\n    batch_size = 16\n    accum = 1\n    precision = 16\n    n_fold = 5\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:31.724054Z","iopub.execute_input":"2021-05-24T15:24:31.724318Z","iopub.status.idle":"2021-05-24T15:24:31.771266Z","shell.execute_reply.started":"2021-05-24T15:24:31.724284Z","shell.execute_reply":"2021-05-24T15:24:31.770415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH = \"../input/plant-pathology-2021-fgvc8/\"\n\n# TRAIN_DIR = PATH + 'train_images/'\nTRAIN_DIR = \"../input/resized-plant2021/img_sz_640/\"\nTEST_DIR = PATH + 'test_images/'","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:31.773955Z","iopub.execute_input":"2021-05-24T15:24:31.774284Z","iopub.status.idle":"2021-05-24T15:24:31.782180Z","shell.execute_reply.started":"2021-05-24T15:24:31.774251Z","shell.execute_reply":"2021-05-24T15:24:31.781422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:31.783485Z","iopub.execute_input":"2021-05-24T15:24:31.783986Z","iopub.status.idle":"2021-05-24T15:24:31.797392Z","shell.execute_reply.started":"2021-05-24T15:24:31.783935Z","shell.execute_reply":"2021-05-24T15:24:31.796503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = pd.read_csv(PATH + \"train.csv\")\nif DEBUG == True:\n    df_all = df_all[:200]\n    CFG.num_epochs = 30\n\ndf_all.shape","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:31.800199Z","iopub.execute_input":"2021-05-24T15:24:31.800440Z","iopub.status.idle":"2021-05-24T15:24:31.839183Z","shell.execute_reply.started":"2021-05-24T15:24:31.800414Z","shell.execute_reply":"2021-05-24T15:24:31.838136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import defaultdict\n\n\ndct = defaultdict(list)\n\nfor i, label in enumerate(df_all.labels):\n    for category in label.split():\n        dct[category].append(i)\n \ndct = {key: np.array(val) for key, val in dct.items()}\ndct","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:31.842765Z","iopub.execute_input":"2021-05-24T15:24:31.843004Z","iopub.status.idle":"2021-05-24T15:24:31.872927Z","shell.execute_reply.started":"2021-05-24T15:24:31.842981Z","shell.execute_reply":"2021-05-24T15:24:31.872296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_df = pd.DataFrame(np.zeros((df_all.shape[0], len(dct.keys())), dtype=np.int8), columns=dct.keys())\n\nfor key, val in dct.items():\n    new_df.loc[val, key] = 1\n\nnew_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:31.874712Z","iopub.execute_input":"2021-05-24T15:24:31.875001Z","iopub.status.idle":"2021-05-24T15:24:31.897912Z","shell.execute_reply.started":"2021-05-24T15:24:31.874978Z","shell.execute_reply":"2021-05-24T15:24:31.897286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = pd.concat([df_all, new_df], axis=1)\ndf_all.to_csv('better_train.csv', index = False)\ndf_all.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:31.900700Z","iopub.execute_input":"2021-05-24T15:24:31.900979Z","iopub.status.idle":"2021-05-24T15:24:32.130553Z","shell.execute_reply.started":"2021-05-24T15:24:31.900955Z","shell.execute_reply":"2021-05-24T15:24:32.129774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Remove duplicates\n\nThe output of the following [notebook](https://www.kaggle.com/nickuzmenkov/pp2021-duplicates-revealing/output) was used as a reference.\nIf you find it useful, please vote not only for this notebook, but also for the notebook it refers to!","metadata":{}},{"cell_type":"code","source":"duplicates = pd.read_csv(\"../input/pp2021-duplicates-revealing/duplicates.csv\",  names=('image1', 'image2'))\nsorted_index = duplicates['image1'].sort_values().index\nduplicates = duplicates.iloc[sorted_index].reset_index(drop=True)\nduplicates.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:32.133438Z","iopub.execute_input":"2021-05-24T15:24:32.133691Z","iopub.status.idle":"2021-05-24T15:24:32.159984Z","shell.execute_reply.started":"2021-05-24T15:24:32.133665Z","shell.execute_reply":"2021-05-24T15:24:32.159168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Duplicate images removed and the label should be the sum of the two.","metadata":{}},{"cell_type":"code","source":"if DEBUG != True:\n    for idx, images in duplicates.iterrows():\n    #     print(images['image1'])\n        mask1 = df_all['image'] == images['image1']\n        mask2 = df_all['image'] == images['image2']\n        tmp = df_all[mask1].iloc[0, 2:].values | df_all[mask2].iloc[0, 2:].values\n        df_all.loc[mask1, df_all.columns[2:]] = tmp\n        df_all = df_all.drop(df_all[mask2].index)\n    assert (len(new_df) - len(duplicates)) == len(df_all)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:32.161222Z","iopub.execute_input":"2021-05-24T15:24:32.161544Z","iopub.status.idle":"2021-05-24T15:24:32.618688Z","shell.execute_reply.started":"2021-05-24T15:24:32.161510Z","shell.execute_reply":"2021-05-24T15:24:32.617908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MultilabelStratifiedKFold","metadata":{}},{"cell_type":"code","source":"from iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n\n\nmsss = MultilabelStratifiedKFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)\ndf_train = pd.DataFrame(columns=df_all.columns)\ndf_valid = pd.DataFrame(columns=df_all.columns)\n\nfor train_idx, valid_idx in msss.split(df_all['image'], df_all.loc[:, list(df_all.columns[2:].values)]):\n#     df_train = df_all.iloc[train_idx]\n#     df_valid = df_all.iloc[valid_idx]\n    df_train = df_train.append(df_all.iloc[train_idx], ignore_index=True)\n    df_valid = df_valid.append(df_all.iloc[valid_idx], ignore_index=True)\n\nprint(f\"train size: {len(df_train)}\")\nprint(f\"valid size: {len(df_valid)}\")","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:32.620124Z","iopub.execute_input":"2021-05-24T15:24:32.620502Z","iopub.status.idle":"2021-05-24T15:24:33.282254Z","shell.execute_reply.started":"2021-05-24T15:24:32.620466Z","shell.execute_reply":"2021-05-24T15:24:33.281287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Dataset","metadata":{}},{"cell_type":"code","source":"class PlantDataset(Dataset):\n    def __init__(self, df, transform=None, image_dir=TRAIN_DIR):\n        self.image_id = df['image'].values\n        self.labels = df.iloc[:, 2:].values\n        self.transform = transform\n        self.image_dir = image_dir\n\n    def __len__(self):\n        return len(self.labels)\n\n    def __getitem__(self, idx):\n        image_id = self.image_id[idx]\n        label = torch.tensor(self.labels[idx].astype('int8'), dtype=torch.float32)\n        \n        image_path = self.image_dir + image_id\n        image = cv2.imread(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        augmented = self.transform(image=image)\n        image = augmented['image']\n        return {'image':image, 'target': label}","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:43:17.830191Z","iopub.execute_input":"2021-05-24T15:43:17.830553Z","iopub.status.idle":"2021-05-24T15:43:17.839333Z","shell.execute_reply.started":"2021-05-24T15:43:17.830521Z","shell.execute_reply":"2021-05-24T15:43:17.838427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transform(phase: str):\n    if phase == 'train':\n        return Compose([\n            A.RandomResizedCrop(height=CFG.img_size, width=CFG.img_size),\n            A.Flip(p=0.5),\n            A.RandomRotate90(p=0.5),\n            A.ShiftScaleRotate(p=0.5),\n            A.HueSaturationValue(p=0.5),\n            A.OneOf([\n                A.RandomBrightnessContrast(p=0.5),\n                A.RandomGamma(p=0.5),\n            ], p=0.5),\n            A.OneOf([\n                A.Blur(p=0.1),\n                A.GaussianBlur(p=0.1),\n                A.MotionBlur(p=0.1),\n            ], p=0.1),\n            A.OneOf([\n                A.GaussNoise(p=0.1),\n                A.ISONoise(p=0.1),\n                A.GridDropout(ratio=0.5, p=0.2),\n                A.CoarseDropout(max_holes=16, min_holes=8, max_height=16, max_width=16, min_height=8, min_width=8, p=0.2)\n            ], p=0.2),\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n    else:\n        return Compose([\n            A.Resize(height=CFG.img_size, width=CFG.img_size),\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:33.294223Z","iopub.execute_input":"2021-05-24T15:24:33.294617Z","iopub.status.idle":"2021-05-24T15:24:33.306369Z","shell.execute_reply.started":"2021-05-24T15:24:33.294580Z","shell.execute_reply":"2021-05-24T15:24:33.305416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = PlantDataset(df_train, get_transform('train'))\nvalid_dataset = PlantDataset(df_valid, get_transform('valid'))\n\ntrain_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True, pin_memory=True, drop_last=True, num_workers=2)\nvalid_loader = DataLoader(valid_dataset, batch_size=CFG.batch_size, shuffle=False, pin_memory=True, num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:33.307693Z","iopub.execute_input":"2021-05-24T15:24:33.308120Z","iopub.status.idle":"2021-05-24T15:24:33.320004Z","shell.execute_reply.started":"2021-05-24T15:24:33.308083Z","shell.execute_reply":"2021-05-24T15:24:33.319348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG.steps_per_epoch = len(train_loader)\nCFG.steps_per_epoch","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:33.323192Z","iopub.execute_input":"2021-05-24T15:24:33.323503Z","iopub.status.idle":"2021-05-24T15:24:33.332432Z","shell.execute_reply.started":"2021-05-24T15:24:33.323466Z","shell.execute_reply":"2021-05-24T15:24:33.331668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Focal Loss","metadata":{}},{"cell_type":"code","source":"import torch.nn.functional as F\n\n\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, logits=False, reduce=True):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.logits = logits\n        self.reduce = reduce\n\n    def forward(self, inputs, targets):\n        if self.logits:\n            BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduce=False)\n        else:\n            BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)\n        pt = torch.exp(-BCE_loss)\n        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n\n        if self.reduce:\n            return torch.mean(F_loss)\n        else:\n            return F_loss","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:33.333918Z","iopub.execute_input":"2021-05-24T15:24:33.334299Z","iopub.status.idle":"2021-05-24T15:24:33.342811Z","shell.execute_reply.started":"2021-05-24T15:24:33.334263Z","shell.execute_reply":"2021-05-24T15:24:33.341975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Model","metadata":{}},{"cell_type":"code","source":"class CustomModel(nn.Module):\n    def __init__(self, model_name='tf_efficientnet_b0_ns', pretrained=True):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        in_features = self.model.get_classifier().in_features\n        fc = nn.Sequential(\n            nn.Linear(in_features, in_features),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.5),\n            nn.Linear(in_features, CFG.num_classes)\n        )\n        \n        if 'efficientnet' in model_name:\n            self.model.classifier = fc\n        elif 'res' in model_name:\n            self.model.fc = fc\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:33.344629Z","iopub.execute_input":"2021-05-24T15:24:33.345124Z","iopub.status.idle":"2021-05-24T15:24:33.353138Z","shell.execute_reply.started":"2021-05-24T15:24:33.345088Z","shell.execute_reply":"2021-05-24T15:24:33.352340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LitCassava(pl.LightningModule):\n    def __init__(self, model):\n        super(LitCassava, self).__init__()\n        self.model = model\n#         self.metric = pl.metrics.F1(num_classes=CFG.num_classes)\n        self.metric = torchmetrics.F1(CFG.num_classes, average='weighted')\n#         self.criterion = nn.BCELoss()\n        self.criterion = nn.BCEWithLogitsLoss()\n#         self.criterion = FocalLoss()\n        self.sigmoid = nn.Sigmoid()\n        self.lr = CFG.lr\n\n    def forward(self, x, *args, **kwargs):\n        return self.model(x)\n\n    def configure_optimizers(self):\n        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.lr)\n#         self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(self.optimizer, T_max=CFG.t_max, eta_min=CFG.min_lr)\n        self.scheduler = torch.optim.lr_scheduler.OneCycleLR(self.optimizer, \n                                                             epochs=CFG.num_epochs, steps_per_epoch=CFG.steps_per_epoch,\n                                                             max_lr=CFG.max_lr, pct_start=CFG.pct_start, \n                                                             div_factor=CFG.div_factor, final_div_factor=CFG.final_div_factor)\n        scheduler = {'scheduler': self.scheduler, 'interval': 'step',}\n\n        return [self.optimizer], [scheduler]\n\n    def training_step(self, batch, batch_idx):\n        image = batch['image']\n        target = batch['target']\n        output = self.model(image)\n#         output = self.sigmoid(output)\n        loss = self.criterion(output, target)\n        score = self.metric(self.sigmoid(output), target.clone().detach().to(torch.int32))\n        logs = {'train_loss': loss, 'train_f1': score, 'lr': self.optimizer.param_groups[0]['lr']}\n        self.log_dict(\n            logs,\n            on_step=False, on_epoch=True, prog_bar=True, logger=True\n        )\n        return loss\n    \n    def validation_step(self, batch, batch_idx):\n        image = batch['image']\n        target = batch['target']\n        output = self.model(image)\n#         output = self.sigmoid(output)\n        loss = self.criterion(output, target)\n        score = self.metric(self.sigmoid(output), target.clone().detach().to(torch.int32))\n        logs = {'valid_loss': loss, 'valid_f1': score}\n        self.log_dict(\n            logs,\n            on_step=False, on_epoch=True, prog_bar=True, logger=True\n        )\n        return loss","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:33.354939Z","iopub.execute_input":"2021-05-24T15:24:33.355352Z","iopub.status.idle":"2021-05-24T15:24:33.369434Z","shell.execute_reply.started":"2021-05-24T15:24:33.355319Z","shell.execute_reply":"2021-05-24T15:24:33.368668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CustomModel(model_name=CFG.model_name, pretrained=CFG.pretrained)\nlit_model = LitCassava(model.model)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:33.370688Z","iopub.execute_input":"2021-05-24T15:24:33.371235Z","iopub.status.idle":"2021-05-24T15:24:37.173649Z","shell.execute_reply.started":"2021-05-24T15:24:33.371197Z","shell.execute_reply":"2021-05-24T15:24:37.172896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logger = CSVLogger(save_dir='logs/', name=CFG.model_name)\nlogger.log_hyperparams(CFG.__dict__)\ncheckpoint_callback = ModelCheckpoint(monitor='valid_f1',\n                                      save_top_k=1,\n                                      save_last=True,\n                                      save_weights_only=True,\n                                      filename='{epoch:02d}-{valid_loss:.4f}-{valid_f1:.4f}',\n                                      verbose=False,\n                                      mode='max')\n\ntrainer = Trainer(\n    max_epochs=CFG.num_epochs,\n    gpus=[0],\n    accumulate_grad_batches=CFG.accum,\n    precision=CFG.precision,\n#     callbacks=[EarlyStopping(monitor='valid_loss', patience=3, mode='min')],\n    checkpoint_callback=checkpoint_callback,\n    logger=logger,\n    weights_summary='top',\n    amp_backend='native',\n)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:24:37.174984Z","iopub.execute_input":"2021-05-24T15:24:37.175311Z","iopub.status.idle":"2021-05-24T15:24:37.189159Z","shell.execute_reply.started":"2021-05-24T15:24:37.175277Z","shell.execute_reply":"2021-05-24T15:24:37.188408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"trainer.fit(lit_model, train_dataloader=train_loader, val_dataloaders=valid_loader)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T05:53:44.024962Z","iopub.execute_input":"2021-05-24T05:53:44.025286Z","iopub.status.idle":"2021-05-24T12:42:40.945671Z","shell.execute_reply.started":"2021-05-24T05:53:44.025256Z","shell.execute_reply":"2021-05-24T12:42:40.944462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Result","metadata":{}},{"cell_type":"code","source":"metrics = pd.read_csv(f'{trainer.logger.log_dir}/metrics.csv')\n\ntrain_acc = metrics['train_f1'].dropna().reset_index(drop=True)\nvalid_acc = metrics['valid_f1'].dropna().reset_index(drop=True)\n    \nfig = plt.figure(figsize=(7, 6))\nplt.grid(True)\nplt.plot(train_acc, color=\"r\", marker=\"o\", label='train/f1')\nplt.plot(valid_acc, color=\"b\", marker=\"x\", label='valid/f1')\nplt.ylabel('F1', fontsize=24)\nplt.xlabel('Epoch', fontsize=24)\nplt.legend(loc='lower right', fontsize=18)\nplt.savefig(f'{trainer.logger.log_dir}/f1.png')\n\ntrain_loss = metrics['train_loss'].dropna().reset_index(drop=True)\nvalid_loss = metrics['valid_loss'].dropna().reset_index(drop=True)\n\nfig = plt.figure(figsize=(7, 6))\nplt.grid(True)\nplt.plot(train_loss, color=\"r\", marker=\"o\", label='train/loss')\nplt.plot(valid_loss, color=\"b\", marker=\"x\", label='valid/loss')\nplt.ylabel('Loss', fontsize=24)\nplt.xlabel('Epoch', fontsize=24)\nplt.legend(loc='upper right', fontsize=18)\nplt.savefig(f'{trainer.logger.log_dir}/loss.png')\\\n\nlr = metrics['lr'].dropna().reset_index(drop=True)\n\nfig = plt.figure(figsize=(7, 6))\nplt.grid(True)\nplt.plot(lr, color=\"g\", marker=\"o\", label='learning rate')\nplt.ylabel('LR', fontsize=24)\nplt.xlabel('Epoch', fontsize=24)\nplt.legend(loc='upper right', fontsize=18)\nplt.savefig(f'{trainer.logger.log_dir}/lr.png')","metadata":{"execution":{"iopub.status.busy":"2021-05-24T13:06:36.881687Z","iopub.execute_input":"2021-05-24T13:06:36.882063Z","iopub.status.idle":"2021-05-24T13:06:37.674334Z","shell.execute_reply.started":"2021-05-24T13:06:36.882028Z","shell.execute_reply":"2021-05-24T13:06:37.673447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"checkpoint = \"../input/pp-2021-efficientnet-model/last.ckpt\"\nmodel = CustomModel(model_name=CFG.model_name, pretrained=False)\nmodel.load_state_dict(torch.load(checkpoint)['state_dict'])","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:35:26.958164Z","iopub.execute_input":"2021-05-24T15:35:26.958497Z","iopub.status.idle":"2021-05-24T15:35:35.155070Z","shell.execute_reply.started":"2021-05-24T15:35:26.958469Z","shell.execute_reply":"2021-05-24T15:35:35.154412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(PATH + \"sample_submission.csv\")\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:37:46.017053Z","iopub.execute_input":"2021-05-24T15:37:46.017373Z","iopub.status.idle":"2021-05-24T15:37:46.037228Z","shell.execute_reply.started":"2021-05-24T15:37:46.017344Z","shell.execute_reply":"2021-05-24T15:37:46.036390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = pd.DataFrame(np.zeros([len(sub), len(new_df.columns)]), columns=new_df.columns)\nsub = pd.concat([sub, tmp], axis=1)\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:39:29.713635Z","iopub.execute_input":"2021-05-24T15:39:29.713990Z","iopub.status.idle":"2021-05-24T15:39:29.728207Z","shell.execute_reply.started":"2021-05-24T15:39:29.713962Z","shell.execute_reply":"2021-05-24T15:39:29.727325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = PlantDataset(sub, get_transform('valid'), image_dir=TEST_DIR)\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:43:24.806620Z","iopub.execute_input":"2021-05-24T15:43:24.806981Z","iopub.status.idle":"2021-05-24T15:43:24.815324Z","shell.execute_reply.started":"2021-05-24T15:43:24.806951Z","shell.execute_reply":"2021-05-24T15:43:24.813715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.cuda()\nmodel.eval()\n\nsigmoid = nn.Sigmoid()\n\npredictions = []\nfor batch in test_loader:\n    image = batch['image'].cuda()\n    with torch.no_grad():\n        outputs = model(image)\n        preds = outputs.detach().cpu()\n        # The probability of 0.5 or more is considered positive.\n        predictions.append(sigmoid(preds).numpy() > 0.5)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:43:25.442708Z","iopub.execute_input":"2021-05-24T15:43:25.443107Z","iopub.status.idle":"2021-05-24T15:43:27.000577Z","shell.execute_reply.started":"2021-05-24T15:43:25.443076Z","shell.execute_reply":"2021-05-24T15:43:26.999642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = pd.DataFrame(np.concatenate(predictions).astype(np.int), columns=new_df.columns)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:43:45.456481Z","iopub.execute_input":"2021-05-24T15:43:45.456879Z","iopub.status.idle":"2021-05-24T15:43:45.464650Z","shell.execute_reply.started":"2021-05-24T15:43:45.456846Z","shell.execute_reply":"2021-05-24T15:43:45.463500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.iloc[:, 2:] = predictions\nsub","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:43:53.563430Z","iopub.execute_input":"2021-05-24T15:43:53.563736Z","iopub.status.idle":"2021-05-24T15:43:53.579084Z","shell.execute_reply.started":"2021-05-24T15:43:53.563709Z","shell.execute_reply":"2021-05-24T15:43:53.578253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = []\nfor i, row in sub.iloc[:, 2:].iterrows():\n    if ((row['healthy'] == 1) or row.sum() == 0):\n        tmp = 'healthy'\n    else:\n        tmp = ' '.join(new_df.columns[row==row.max()])\n    labels.append(tmp)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:44:53.926466Z","iopub.execute_input":"2021-05-24T15:44:53.926883Z","iopub.status.idle":"2021-05-24T15:44:53.957997Z","shell.execute_reply.started":"2021-05-24T15:44:53.926840Z","shell.execute_reply":"2021-05-24T15:44:53.957302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['labels'] = labels\nsub[['image', 'labels']].to_csv('submission.csv', index=False)\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:46:17.815671Z","iopub.execute_input":"2021-05-24T15:46:17.816026Z","iopub.status.idle":"2021-05-24T15:46:17.830943Z","shell.execute_reply.started":"2021-05-24T15:46:17.816000Z","shell.execute_reply":"2021-05-24T15:46:17.830110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}