{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### A solution based on EfficientNet + Focal-Loss implemented with PytorchLightning\nThe training notebook is here: [Training Notebook](https://www.kaggle.com/crissallan/pytorchlightning-efficientnet-focalloss-training)\n\nWe have encountered a plateau of the performance now, so we share our solutions for everyone to discuss. \n\nThe pretrained model with our highest LB score isn't released, but by running this, you could also got a decent score on the LB.\n\nWe would like to know that is there any better training strategy or loss-designing for this solution!\n\nFeel free to comment below if you met any questions with this notebook.","metadata":{}},{"cell_type":"markdown","source":"### Performace Record\n| model | epoch | CV score | public LB score |\n|:----: | :----:|  :----:  | :----:          |\n| ViT   | 16    | >0.800   | 0.680           |\n| effb5 | 22    | >0.830   | 0.831           |\n| effb6 | 22    | >0.830   | 0.836           |","metadata":{}},{"cell_type":"code","source":"package_paths = ['../input/pytorch-library/pytorch_library/pytorch-image-models-master',]\nimport sys;\nfor pth in package_paths:\n    sys.path.append(pth)\n# load the external python package\nimport timm","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:01:42.376375Z","iopub.execute_input":"2021-05-26T18:01:42.376935Z","iopub.status.idle":"2021-05-26T18:01:43.055295Z","shell.execute_reply.started":"2021-05-26T18:01:42.376842Z","shell.execute_reply":"2021-05-26T18:01:43.054545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport torch\nimport pandas as pd\nimport pytorch_lightning as pl\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom PIL import Image\nfrom torchvision import transforms as tsfm\nfrom torch.utils.data import Dataset, DataLoader\nfrom pytorch_lightning.metrics import Metric","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-26T18:01:43.057459Z","iopub.execute_input":"2021-05-26T18:01:43.058037Z","iopub.status.idle":"2021-05-26T18:01:43.550212Z","shell.execute_reply.started":"2021-05-26T18:01:43.057999Z","shell.execute_reply":"2021-05-26T18:01:43.549384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    # dir\n    test_imgs_dir = \"../input/plant-pathology-2021-fgvc8/test_images\"\n    submit_csv_path = \"../input/plant-pathology-2021-fgvc8/sample_submission.csv\"\n    # data info\n    label_num2str = {0: 'powdery_mildew',\n                     1: 'scab',\n                     2: 'complex',\n                     3: 'frog_eye_leaf_spot',\n                     4: 'rust'}\n    \n    label_str2num = {'powdery_mildew': 0,\n                     'scab': 1,\n                     'complex': 2,\n                     'frog_eye_leaf_spot': 3,\n                     'rust': 4}\n    # model info\n    model_name = 'tf_efficientnet_b6_ns'\n    pretrained_dir = '../input/effb5-and-effb6-batch4-adam-swa'\n    which_to_load = 'best_perform'  # last or best_perform\n    needed_fold = [0, 1, 2, 3, 4, 5]\n    #\n    seed = 77\n    num_classes = 5\n    img_size = [528, 528]\n    tta_step = 0\n    threshold = [0.5, 0.5, 0.5, 0.5, 0.5]","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:01:43.551589Z","iopub.execute_input":"2021-05-26T18:01:43.551944Z","iopub.status.idle":"2021-05-26T18:01:43.563404Z","shell.execute_reply.started":"2021-05-26T18:01:43.551909Z","shell.execute_reply":"2021-05-26T18:01:43.562630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"\"\"\"\nDefine dataset class\n\"\"\"\n\nclass PlantDataset(Dataset):\n    def __init__(self, img_dir, img_names: list, labels: list, transform=None):\n        self.img_dir = img_dir\n        self.img_names = img_names\n        self.labels = labels\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.img_names)\n    \n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_dir, self.img_names[idx])\n        img = Image.open(img_path).convert('RGB')\n        img_ts = self.transform(img)\n        label_ts = self.labels[idx]\n        return img_ts, label_ts","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:01:43.565131Z","iopub.execute_input":"2021-05-26T18:01:43.565823Z","iopub.status.idle":"2021-05-26T18:01:43.574731Z","shell.execute_reply.started":"2021-05-26T18:01:43.565785Z","shell.execute_reply":"2021-05-26T18:01:43.573920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine test image transformation\n\"\"\"\n\ntest_transform_normal = tsfm.Compose([tsfm.Resize(CFG.img_size),\n                                      tsfm.ToTensor(),\n                                      tsfm.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),])\n\ntest_transform_tta =  tsfm.Compose([tsfm.Resize(CFG.img_size),\n                                    tsfm.RandomApply([tsfm.ColorJitter(0.2, 0.2, 0.2),tsfm.RandomPerspective(distortion_scale=0.2),], p=0.5),\n                                    tsfm.RandomApply([tsfm.ColorJitter(0.2, 0.2, 0.2),tsfm.RandomAffine(degrees=10),], p=0.5),\n                                    tsfm.RandomVerticalFlip(p=0.5),\n                                    tsfm.RandomHorizontalFlip(p=0.5),\n                                    tsfm.ToTensor(),\n                                    tsfm.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), ])","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:01:43.576463Z","iopub.execute_input":"2021-05-26T18:01:43.576899Z","iopub.status.idle":"2021-05-26T18:01:43.587233Z","shell.execute_reply.started":"2021-05-26T18:01:43.576847Z","shell.execute_reply":"2021-05-26T18:01:43.586456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nInit dataset instance and dataloader\n\"\"\"\ntest_img_names = os.listdir(CFG.test_imgs_dir)\nif CFG.tta_step > 0:\n    print(\"Using TTA, TTA step is: \", CFG.tta_step)\n    test_dataset = PlantDataset(CFG.test_imgs_dir, test_img_names, range(len(test_img_names)), test_transform_tta)\nelse:\n    print(\"Not using TTA\")\n    test_dataset = PlantDataset(CFG.test_imgs_dir, test_img_names, range(len(test_img_names)), test_transform_normal)\n\ntest_loader = DataLoader(test_dataset, batch_size=4, num_workers=4, shuffle=False, drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:01:43.591669Z","iopub.execute_input":"2021-05-26T18:01:43.592050Z","iopub.status.idle":"2021-05-26T18:01:43.600781Z","shell.execute_reply.started":"2021-05-26T18:01:43.592013Z","shell.execute_reply":"2021-05-26T18:01:43.599678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine Focal-Loss\n\"\"\"\n\nclass FocalLoss(nn.Module):\n    \"\"\"\n    The focal loss for fighting against class-imbalance\n    \"\"\"\n    def __init__(self, alpha=1, gamma=2):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.epsilon = 1e-12  # prevent training from Nan-loss error \n    \n    def forward(self, logits, target):\n        \"\"\"\n        logits & target should be tensors with shape [batch_size, num_classes]\n        \"\"\"\n        probs = F.sigmoid(logits)\n        one_subtract_probs = 1.0 - probs\n        # add epsilon\n        probs_new = probs + self.epsilon\n        one_subtract_probs_new = one_subtract_probs + self.epsilon\n        # calculate focal loss\n        log_pt =  target * torch.log(probs_new) + (1.0 - target) * torch.log(one_subtract_probs_new)\n        pt = torch.exp(log_pt)\n        focal_loss = -1.0 * (self.alpha * (1 - pt) ** self.gamma) * log_pt\n        return torch.mean(focal_loss)\n        ","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:01:43.602571Z","iopub.execute_input":"2021-05-26T18:01:43.602999Z","iopub.status.idle":"2021-05-26T18:01:43.610950Z","shell.execute_reply.started":"2021-05-26T18:01:43.602965Z","shell.execute_reply":"2021-05-26T18:01:43.610048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine F1 score metric\n\"\"\"\nclass MyF1Score(Metric):\n    def __init__(self, cfg, threshold: float=0.5, dist_sync_on_step=False):\n        super().__init__(dist_sync_on_step=dist_sync_on_step)\n        self.cfg = cfg\n        self.threshold = threshold\n        self.add_state(\"tp\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n        self.add_state(\"fp\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n        self.add_state(\"fn\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n\n    def update(self, preds: torch.Tensor, target: torch.Tensor):\n        assert preds.shape == target.shape\n        preds_str_batch = self.num_to_str(preds)\n        target_str_batch = self.num_to_str(target)\n        tp, fp, fn = 0, 0, 0\n        for pred_str_list, target_str_list in zip(preds_str_batch, target_str_batch):\n            for pred_str in pred_str_list:\n                if pred_str in target_str_list:\n                    tp += 1\n                if pred_str not in target_str_list:\n                    fp += 1\n            \n            for target_str in target_str_list:\n                if target_str not in pred_str_list:\n                    fn += 1\n        self.tp += tp\n        self.fp += fp\n        self.fn += fn\n\n    def compute(self):\n        f1 = 2.0 * self.tp / (2.0 * self.tp + self.fn + self.fp)\n        return f1\n    \n    def num_to_str(self, ts: torch.Tensor) -> list:\n        batch_bool_list = (ts > self.threshold).detach().cpu().numpy().tolist()\n        batch_str_list = []\n        for one_sample_bool in batch_bool_list:\n            lb_str_list = [self.cfg.label_num2str[lb_idx] for lb_idx, bool_val in enumerate(one_sample_bool) if bool_val]\n            if len(lb_str_list) == 0:\n                lb_str_list = ['healthy']\n            batch_str_list.append(lb_str_list)\n        return batch_str_list","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:01:43.612492Z","iopub.execute_input":"2021-05-26T18:01:43.613123Z","iopub.status.idle":"2021-05-26T18:01:43.626803Z","shell.execute_reply.started":"2021-05-26T18:01:43.613085Z","shell.execute_reply":"2021-05-26T18:01:43.625786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine neural network model\n\"\"\"\n\nclass MyNetwork(pl.LightningModule):\n    def __init__(self, cfg):\n        super(MyNetwork, self).__init__()\n        self.cfg = cfg\n        self.model = timm.create_model(cfg.model_name, pretrained=False, num_classes=cfg.num_classes)\n        self.criterion = FocalLoss()\n        self.metric = self.metric = MyF1Score(cfg)\n       \n    def forward(self, x):\n        return self.model(x)\n    \n    def configure_optimizers(self):\n        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.cfg.lr)\n        self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(self.optimizer,\n                                                                    T_max=self.cfg.t_max,\n                                                                    eta_min=self.cfg.min_lr,\n                                                                    verbose=True)\n        return {'optimizer': self.optimizer, 'lr_scheduler': self.scheduler}\n    \n    def training_step(self, batch, batch_idx):\n        img_ts, lb_ts = batch\n        pred_ts = self.model(img_ts)\n        loss = self.criterion(pred_ts, lb_ts)\n        score = self.metric(pred_ts, lb_ts)\n        logs = {'train_loss': loss, 'train_f1': score, 'lr': self.optimizer.param_groups[0]['lr']}\n        self.log_dict(logs, on_step=False, on_epoch=True, prog_bar=True, logger=True)\n        return loss\n    \n    def validation_step(self, batch, batch_idx):\n        img_ts, lb_ts = batch\n        pred_ts = self.model(img_ts)\n        loss = self.criterion(pred_ts, lb_ts)\n        score = self.metric(pred_ts, lb_ts)\n        logs = {'valid_loss': loss, 'valid_f1': score}\n        self.log_dict(logs, on_step=False, on_epoch=True, prog_bar=True, logger=True)\n        return loss","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:01:43.628229Z","iopub.execute_input":"2021-05-26T18:01:43.628772Z","iopub.status.idle":"2021-05-26T18:01:43.641734Z","shell.execute_reply.started":"2021-05-26T18:01:43.628736Z","shell.execute_reply":"2021-05-26T18:01:43.640870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nInit models\n\"\"\"\nmodels_list = []\nfor fold_idx in CFG.needed_fold:\n    ckpt_path = os.path.join(CFG.pretrained_dir,\n                             f\"fold{fold_idx}_logs/{CFG.model_name}/version_0/checkpoints/{CFG.which_to_load}.ckpt\")\n    \n    model = MyNetwork.load_from_checkpoint(ckpt_path, cfg=CFG)\n    model.cuda()\n    model.eval()\n    models_list.append(model)\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:01:43.643084Z","iopub.execute_input":"2021-05-26T18:01:43.643452Z","iopub.status.idle":"2021-05-26T18:02:03.324923Z","shell.execute_reply.started":"2021-05-26T18:01:43.643419Z","shell.execute_reply":"2021-05-26T18:02:03.323787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold = np.array([CFG.threshold])\nsubmit_df = pd.read_csv(CFG.submit_csv_path)\n\ndef convert_num_to_str(pred: np.ndarray) -> str:\n    \"\"\"convert the numerical labels to string labels\"\"\"\n    lb_str_list = []\n    for lb_idx, bool_val in enumerate(pred):\n        if bool_val:\n            lb_str = CFG.label_num2str[lb_idx]\n            lb_str_list.append(lb_str)\n    if len(lb_str_list) == 0:\n        final_label = 'healthy'\n    else:\n        final_label = ' '.join(lb_str_list)\n    return final_label\n\n\nwith torch.no_grad():\n    test_img_idx = 0\n    pred_list_all = []\n    logit_list_all = []\n    for img_ts, lb_ts in test_loader:\n        img_ts = img_ts.cuda()\n        n_fold_pred_list = []\n        for model in models_list:\n            pred_ts = torch.sigmoid(model(img_ts)).detach().cpu()\n            pred_ts = pred_ts.unsqueeze(2)\n            #print(pred_ts.size())\n            n_fold_pred_list.append(pred_ts)\n        pred_np = torch.cat(n_fold_pred_list, axis=2).mean(dim=2).numpy()\n        pred = (pred_np > threshold).tolist()\n        logit_list_all.append(pred_np)\n        pred_list_all.append(pred)\n    pred_np_all = np.concatenate(pred_list_all, axis=0)\n    logit_np_all = np.concatenate(logit_list_all, axis=0)\n\nprint(logit_np_all)\nprint(pred_np_all)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:02:03.326434Z","iopub.execute_input":"2021-05-26T18:02:03.326828Z","iopub.status.idle":"2021-05-26T18:02:05.057778Z","shell.execute_reply.started":"2021-05-26T18:02:03.326785Z","shell.execute_reply":"2021-05-26T18:02:05.056307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for test_img_idx, pred in enumerate(pred_np_all):\n    # convert numerical label into string\n    final_label = convert_num_to_str(pred)\n    img_name = test_img_names[test_img_idx]\n    row_idx = submit_df[submit_df.image == img_name].index.tolist()[0]\n    submit_df.iloc[row_idx, 1] = final_label\n    test_img_idx += 1","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:02:05.059371Z","iopub.execute_input":"2021-05-26T18:02:05.059748Z","iopub.status.idle":"2021-05-26T18:02:05.069247Z","shell.execute_reply.started":"2021-05-26T18:02:05.059709Z","shell.execute_reply":"2021-05-26T18:02:05.068044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save prediction into csv file\nsubmit_df.to_csv(\"./submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:02:05.071002Z","iopub.execute_input":"2021-05-26T18:02:05.071402Z","iopub.status.idle":"2021-05-26T18:02:05.155698Z","shell.execute_reply.started":"2021-05-26T18:02:05.071367Z","shell.execute_reply":"2021-05-26T18:02:05.154954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:02:05.156848Z","iopub.execute_input":"2021-05-26T18:02:05.157229Z","iopub.status.idle":"2021-05-26T18:02:05.172018Z","shell.execute_reply.started":"2021-05-26T18:02:05.157192Z","shell.execute_reply":"2021-05-26T18:02:05.171026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}