{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport timm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torchcontrib","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pytorch_lightning as pl\nimport torch\nimport pandas as pd\nimport timm\nimport torch.nn as nn\n\nfrom PIL import Image\nfrom sklearn.model_selection import KFold\nfrom torchvision import transforms as tsfm\nfrom torch.utils.data import Dataset, DataLoader\nfrom pytorch_lightning import Trainer, seed_everything\nfrom pytorch_lightning.loggers import CSVLogger\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\nfrom pytorch_lightning.metrics import Metric\nfrom torchcontrib.optim import SWA","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    # dir\n    root_dir_origin = \"../input/plant-pathology-2021-fgvc8/\"\n    root_dir_resized = \"../input/resized-plantpathology2021fgvc8-train-data-new/resized_plant-pathology-2021-fgvc8_train_data\"\n    train_csv_path = os.path.join(root_dir_origin, 'train.csv')\n    train_imgs_dir = os.path.join(root_dir_resized, 'resized_train_images_360_512')\n    # data info\n    label_num2str = {0: 'powdery_mildew',\n                     1: 'scab',\n                     2: 'complex',\n                     3: 'frog_eye_leaf_spot',\n                     4: 'rust'}\n    \n    label_str2num = {'powdery_mildew': 0,\n                     'scab': 1,\n                     'complex': 2,\n                     'frog_eye_leaf_spot': 3,\n                     'rust': 4}\n    # model info\n    model_name = 'tf_efficientnet_b4_ns'\n    pretrained_dir = '../input/efficientb4-focalloss'\n    which_to_load = 'best_perform'  # last or best_perform\n    needed_fold = [0, 1, 2, 3, 4, 5]\n    #\n    seed = 77\n    num_classes = 5\n    img_size = [360, 512]\n    n_fold = 6\n    batch_size = 8\n    num_workers = 8\n    fl_alpha = 1.0  \n    fl_gamma = 2.0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine dataset class\n\"\"\"\nclass PlantDataset(Dataset):\n    def __init__(self, cfg, img_names: list, labels: list, transform=None):\n        self.img_dir = cfg.train_imgs_dir\n        self.img_names = img_names\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.img_names)\n\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_dir, self.img_names[idx])\n        img = Image.open(img_path).convert('RGB')\n        img_ts = self.transform(img)\n        label_ts = self.labels[idx]\n        return img_ts, label_ts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(CFG.train_csv_path)\n\ntrain_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = tsfm.Compose([tsfm.Resize(CFG.img_size),\n                                      tsfm.ToTensor(),\n                                      tsfm.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),])\nvalid_transform = tsfm.Compose([tsfm.Resize(CFG.img_size),\n                                      tsfm.ToTensor(),\n                                      tsfm.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nSplit train & validation into Cross-Validation Folds\n\"\"\"\n\nall_img_names: list = train_df[\"image\"].values.tolist()\nall_img_labels: list = train_df[\"labels\"].values.tolist()\nprint(\"Befor reomve duplicates:\", len(all_img_names), \", \", len(all_img_labels))\n\n\n    \n\"\"\"\nRemove duplicated samples from the training image\n\"\"\"\ndplct_csv_path = \"../input/duplicate-images-csv/duplicates.csv\"\ndplct_pd = pd.read_csv(dplct_csv_path)\ndplct_img_names = dplct_pd.iloc[:, 0].values.tolist() + dplct_pd.iloc[:, 1].values.tolist()\ndplct_img_names = list(set(dplct_img_names))\nprint(\"Num of duplicated samples: \", len(dplct_img_names))\n\nimg_names_no_dplct = []\nimg_labels_no_dplct = []\nfor img_name, img_label in zip(all_img_names, all_img_labels):\n    if img_name not in dplct_img_names:\n        img_names_no_dplct.append(img_name)\n        img_labels_no_dplct.append(img_label)\n        \nall_img_names = img_names_no_dplct\nall_img_labels = img_labels_no_dplct\nprint(\"After reomve duplicates:\", len(all_img_names), \", \", len(all_img_labels))\n    \nall_img_labels_ts = []\nfor tmp_lb in all_img_labels:\n    tmp_label = torch.zeros([CFG.num_classes], dtype=torch.float)\n    for str_lb in tmp_lb.split(sep=\" \"):\n        if str_lb != 'healthy':\n            tmp_label[CFG.label_str2num[str_lb]] = 1.0\n    all_img_labels_ts.append(tmp_label)\n    \nk_fold = KFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine F1 score metric\n\"\"\"\nclass MyF1Score(Metric):\n    def __init__(self, cfg, threshold: float = 0.5, dist_sync_on_step=False):\n        super().__init__(dist_sync_on_step=dist_sync_on_step)\n        self.cfg = cfg\n        self.threshold = threshold\n        self.add_state(\"tp\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n        self.add_state(\"fp\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n        self.add_state(\"fn\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n\n    def update(self, preds: torch.Tensor, target: torch.Tensor):\n        assert preds.shape == target.shape\n        preds_str_batch = self.num_to_str(torch.sigmoid(preds))\n        target_str_batch = self.num_to_str(target)\n        tp, fp, fn = 0, 0, 0\n        for pred_str_list, target_str_list in zip(preds_str_batch, target_str_batch):\n            for pred_str in pred_str_list:\n                if pred_str in target_str_list:\n                    tp += 1\n                if pred_str not in target_str_list:\n                    fp += 1\n\n            for target_str in target_str_list:\n                if target_str not in pred_str_list:\n                    fn += 1\n        self.tp += tp\n        self.fp += fp\n        self.fn += fn\n\n    def compute(self):\n        f1 = 2.0 * self.tp / (2.0 * self.tp + self.fn + self.fp)\n        return f1\n    \n    def num_to_str(self, ts: torch.Tensor) -> list:\n        batch_bool_list = (ts > self.threshold).detach().cpu().numpy().tolist()\n        batch_str_list = []\n        for one_sample_bool in batch_bool_list:\n            lb_str_list = [self.cfg.label_num2str[lb_idx] for lb_idx, bool_val in enumerate(one_sample_bool) if bool_val]\n            if len(lb_str_list) == 0:\n                lb_str_list = ['healthy']\n            batch_str_list.append(lb_str_list)\n        return batch_str_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine Focal-Loss\n\"\"\"\n\nclass FocalLoss(nn.Module):\n    \"\"\"\n    The focal loss for fighting against class-imbalance\n    \"\"\"\n    def __init__(self, alpha=1, gamma=2):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.epsilon = 1e-12  # prevent training from Nan-loss error \n    \n    def forward(self, logits, target):\n        \"\"\"\n        logits & target should be tensors with shape [batch_size, num_classes]\n        \"\"\"\n        probs = F.sigmoid(logits)\n        one_subtract_probs = 1.0 - probs\n        # add epsilon\n        probs_new = probs + self.epsilon\n        one_subtract_probs_new = one_subtract_probs + self.epsilon\n        # calculate focal loss\n        log_pt =  target * torch.log(probs_new) + (1.0 - target) * torch.log(one_subtract_probs_new)\n        pt = torch.exp(log_pt)\n        focal_loss = -1.0 * (self.alpha * (1 - pt) ** self.gamma) * log_pt\n        return torch.mean(focal_loss)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine neural network model\n\"\"\"\n\nclass MyNetwork(pl.LightningModule):\n    def __init__(self, cfg):\n        super(MyNetwork, self).__init__()\n        self.cfg = cfg\n        self.model = timm.create_model(cfg.model_name, pretrained=True, num_classes=cfg.num_classes)\n        self.criterion = FocalLoss(cfg.fl_alpha, cfg.fl_gamma)\n        self.metric = MyF1Score(cfg)\n       \n    def forward(self, x):\n        return self.model(x)\n    \n    def configure_optimizers(self):\n        if self.cfg.use_swa:\n            self.optimizer = SWA(torch.optim.Adam(self.model.parameters(), lr=self.cfg.lr))\n        else:\n            self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.cfg.lr)\n            \n        self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(self.optimizer,\n                                                                    T_max=self.cfg.t_max,\n                                                                    eta_min=self.cfg.min_lr,\n                                                                    verbose=True)\n        return {'optimizer': self.optimizer, 'lr_scheduler': self.scheduler}\n    \n    def training_step(self, batch, batch_idx):\n        img_ts, lb_ts = batch\n        pred_ts = self.model(img_ts)\n        loss = self.criterion(pred_ts, lb_ts)\n        score = self.metric(pred_ts, lb_ts)\n        logs = {'train_loss': loss, 'train_f1': score, 'lr': self.optimizer.param_groups[0]['lr']}\n        self.log_dict(logs, on_step=False, on_epoch=True, prog_bar=True, logger=True)\n        return loss\n    \n    def validation_step(self, batch, batch_idx):\n        img_ts, lb_ts = batch\n        pred_ts = self.model(img_ts)\n        loss = self.criterion(pred_ts, lb_ts)\n        score = self.metric(pred_ts, lb_ts)\n        logs = {'valid_loss': loss, 'valid_f1': score}\n        self.log_dict(logs, on_step=False, on_epoch=True, prog_bar=True, logger=True)\n        return loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold_idx, (train_indices, valid_indices) in enumerate(k_fold.split(all_img_names)):    \n    \"\"\"\n    Init dataset & dataloader\n    \"\"\"\n    # get image names and labels\n    fold_train_img_names = [all_img_names[idx] for idx in train_indices]\n    fold_valid_img_names = [all_img_names[idx] for idx in valid_indices]\n    fold_train_img_labels = [all_img_labels_ts[idx] for idx in train_indices]\n    fold_valid_img_labels = [all_img_labels_ts[idx] for idx in valid_indices]\n    # dataset\n    train_dataset = PlantDataset(CFG, fold_train_img_names, fold_train_img_labels, train_transform)\n    valid_dataset = PlantDataset(CFG, fold_valid_img_names, fold_valid_img_labels, valid_transform)\n    # dataloader\n    train_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True, num_workers=CFG.num_workers, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.num_workers)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nInit models\n\"\"\"\nmodels_list = []\nfor fold_idx in CFG.needed_fold:\n    ckpt_path = os.path.join(CFG.pretrained_dir,\n                             f\"fold{fold_idx}_logs/{CFG.model_name}/version_0/checkpoints/{CFG.which_to_load}.ckpt\")\n    \n    model = MyNetwork.load_from_checkpoint(ckpt_path, cfg=CFG)\n    model.cuda()\n    model.eval()\n    models_list.append(model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt \n%matplotlib inline\n\nthreshold = np.array([0.4333, 0.4333, 0.4333, 0.4333, 0.4333])\n\ndef convert_num_to_str(pred: np.ndarray) -> str:\n    \"\"\"convert the numerical labels to string labels\"\"\"\n    lb_str_list = []\n    for lb_idx, bool_val in enumerate(pred):\n        if bool_val:\n            lb_str = CFG.label_num2str[lb_idx]\n            lb_str_list.append(lb_str)\n    if len(lb_str_list) == 0:\n        final_label = 'healthy'\n    else:\n        final_label = ' '.join(lb_str_list)\n    return final_label\n\npreds_list = []\nwith torch.no_grad():\n    for img_ts, lb_ts in valid_loader:\n        img_ts = img_ts.cuda()\n        n_fold_pred_list = []\n        for model in models_list:\n            pred_ts = torch.sigmoid(model(img_ts)).detach().cpu()\n            n_fold_pred_list.append(pred_ts)\n        pred_np_stack = torch.stack([item for item in n_fold_pred_list], dim=2)\n        pred_np = pred_np_stack.mean(dim=2)\n        \n        preds = (pred_np > torch.from_numpy(threshold))\n        \n        # convert numerical label into string\n#         final_labels = [convert_num_to_str(pred) for pred in preds]\n#         final_true_labels = [convert_num_to_str(lb_t) for lb_t in lb_ts]\n        for pred in preds:\n            preds_list.append(convert_num_to_str(pred))\n        \n        \n#         for i in range(CFG.batch_size):\n#             fig = plt.figure(figsize=(10,10 ))\n#             ax = fig.add_subplot(8, 1, i+1, xticks=[], yticks=[])\n#             ax.imshow(img_ts[i].permute(1,2,0).cpu())\n#             ax.set_title(f'Pred: {final_labels[i]}, Label: {final_true_labels[i]}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\n#cplx,cplx frog,...,cplx rust,frog,frog rust,hthy,pow,cplx pow,...,rust,scab,...,cplx frog scab, frog scab\ntrue_labels_list = [convert_num_to_str(label) for label in valid_dataset.labels]\nconfusion_matrix(true_labels_list, preds_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import itertools\n\n\nlabels_all = list(itertools.chain(*[lbs.split(\" \") for lbs in true_labels_list]))\n\nlabels_combine = {}\nfor comb in true_labels_list:\n    labels_combine[comb] = labels_combine.get(comb, 0) + 1\n\nshow_counts = '\\n'.join(sorted(f'\\t{k}: {v}' for k, v in labels_combine.items()))\nprint(f\"unique combinations: \\n\" + show_counts)\nprint(f\"total: {sum(labels_combine.values())}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install -q scikit-plot","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import scikitplot as skplt\n\nskplt.metrics.plot_confusion_matrix(\n    true_labels_list, \n    preds_list,\n    figsize=(8,8),\n    normalize=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\n\nnames = ['complex', 'complex frog_eye_leaf_spot', 'complex frog_eye_leaf_spot rust', 'complex rust', 'frog_eye_leaf_spot', 'frog_eye_leaf_spot rust', 'healthy', 'powdery_milder', 'complex powdery_milder', 'powdery_milder rust', 'rust', 'scab', 'scab complex', 'complex frog_eye_leaf_spot scab', 'frog_eye_leaf_spot scab'] \n\nprint(classification_report(true_labels_list, preds_list, target_names=names))","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}