{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nimport sys\nsys.path.append('../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master')\n\nfrom torchmetrics.metric import Metric\nimport torch\nimport torchvision\nfrom torchvision import transforms\nimport albumentations as A\nfrom albumentations.augmentations.geometric.transforms import Affine\nfrom albumentations.core.composition import Compose, OneOf\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split, KFold\nfrom torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nfrom torch import Tensor, nn\n! pip install torchsummary\n! pip install torchcontrib\nfrom efficientnet_pytorch import EfficientNet\nfrom torchsummary import summary\nfrom pytorch_lightning import Trainer, seed_everything\nfrom pytorch_lightning.loggers import CSVLogger\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\nfrom torchcontrib.optim import SWA\nimport pytorch_lightning as pl\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-28T14:59:23.138891Z","iopub.execute_input":"2021-11-28T14:59:23.139202Z","iopub.status.idle":"2021-11-28T14:59:48.272905Z","shell.execute_reply.started":"2021-11-28T14:59:23.139122Z","shell.execute_reply":"2021-11-28T14:59:48.272005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/plant-pathology-2021-fgvc8/train.csv')\ndf","metadata":{"execution":{"iopub.status.busy":"2021-11-28T14:59:48.274748Z","iopub.execute_input":"2021-11-28T14:59:48.275025Z","iopub.status.idle":"2021-11-28T14:59:48.323904Z","shell.execute_reply.started":"2021-11-28T14:59:48.274987Z","shell.execute_reply":"2021-11-28T14:59:48.323196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cài đặt tham số","metadata":{}},{"cell_type":"code","source":"class CFG:\n    # data path\n    train_csv_path = '../input/plant-pathology-2021-fgvc8/train.csv'\n#     train_imgs_dir = '../input/resized-plant2021/img_sz_256'\n    train_imgs_dir = '../input/resized-plant2021/img_sz_512'\n    # label info\n    label_num2str = {0: 'powdery_mildew',\n                     1: 'scab',\n                     2: 'complex',\n                     3: 'frog_eye_leaf_spot',\n                     4: 'rust'}\n    \n    label_str2num = {'powdery_mildew': 0,\n                     'scab': 1,\n                     'complex': 2,\n                     'frog_eye_leaf_spot': 3,\n                     'rust': 4}\n    # model info\n    model_name = 'pp-eff-n3'\n    pretrained_backbone = 'efficientnet-b7'\n    # training hyper-parameters\n    fl_alpha = 1.0  # alpha of focal_loss\n    fl_gamma = 2.0  # gamma of focal_loss\n    cls_weight = [3.6480, 1.0001, 2.1840, 1.5001, 2.2901] # class weights for we calculating loss\n    use_swa = True\n    seed = 77\n    num_classes = 5\n    num_epochs = 5\n    batch_size = 16\n    t_max = 18\n    lr = 1e-3\n    min_lr = 1e-6\n    n_fold = 6\n    num_workers = 8\n    accum_grad_batch = 1\n    early_stop_delta = 1e-7\n    # if you have multi-GPU on your machine, set gpu_list to [0, 1, ...]\n    gpu_idx = 0\n    device = torch.device(f'cuda:{gpu_idx}' if torch.cuda.is_available() else 'cpu')\n    gpu_list = [gpu_idx]","metadata":{"execution":{"iopub.status.busy":"2021-11-28T14:59:48.328741Z","iopub.execute_input":"2021-11-28T14:59:48.329238Z","iopub.status.idle":"2021-11-28T14:59:48.377537Z","shell.execute_reply.started":"2021-11-28T14:59:48.3292Z","shell.execute_reply":"2021-11-28T14:59:48.376671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encode_label():\n    label_split = []\n    for label in df['labels']:\n        list_single_label = label.split(' ')\n        label_enc = np.array([0, 0, 0, 0, 0])\n        for single_label in list_single_label:\n            if single_label != 'healthy':\n                label_enc[CFG.label_str2num.get(single_label)] = 1\n        label_split.append(label_enc)\n    return np.array(label_split)\n\nencoded_label = Tensor(encode_label())","metadata":{"execution":{"iopub.status.busy":"2021-11-28T14:59:48.378891Z","iopub.execute_input":"2021-11-28T14:59:48.379144Z","iopub.status.idle":"2021-11-28T14:59:48.485931Z","shell.execute_reply.started":"2021-11-28T14:59:48.37911Z","shell.execute_reply":"2021-11-28T14:59:48.485213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentation cho ảnh","metadata":{}},{"cell_type":"code","source":"SIZE=224\nDATASET_IMAGE_MEAN = (0.485, 0.456, 0.406)\nDATASET_IMAGE_STD = (0.229, 0.224, 0.225)\n\n\ntrain_transform = Compose([\n#     A.RandomResizedCrop(height=SIZE, width=SIZE, p=1.0),\n    Affine(\n        scale=(0.8, 1.0), \n        translate_percent=(0.1, 0.3), \n        rotate=(-360, 360),\n        cval=0, cval_mask=255, fit_output=True, \n        always_apply=False, p=0.8\n    ),\n    A.Resize(height=SIZE, width=SIZE, p=1.0),\n    A.Normalize(mean=DATASET_IMAGE_MEAN, std=DATASET_IMAGE_STD, p=1.0),\n    ToTensorV2()\n])\n\nvalid_transform = Compose([\n    A.Resize(height=SIZE, width=SIZE, p=1.0),\n    A.Normalize(mean=DATASET_IMAGE_MEAN, std=DATASET_IMAGE_STD, p=1.0),\n    ToTensorV2()\n])","metadata":{"execution":{"iopub.status.busy":"2021-11-28T14:59:48.487366Z","iopub.execute_input":"2021-11-28T14:59:48.48777Z","iopub.status.idle":"2021-11-28T14:59:48.497058Z","shell.execute_reply.started":"2021-11-28T14:59:48.487735Z","shell.execute_reply":"2021-11-28T14:59:48.496359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Xây dựng Dataset và DataLoader","metadata":{}},{"cell_type":"code","source":"class PlantDataset(Dataset):\n    def __init__(self, root_dir, file_list, labels, transform):\n        self.file_list = file_list\n        self.labels = labels\n        self.transform = transform\n        self.root_dir = root_dir\n   \n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, index):\n        img_path = self.file_list[index]\n        img = np.array(Image.open(os.path.join(self.root_dir, img_path)))\n\n        img_transformed = self.transform(image=img)['image']\n\n        return img_transformed, self.labels[index]","metadata":{"execution":{"iopub.status.busy":"2021-11-28T14:59:48.498649Z","iopub.execute_input":"2021-11-28T14:59:48.498925Z","iopub.status.idle":"2021-11-28T14:59:48.508818Z","shell.execute_reply.started":"2021-11-28T14:59:48.498886Z","shell.execute_reply":"2021-11-28T14:59:48.508058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TRAIN_IMG_ROOT = '../input/resized-plant2021/img_sz_384'\n\n# train_dataset = PlantDataset(\n#     root_dir=CFG.train_imgs_dir, \n#     file_list=df['image'][idx_train].to_numpy(), labels=y_train, \n#     transform=train_transform\n# )\n\n# valid_dataset = PlantDataset(\n#     root_dir=CFG.train_imgs_dir, \n#     file_list=df['image'][idx_valid].to_numpy(), labels=y_valid, \n#     transform=valid_transform\n# )\n\n# train_dataloader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True)\n# valid_dataloader = DataLoader(valid_dataset, batch_size=CFG.batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-28T14:59:48.510098Z","iopub.execute_input":"2021-11-28T14:59:48.511436Z","iopub.status.idle":"2021-11-28T14:59:48.517863Z","shell.execute_reply.started":"2021-11-28T14:59:48.511406Z","shell.execute_reply":"2021-11-28T14:59:48.517075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, axs = plt.subplots(nrows=8, ncols=4, dpi=100, figsize=(18, 20))\n\n# for i, (inputs, targets) in enumerate(train_dataloader):\n#     [axs[i][j].imshow(inputs[i * 4 + j].permute(1, 2, 0)) for i in range(8) for j in range(4)]\n#     break","metadata":{"execution":{"iopub.status.busy":"2021-11-28T14:59:48.519651Z","iopub.execute_input":"2021-11-28T14:59:48.520008Z","iopub.status.idle":"2021-11-28T14:59:48.526587Z","shell.execute_reply.started":"2021-11-28T14:59:48.519971Z","shell.execute_reply":"2021-11-28T14:59:48.525776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Xây dựng Model","metadata":{}},{"cell_type":"code","source":"\"\"\"\nXây dựng class FocalLoss phục vụ cho training\n\"\"\"\n\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.epsilon = 1e-12  # prevent training from Nan-loss error\n        self.cls_weights = torch.tensor([CFG.cls_weight],dtype=torch.float, requires_grad=False, device=CFG.device)\n\n    def forward(self, logits, target):\n        \"\"\"\n        logits & target có dạng [batch_size, num_classes]\n        \"\"\"\n        probs = torch.sigmoid(logits)\n        one_subtract_probs = 1.0 - probs\n        # add epsilon\n        probs_new = probs + self.epsilon\n        one_subtract_probs_new = one_subtract_probs + self.epsilon\n        # calculate focal loss\n        log_pt = target * torch.log(probs_new) + (1.0 - target) * torch.log(one_subtract_probs_new)\n        pt = torch.exp(log_pt)\n        focal_loss = -1.0 * (self.alpha * (1 - pt) ** self.gamma) * log_pt\n        focal_loss = focal_loss * self.cls_weights\n        return torch.mean(focal_loss)","metadata":{"execution":{"iopub.status.busy":"2021-11-28T14:59:48.529954Z","iopub.execute_input":"2021-11-28T14:59:48.530355Z","iopub.status.idle":"2021-11-28T14:59:48.540253Z","shell.execute_reply.started":"2021-11-28T14:59:48.530318Z","shell.execute_reply":"2021-11-28T14:59:48.539435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nĐịnh nghĩa cách tính điểm F1-Score \n\"\"\"\nclass MyF1Score(Metric):\n    def __init__(self, cfg, threshold: float = 0.5, dist_sync_on_step=False):\n        super().__init__(dist_sync_on_step=dist_sync_on_step)\n        self.cfg = cfg\n        self.threshold = threshold\n        self.add_state(\"tp\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n        self.add_state(\"fp\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n        self.add_state(\"fn\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n\n    def update(self, preds: torch.Tensor, target: torch.Tensor):\n        assert preds.shape == target.shape\n        preds_str_batch = self.num_to_str(torch.sigmoid(preds))\n        target_str_batch = self.num_to_str(target)\n        tp, fp, fn = 0, 0, 0\n        for pred_str_list, target_str_list in zip(preds_str_batch, target_str_batch):\n            for pred_str in pred_str_list:\n                if pred_str in target_str_list:\n                    tp += 1\n                if pred_str not in target_str_list:\n                    fp += 1\n\n            for target_str in target_str_list:\n                if target_str not in pred_str_list:\n                    fn += 1\n        self.tp += tp\n        self.fp += fp\n        self.fn += fn\n\n    def compute(self):\n        f1 = 2.0 * self.tp / (2.0 * self.tp + self.fn + self.fp)\n        return f1\n    \n    def num_to_str(self, ts: torch.Tensor) -> list:\n        batch_bool_list = (ts > self.threshold).detach().cpu().numpy().tolist()\n        batch_str_list = []\n        for one_sample_bool in batch_bool_list:\n            lb_str_list = [self.cfg.label_num2str[lb_idx] for lb_idx, bool_val in enumerate(one_sample_bool) if bool_val]\n            if len(lb_str_list) == 0:\n                lb_str_list = ['healthy']\n            batch_str_list.append(lb_str_list)\n        return batch_str_list","metadata":{"execution":{"iopub.status.busy":"2021-11-28T14:59:48.541566Z","iopub.execute_input":"2021-11-28T14:59:48.541897Z","iopub.status.idle":"2021-11-28T14:59:48.557188Z","shell.execute_reply.started":"2021-11-28T14:59:48.541859Z","shell.execute_reply":"2021-11-28T14:59:48.556551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for name, param in model.named_parameters():\n#     print(name)\n#     param.requires_grad = False\n# model(torch.randn(10, 3, 240, 240)).shape\n# summary(model, (3, 240, 240))","metadata":{"execution":{"iopub.status.busy":"2021-11-28T14:59:48.558367Z","iopub.execute_input":"2021-11-28T14:59:48.558858Z","iopub.status.idle":"2021-11-28T14:59:48.569864Z","shell.execute_reply.started":"2021-11-28T14:59:48.558821Z","shell.execute_reply":"2021-11-28T14:59:48.569069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nĐịnh nghĩa mô hình\n\"\"\"\n\nclass MyNetwork(pl.LightningModule):\n    def __init__(self, cfg):\n        super(MyNetwork, self).__init__()\n        self.cfg = cfg\n        self.model = EfficientNet.from_pretrained(cfg.pretrained_backbone, num_classes=CFG.num_classes)\n        self.criterion = FocalLoss(cfg.fl_alpha, cfg.fl_gamma)\n        self.metric = MyF1Score(cfg)\n       \n    def forward(self, x):\n        return self.model(x)\n    \n    def configure_optimizers(self):\n        if self.cfg.use_swa:\n            self.optimizer = SWA(torch.optim.Adam(self.model.parameters(), lr=self.cfg.lr))\n        else:\n            self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.cfg.lr)\n            \n        self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(self.optimizer,\n                                                                    T_max=self.cfg.t_max,\n                                                                    eta_min=self.cfg.min_lr,\n                                                                    verbose=True)\n        return {'optimizer': self.optimizer, 'lr_scheduler': self.scheduler}\n    \n    def training_step(self, batch, batch_idx):\n        img_ts, lb_ts = batch\n        pred_ts = self.model(img_ts)\n        loss = self.criterion(pred_ts, lb_ts)\n        score = self.metric(pred_ts, lb_ts)\n        logs = {'train_loss': loss, 'train_f1': score, 'lr': self.optimizer.param_groups[0]['lr']}\n        self.log_dict(logs, on_step=False, on_epoch=True, prog_bar=True, logger=True)\n        return loss\n    \n    def validation_step(self, batch, batch_idx):\n        img_ts, lb_ts = batch\n        pred_ts = self.model(img_ts)\n        loss = self.criterion(pred_ts, lb_ts)\n        score = self.metric(pred_ts, lb_ts)\n        logs = {'valid_loss': loss, 'valid_f1': score}\n        self.log_dict(logs, on_step=False, on_epoch=True, prog_bar=True, logger=True)\n        return loss","metadata":{"execution":{"iopub.status.busy":"2021-11-28T14:59:48.572769Z","iopub.execute_input":"2021-11-28T14:59:48.573551Z","iopub.status.idle":"2021-11-28T14:59:48.586975Z","shell.execute_reply.started":"2021-11-28T14:59:48.573497Z","shell.execute_reply":"2021-11-28T14:59:48.586226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tối ưu tham số của mô hình","metadata":{}},{"cell_type":"code","source":"def start_train(fold_idx, idx_train, idx_valid):\n    \"\"\"\n    Khởi tạo Trainer và Logger\n    \"\"\"\n    logger = CSVLogger(save_dir=f'fold{fold_idx}_logs/', name=CFG.model_name)\n    #     logger.log_hyperparams(CFG.__dict__)\n    checkpoint_callback = ModelCheckpoint(monitor='valid_f1',\n                                          save_top_k=1,\n                                          save_last=True,\n                                          save_weights_only=True,\n                                          filename='best_perform',\n                                          verbose=False,\n                                          mode='max')\n\n    trainer = Trainer(max_epochs=CFG.num_epochs,\n                      gpus=1,\n                      accumulate_grad_batches=CFG.accum_grad_batch,\n                      callbacks=[checkpoint_callback],\n                      logger=logger,\n                      weights_summary='top')\n    \"\"\"\n    Khởi tạo Dataset và DataLoader\n    \"\"\"\n    train_dataset = PlantDataset(\n        root_dir=CFG.train_imgs_dir,\n        file_list=df['image'][idx_train].to_numpy(), labels=encoded_label[idx_train], \n        transform=train_transform\n    )\n\n    valid_dataset = PlantDataset(\n        root_dir=CFG.train_imgs_dir, \n        file_list=df['image'][idx_valid].to_numpy(), labels=encoded_label[idx_valid], \n        transform=valid_transform\n    )\n\n    train_dataloader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True)\n    valid_dataloader = DataLoader(valid_dataset, batch_size=CFG.batch_size, shuffle=False)\n\n    \"\"\"\n    Khởi tạo Model\n    \"\"\"\n    model = MyNetwork(CFG)\n\n    \"\"\"\n    Huấn luyện mô hình\n    \"\"\"\n    trainer.fit(model, train_dataloaders=train_dataloader, val_dataloaders=valid_dataloader)","metadata":{"execution":{"iopub.status.busy":"2021-11-28T14:59:48.589969Z","iopub.execute_input":"2021-11-28T14:59:48.590164Z","iopub.status.idle":"2021-11-28T14:59:48.60131Z","shell.execute_reply.started":"2021-11-28T14:59:48.590134Z","shell.execute_reply":"2021-11-28T14:59:48.600545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"k_fold = KFold(n_splits=CFG.n_fold, shuffle=True)\nidx = [i for i in range(len(df.index))]\n\nfor fold_idx, (idx_train, idx_valid) in enumerate(k_fold.split(idx)):\n    start_train(fold_idx, idx_train, idx_valid)","metadata":{"execution":{"iopub.status.busy":"2021-11-28T14:59:48.602507Z","iopub.execute_input":"2021-11-28T14:59:48.603447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CFG.n_fold = 1\n# idx = [i for i in range(len(df.index))]\n# idx_train, idx_valid, y_train, y_valid = train_test_split(idx, encoded_label)\n# start_train(0, idx_train, idx_valid)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nVẽ biểu đồ độ thay đổi của f1-score và hàm loss\n\"\"\"\n\n\nfig = plt.figure(figsize=(32, 10), constrained_layout=True)\ngs = gridspec.GridSpec(2, CFG.n_fold, figure=fig)\n\n\nfor fold_idx in range(CFG.n_fold):\n    tmp_log_dir = f\"fold{fold_idx}_logs/{CFG.model_name}/version_0\"\n    metrics = pd.read_csv(os.path.join(tmp_log_dir, 'metrics.csv'))\n\n    train_acc = metrics['train_f1'].dropna().reset_index(drop=True)\n    valid_acc = metrics['valid_f1'].dropna().reset_index(drop=True)\n    \n    ax = fig.add_subplot(gs[0, fold_idx])\n    ax.plot(train_acc, color=\"r\", marker=\"o\", label='train/f1')\n    ax.plot(valid_acc, color=\"b\", marker=\"x\", label='valid/f1')\n    ax.set_xlabel('Epoch', fontsize=24)\n    ax.set_ylabel('F1', fontsize=24)\n    ax.set_title(f'fold {fold_idx}')\n    ax.legend(loc='lower right', fontsize=18)\n\n\n    train_loss = metrics['train_loss'].dropna().reset_index(drop=True)\n    valid_loss = metrics['valid_loss'].dropna().reset_index(drop=True)\n\n    ax = fig.add_subplot(gs[1, fold_idx])\n    ax.plot(train_loss, color=\"r\", marker=\"o\", label='train/loss')\n    ax.plot(valid_loss, color=\"b\", marker=\"x\", label='valid/loss')\n    ax.set_ylabel('Loss', fontsize=24)\n    ax.set_xlabel('Epoch', fontsize=24)\n    ax.legend(loc='upper right', fontsize=18)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ! ls ./fold0_logs/pp-eff-n3/version_0/checkpoints","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}