{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm\n!pip install torchcontrib","metadata":{"execution":{"iopub.status.busy":"2021-09-05T19:40:58.696483Z","iopub.execute_input":"2021-09-05T19:40:58.696789Z","iopub.status.idle":"2021-09-05T19:41:14.494873Z","shell.execute_reply.started":"2021-09-05T19:40:58.696711Z","shell.execute_reply":"2021-09-05T19:41:14.493746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pytorch_lightning as pl\nimport torch\nimport pandas as pd\nimport timm\nimport torch.nn as nn\n\nfrom PIL import Image\nfrom sklearn.model_selection import KFold\nfrom torchvision import transforms as tsfm\nfrom torch.utils.data import Dataset, DataLoader\nfrom pytorch_lightning import Trainer, seed_everything\nfrom pytorch_lightning.loggers import CSVLogger\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\nfrom pytorch_lightning.metrics import Metric\nfrom torchcontrib.optim import SWA\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2021-09-05T19:41:14.499022Z","iopub.execute_input":"2021-09-05T19:41:14.499316Z","iopub.status.idle":"2021-09-05T19:41:18.381092Z","shell.execute_reply.started":"2021-09-05T19:41:14.499287Z","shell.execute_reply":"2021-09-05T19:41:18.380297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    root_dir_origin = \"../input/plant-pathology-2021-fgvc8/\"\n    root_dir_resized = \"../input/plant-pathology-resized-images-shanshan-yu/224_224_resized/224_224\"\n    train_csv_path = os.path.join(root_dir_origin, 'train.csv')\n    train_imgs_dir = os.path.join(root_dir_resized, 'train_images')\n    # data info\n    label_num2str = {0: 'powdery_mildew',\n                     1: 'scab',\n                     2: 'complex',\n                     3: 'frog_eye_leaf_spot',\n                     4: 'rust'}\n    \n    label_str2num = {'powdery_mildew': 0,\n                     'scab': 1,\n                     'complex': 2,\n                     'frog_eye_leaf_spot': 3,\n                     'rust': 4}\n    # model info\n    model_name = 'tf_efficientnet_b2_ns'\n    # pretrained_dir = '../input/efficientnet-pp-train-weights'\n    pretrained_dir = \"./\"\n    data_argument_dir = '../input/ppdataargument3000/data_argument_224_224/train_images'\n    extra_train_dir = '../input/ppdataargument3000/data_argument_224_224/extra_train.csv'\n    # training hyper-parameters\n    fl_alpha = 1.0  # alpha of focal_loss\n    fl_gamma = 2.0  # gamma of focal_loss\n    use_swa = True\n    seed = 77\n    num_classes = 5\n    num_epochs = 20\n    batch_size = 8\n    t_max = 20\n    lr = 0.0005\n    min_lr = 1e-7\n    n_fold = 5\n    num_workers = 0\n    accum_grad_batch = 1\n    early_stop_delta = 1e-7\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n","metadata":{"execution":{"iopub.status.busy":"2021-09-05T20:14:36.001154Z","iopub.execute_input":"2021-09-05T20:14:36.00153Z","iopub.status.idle":"2021-09-05T20:14:36.009637Z","shell.execute_reply.started":"2021-09-05T20:14:36.001497Z","shell.execute_reply":"2021-09-05T20:14:36.008609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2021-09-05T19:41:37.833222Z","iopub.execute_input":"2021-09-05T19:41:37.833573Z","iopub.status.idle":"2021-09-05T19:41:37.849076Z","shell.execute_reply.started":"2021-09-05T19:41:37.833542Z","shell.execute_reply":"2021-09-05T19:41:37.84842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nAdd numerical labels for dataframe\n\"\"\"\nTRAIN_DF = pd.read_csv(CFG.train_csv_path)\nEXTRA_DF = pd.read_csv(CFG.extra_train_dir)\n\n\"\"\"\nremove duplicated images\n\"\"\"\nprint(f\"length before remove: {len(TRAIN_DF)}\")\ndup_csv = pd.read_csv(\"../input/duplicate-images-csv/duplicates.csv\")\n\nfor row in dup_csv.iterrows():\n    for col_idx in range(len(row)):\n        TRAIN_DF = TRAIN_DF.drop(TRAIN_DF[TRAIN_DF['image'] == row[1][col_idx]].index)\nprint(f\"length after remove: {len(TRAIN_DF)}\")\n\"\"\"\nnumerical value convertion\n\"\"\"\nfor df in [TRAIN_DF, EXTRA_DF]:\n    all_numeric_labels = []\n    for row_idx, row in df.iterrows():\n        labels_list = row['labels'].split(\" \")\n        numeric_label_list = [CFG.label_str2num[each] for each in labels_list if each != 'healthy']\n        all_numeric_labels.append(numeric_label_list)\n    df['numerical labels'] = all_numeric_labels\n","metadata":{"execution":{"iopub.status.busy":"2021-09-05T20:02:27.758088Z","iopub.execute_input":"2021-09-05T20:02:27.758453Z","iopub.status.idle":"2021-09-05T20:02:30.251204Z","shell.execute_reply.started":"2021-09-05T20:02:27.758423Z","shell.execute_reply":"2021-09-05T20:02:30.25043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined = pd.concat([TRAIN_DF, EXTRA_DF])\ncombined","metadata":{"execution":{"iopub.status.busy":"2021-09-05T20:02:36.225083Z","iopub.execute_input":"2021-09-05T20:02:36.225427Z","iopub.status.idle":"2021-09-05T20:02:36.247958Z","shell.execute_reply.started":"2021-09-05T20:02:36.225397Z","shell.execute_reply":"2021-09-05T20:02:36.247174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine train & valid image transformation\n\"\"\"\nDATASET_IMAGE_MEAN = (0.408, 0.626, 0.487)\nDATASET_IMAGE_STD = (0.178, 0.152, 0.171)\n\ntrain_transform = tsfm.Compose([tsfm.RandomApply([tsfm.ColorJitter(0.2, 0.2, 0.2),tsfm.RandomPerspective(distortion_scale=0.2),], p=0.3),\n                                tsfm.RandomApply([tsfm.ColorJitter(0.2, 0.2, 0.2),tsfm.RandomAffine(degrees=10),], p=0.3),\n                                tsfm.RandomVerticalFlip(p=0.3),\n                                tsfm.RandomHorizontalFlip(p=0.3),\n                                tsfm.ToTensor(),\n                                tsfm.Normalize(DATASET_IMAGE_MEAN, DATASET_IMAGE_STD), ])\n\nvalid_transform = tsfm.Compose([tsfm.ToTensor(),\n                                tsfm.Normalize(DATASET_IMAGE_MEAN, DATASET_IMAGE_STD), ])","metadata":{"execution":{"iopub.status.busy":"2021-09-05T20:02:39.701449Z","iopub.execute_input":"2021-09-05T20:02:39.701773Z","iopub.status.idle":"2021-09-05T20:02:39.71014Z","shell.execute_reply.started":"2021-09-05T20:02:39.701741Z","shell.execute_reply":"2021-09-05T20:02:39.709357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_weights = [0] * 5\nfor labels in TRAIN_DF['labels']:\n    for disease in labels.split(' '):\n        if disease == \"healthy\":\n            continue\n        else:\n            index = CFG.label_str2num[disease]\n            labels_weights[index] += 1\nlabels_weights = [(i, num / sum(labels_weights)) for i, num in enumerate(labels_weights)]\nsorted_weights = sorted(labels_weights, key=lambda x:x[1])\ninversed_weights = sorted_weights[::-1]","metadata":{"execution":{"iopub.status.busy":"2021-09-05T20:03:32.263639Z","iopub.execute_input":"2021-09-05T20:03:32.264037Z","iopub.status.idle":"2021-09-05T20:03:32.300174Z","shell.execute_reply.started":"2021-09-05T20:03:32.264002Z","shell.execute_reply":"2021-09-05T20:03:32.299393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_weights = np.zeros(5)\nfor i, value in sorted_weights:\n    labels_weights[i] = inversed_weights[sorted_weights[i][0]][1]\nlabels_weights","metadata":{"execution":{"iopub.status.busy":"2021-09-05T20:03:48.976257Z","iopub.execute_input":"2021-09-05T20:03:48.976619Z","iopub.status.idle":"2021-09-05T20:03:48.987977Z","shell.execute_reply.started":"2021-09-05T20:03:48.976589Z","shell.execute_reply":"2021-09-05T20:03:48.986982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine dataset class\n\"\"\"\nclass PlantDataset(Dataset):\n    def __init__(self, cfg, img_names: list, labels: list, transform=None):\n        self.img_dir = cfg.train_imgs_dir\n        self.img_names = img_names\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.img_names)\n\n    def __getitem__(self, idx):\n        img_path = self.img_names[idx]\n        if len(img_path) <= 9:\n            root_path = CFG.data_argument_dir\n        else:\n            root_path = CFG.train_imgs_dir\n        img = Image.open(os.path.join(root_path, img_path)).convert('RGB')\n        img_ts = self.transform(img)\n        label_ts = self.labels[idx]\n        return img_ts, label_ts","metadata":{"execution":{"iopub.status.busy":"2021-09-05T20:15:53.928495Z","iopub.execute_input":"2021-09-05T20:15:53.928824Z","iopub.status.idle":"2021-09-05T20:15:53.938478Z","shell.execute_reply.started":"2021-09-05T20:15:53.928794Z","shell.execute_reply":"2021-09-05T20:15:53.937439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine Focal-Loss\n\"\"\"\n\nclass FocalLoss(nn.Module):\n    \"\"\"\n    The focal loss for fighting against class-imbalance\n    \"\"\"\n    def __init__(self, alpha=1, gamma=2):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.epsilon = 1e-12  # prevent training from Nan-loss error\n        self.cls_weights = torch.tensor([list(labels_weights)],dtype=torch.float, requires_grad=False, device=CFG.device)\n        self.lb_smooth = 0.1\n\n    def forward(self, logits, target):\n        \"\"\"\n        logits & target should be tensors with shape [batch_size, num_classes]\n        \"\"\"\n        probs = torch.sigmoid(logits)\n        one_subtract_probs = 1.0 - probs\n        # add epsilon\n        probs_new = probs + self.epsilon\n        one_subtract_probs_new = one_subtract_probs + self.epsilon\n        # calculate focal loss\n        target = torch.abs(target - self.lb_smooth)\n        log_pt = target * torch.log(probs_new) + (1.0 - target) * torch.log(one_subtract_probs_new)\n        pt = torch.exp(log_pt)\n        focal_loss = -1.0 * (self.alpha * (1 - pt) ** self.gamma) * log_pt\n        focal_loss = focal_loss * self.cls_weights\n        return torch.mean(focal_loss)\n        ","metadata":{"execution":{"iopub.status.busy":"2021-09-05T20:09:58.373886Z","iopub.execute_input":"2021-09-05T20:09:58.374227Z","iopub.status.idle":"2021-09-05T20:09:58.382509Z","shell.execute_reply.started":"2021-09-05T20:09:58.374192Z","shell.execute_reply":"2021-09-05T20:09:58.381398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine F1 score metric\n\"\"\"\nclass MyF1Score(Metric):\n    def __init__(self, cfg, threshold: float = 0.5, dist_sync_on_step=False):\n        super().__init__(dist_sync_on_step=dist_sync_on_step)\n        self.cfg = cfg\n        self.threshold = threshold\n        self.add_state(\"tp\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n        self.add_state(\"fp\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n        self.add_state(\"fn\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n\n    def update(self, preds: torch.Tensor, target: torch.Tensor):\n        assert preds.shape == target.shape\n        preds_str_batch = self.num_to_str(torch.sigmoid(preds))\n        target_str_batch = self.num_to_str(target)\n        tp, fp, fn = 0, 0, 0\n        for pred_str_list, target_str_list in zip(preds_str_batch, target_str_batch):\n            for pred_str in pred_str_list:\n                if pred_str in target_str_list:\n                    tp += 1\n                if pred_str not in target_str_list:\n                    fp += 1\n\n            for target_str in target_str_list:\n                if target_str not in pred_str_list:\n                    fn += 1\n        self.tp += tp\n        self.fp += fp\n        self.fn += fn\n\n    def compute(self):\n        f1 = 2.0 * self.tp / (2.0 * self.tp + self.fn + self.fp)\n        return f1\n    \n    def num_to_str(self, ts: torch.Tensor) -> list:\n        batch_bool_list = (ts > self.threshold).detach().cpu().numpy().tolist()\n        batch_str_list = []\n        for one_sample_bool in batch_bool_list:\n            lb_str_list = [self.cfg.label_num2str[lb_idx] for lb_idx, bool_val in enumerate(one_sample_bool) if bool_val]\n            if len(lb_str_list) == 0:\n                lb_str_list = ['healthy']\n            batch_str_list.append(lb_str_list)\n        return batch_str_list","metadata":{"execution":{"iopub.status.busy":"2021-09-05T20:10:00.840025Z","iopub.execute_input":"2021-09-05T20:10:00.840393Z","iopub.status.idle":"2021-09-05T20:10:00.85267Z","shell.execute_reply.started":"2021-09-05T20:10:00.840359Z","shell.execute_reply":"2021-09-05T20:10:00.851518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine neural network model\n\"\"\"\n\nclass MyNetwork(pl.LightningModule):\n    def __init__(self, cfg):\n        super(MyNetwork, self).__init__()\n        self.cfg = cfg\n        self.model = timm.create_model(cfg.model_name, pretrained=True, num_classes=cfg.num_classes)\n        self.criterion = FocalLoss(cfg.fl_alpha, cfg.fl_gamma)\n        self.metric = MyF1Score(cfg)\n       \n    def forward(self, x):\n        return self.model(x)\n    \n    def configure_optimizers(self):\n        if self.cfg.use_swa:\n            self.optimizer = SWA(torch.optim.Adam(self.model.parameters(), lr=self.cfg.lr))\n        else:\n            self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.cfg.lr)\n            \n        self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(self.optimizer,\n                                                                    T_max=self.cfg.t_max,\n                                                                    eta_min=self.cfg.min_lr,\n                                                                    verbose=True)\n        return {'optimizer': self.optimizer, 'lr_scheduler': self.scheduler}\n    \n    def training_step(self, batch, batch_idx):\n        img_ts, lb_ts = batch\n        pred_ts = self.model(img_ts)\n        loss = self.criterion(pred_ts, lb_ts)\n        score = self.metric(pred_ts, lb_ts)\n        logs = {'train_loss': loss, 'train_f1': score, 'lr': self.optimizer.param_groups[0]['lr']}\n        self.log_dict(logs, on_step=False, on_epoch=True, prog_bar=True, logger=True)\n        return loss\n    \n    def validation_step(self, batch, batch_idx):\n        img_ts, lb_ts = batch\n        pred_ts = self.model(img_ts)\n        loss = self.criterion(pred_ts, lb_ts)\n        score = self.metric(pred_ts, lb_ts)\n        logs = {'valid_loss': loss, 'valid_f1': score}\n        self.log_dict(logs, on_step=False, on_epoch=True, prog_bar=True, logger=True)\n        return loss","metadata":{"execution":{"iopub.status.busy":"2021-09-05T20:10:03.635589Z","iopub.execute_input":"2021-09-05T20:10:03.635935Z","iopub.status.idle":"2021-09-05T20:10:03.648082Z","shell.execute_reply.started":"2021-09-05T20:10:03.635899Z","shell.execute_reply":"2021-09-05T20:10:03.646843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nSplit train & validation into Cross-Validation Folds\n\"\"\"\n\nall_img_names: list = combined[\"image\"].values.tolist()\nall_img_labels: list = combined[\"numerical labels\"].values.tolist()\nprint(\"Befor reomve duplicates:\", len(all_img_names), \", \", len(all_img_labels))\n    \n\"\"\"\nRemove duplicated samples from the training image\n\"\"\"\ndplct_csv_path = \"../input/duplicate-images-csv/duplicates.csv\"\ndplct_pd = pd.read_csv(dplct_csv_path)\ndplct_img_names = dplct_pd.iloc[:, 0].values.tolist() + dplct_pd.iloc[:, 1].values.tolist()\ndplct_img_names = list(set(dplct_img_names))\nprint(\"Num of duplicated samples: \", len(dplct_img_names))\n\nimg_names_no_dplct = []\nimg_labels_no_dplct = []\nfor img_name, img_label in zip(all_img_names, all_img_labels):\n    if img_name not in dplct_img_names:\n        img_names_no_dplct.append(img_name)\n        img_labels_no_dplct.append(img_label)\n        \nall_img_names = img_names_no_dplct\nall_img_labels = img_labels_no_dplct\nprint(\"After reomve duplicates:\", len(all_img_names), \", \", len(all_img_labels))\n    \nall_img_labels_ts = []\nfor tmp_lb in all_img_labels:\n    tmp_label = torch.zeros([CFG.num_classes], dtype=torch.float)\n    for idx in tmp_lb:\n        tmp_label[idx] = 1.0\n    all_img_labels_ts.append(tmp_label)\n    \nk_fold = KFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2021-09-05T20:10:07.449461Z","iopub.execute_input":"2021-09-05T20:10:07.44979Z","iopub.status.idle":"2021-09-05T20:10:07.911462Z","shell.execute_reply.started":"2021-09-05T20:10:07.449759Z","shell.execute_reply":"2021-09-05T20:10:07.910552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nTraining\n\"\"\"\n\nfor fold_idx, (train_indices, valid_indices) in enumerate(k_fold.split(all_img_names)):\n    \"\"\"\n    Init trainer\n    \"\"\"\n    logger = CSVLogger(save_dir=f'fold{fold_idx}_logs/', name=CFG.model_name)\n    logger.log_hyperparams(CFG.__dict__)\n    checkpoint_callback = ModelCheckpoint(monitor='valid_f1',\n                                          save_top_k=1,\n                                          save_last=True,\n                                          save_weights_only=True,\n                                          filename='best_perform',\n                                          verbose=False,\n                                          mode='max')\n    early_stop_callback = EarlyStopping(monitor='valid_loss', \n                                        min_delta=CFG.early_stop_delta, \n                                        patience=3, \n                                        mode='min')\n    \n        \n    trainer = Trainer(max_epochs=CFG.num_epochs,\n                      gpus=1,\n                      accumulate_grad_batches=CFG.accum_grad_batch,\n                      callbacks=[early_stop_callback],\n                      checkpoint_callback=checkpoint_callback,\n                      logger=logger,\n                      weights_summary='top',)\n    \"\"\"\n    Init dataset & dataloader\n    \"\"\"\n    # get image names and labels\n    fold_train_img_names = [all_img_names[idx] for idx in train_indices]\n    fold_valid_img_names = [all_img_names[idx] for idx in valid_indices]\n    fold_train_img_labels = [all_img_labels_ts[idx] for idx in train_indices]\n    fold_valid_img_labels = [all_img_labels_ts[idx] for idx in valid_indices]\n    # dataset\n    train_dataset = PlantDataset(CFG, fold_train_img_names, fold_train_img_labels, train_transform)\n    valid_dataset = PlantDataset(CFG, fold_valid_img_names, fold_valid_img_labels, valid_transform)\n    # dataloader\n    train_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True, num_workers=CFG.num_workers, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.num_workers)\n    \n    \n    \"\"\"\n    Init model\n    \"\"\"\n    model = MyNetwork(CFG)\n    \n    \"\"\"\n    Fit(train) the model\n    \"\"\"\n    trainer.fit(model, train_dataloader=train_loader, val_dataloaders=valid_loader)","metadata":{"execution":{"iopub.status.busy":"2021-09-05T20:15:58.007032Z","iopub.execute_input":"2021-09-05T20:15:58.007508Z","iopub.status.idle":"2021-09-05T20:16:34.830871Z","shell.execute_reply.started":"2021-09-05T20:15:58.007455Z","shell.execute_reply":"2021-09-05T20:16:34.830204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nPlot training results\n\"\"\"\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\n\nfig = plt.figure(figsize=(32, 10), constrained_layout=True)\ngs = gridspec.GridSpec(2, CFG.n_fold, figure=fig)\n\n\nfor fold_idx in range(CFG.n_fold):\n    tmp_log_dir = f\"fold{fold_idx}_logs/{CFG.model_name}/version_0\"\n    metrics = pd.read_csv(os.path.join(tmp_log_dir, 'metrics.csv'))\n\n    train_acc = metrics['train_f1'].dropna().reset_index(drop=True)\n    valid_acc = metrics['valid_f1'].dropna().reset_index(drop=True)\n    \n    ax = fig.add_subplot(gs[0, fold_idx])\n    ax.plot(train_acc, color=\"r\", marker=\"o\", label='train/f1')\n    ax.plot(valid_acc, color=\"b\", marker=\"x\", label='valid/f1')\n    ax.set_xlabel('Epoch', fontsize=24)\n    ax.set_ylabel('F1', fontsize=24)\n    ax.set_title(f'fold {fold_idx}')\n    ax.legend(loc='lower right', fontsize=18)\n\n\n    train_loss = metrics['train_loss'].dropna().reset_index(drop=True)\n    valid_loss = metrics['valid_loss'].dropna().reset_index(drop=True)\n\n    ax = fig.add_subplot(gs[1, fold_idx])\n    ax.plot(train_loss, color=\"r\", marker=\"o\", label='train/loss')\n    ax.plot(valid_loss, color=\"b\", marker=\"x\", label='valid/loss')\n    ax.set_ylabel('Loss', fontsize=24)\n    ax.set_xlabel('Epoch', fontsize=24)\n    ax.legend(loc='upper right', fontsize=18)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}