{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Trial of Pseudo-labeling by Cassava Model [Train with pseudo-label]\n\n## Purpose: Imrove the generality of the model of by using the pseudo-labeled dataset\n\nIn this notebook, I tried to train with pseudo labeled data of [New Plants Disease Dataset](https://www.kaggle.com/vipoooool/new-plant-diseases-dataset) by an efficientnet.  \n\nBy using the pseudo labeled dataset, I can increase the dataset and maybe can make the model's generalization.  \nThe motivation is based on the [Noisy Student](https://arxiv.org/abs/1911.04252)\n\n\nThis note book referenced https://www.kaggle.com/yasufuminakama/cassava-resnext50-32x4d-starter-training."},{"metadata":{"trusted":true},"cell_type":"code","source":"# !curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n# !python pytorch-xla-env-setup.py","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install torch_optimizer","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport glob\nimport os\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For the training, I used pre-trained model and Radam optimizer, I installed these libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import torch_xla\n# import torch_xla.core.xla_model as xm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Definition of hyper parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 719,\n    'model_arch': 'tf_efficientnet_b4_ns',\n    'img_size': 512,\n    'epochs': 10,\n    'train_bs': 10,\n    'valid_bs': 32,\n    'T_0': 10,\n    'lr': 5e-3,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_workers': 4,\n    'accum_iter': 2, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda',\n    'target_size': 5,\n    \"gradient_accumulation_steps\": 1, \n    \"max_grad_norm\": 5,\n    \"print_freq\": 10,\n    \"label_smoothing\": 0.0,\n    \"t1\": 1.0,\n    \"t2\": 1.0,\n    \"loss\": \"bce\", # bi_tempered_loss, logloss\n    \"optimizer\": \"AdamW\", # Radam AdamW\n    \"scheduler\": \"OneCycleLR\",\n    \"model_average\": 3,\n    \"pre-train\": True,\n    \"use_2019\": True,\n}\nTRAIN_FOLDS = [0]\n\n# gpu run\ndevice = \"cuda\"\n# device = xm.xla_device()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load dataframes"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dirs = glob.glob(\"/kaggle/input/new-plant-diseases-dataset/New Plant Diseases Dataset(Augmented)/New Plant Diseases Dataset(Augmented)/*/*/*.JPG\")\nplants_df = pd.read_csv(\"../input/cassava-pseudo-labeling-of-plants-disease/plants_df.csv\")\n\n# remove images under confidence level of 0.8\nth = 0.7\nplants_df = plants_df[(plants_df[\"0\"] > th) | (plants_df[\"1\"] > th) | (plants_df[\"2\"] > th) | (plants_df[\"3\"] > th) | (plants_df[\"4\"] > th)].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plants_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntest = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nlabel_map = pd.read_json('../input/cassava-leaf-disease-classification/label_num_to_disease_map.json', \n                         orient='index')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH_2019 = '../input/cassava-leaf-disease-merged/'\nTRAIN_DIR_2019 = DATA_PATH_2019 + 'train/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_merged = pd.read_csv(\"../input/cassava-leaf-disease-merged/merged.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\nTRAIN_PATH = '../input/cassava-leaf-disease-classification/train_images'\nTEST_PATH = '../input/cassava-leaf-disease-classification/test_images'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\nimport matplotlib.pyplot as plt\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau, OneCycleLR\nfrom skimage import io, transform\n\nimport torch_optimizer as optim\nimport timm\nfrom torchvision.transforms import transforms\n\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose\n    )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport warnings \nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataloader"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df[\"image_id\"].values\n        self.labels = df[['0', '1', '2', '3', '4']].values\n        self.labels.astype(\"float32\")\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = file_name\n        image = io.imread(file_path)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented[\"image\"]\n        label = torch.tensor(self.labels[idx])\n        return image.float(), label.float()\n\nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df[\"image_id\"].values\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}'\n        image = io.imread(file_path)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented[\"image\"]\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize, RGBShift\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_transform():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            RGBShift(r_shift_limit=15, g_shift_limit=15, b_shift_limit=15, p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(),\n        ], p=1.)\n    \n\ndef val_transform():\n    return Compose([\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EfficientNet(nn.Module):\n    def __init__(self, model_name=\"tf_efficientnet_b0_ns\"):\n        super(EfficientNet, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=True)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, CFG[\"target_size\"])\n    \n    def forward(self, x):\n        return self.model(x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Bi-tempered-logistic-loss\nIt makes model robust for the outliers and noisy datasets.  \nReference: https://github.com/mlpanda/bi-tempered-loss-pytorch"},{"metadata":{"trusted":true},"cell_type":"code","source":"## Bi-tempered-logistic-loss \n\ndef log_t(u, t):\n    \"\"\"Compute log_t for `u`.\"\"\"\n\n    if t == 1.0:\n        return torch.log(u)\n    else:\n        return (u ** (1.0 - t) - 1.0) / (1.0 - t)\n\n\ndef exp_t(u, t):\n    \"\"\"Compute exp_t for `u`.\"\"\"\n\n    if t == 1.0:\n        return torch.exp(u)\n    else:\n        return torch.relu(1.0 + (1.0 - t) * u) ** (1.0 / (1.0 - t))\n\n\ndef compute_normalization_fixed_point(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example (t > 1.0).\n    Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    t: Temperature 2 (> 1.0 for tail heaviness).\n    num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n\n    mu = torch.max(activations, dim=-1).values.view(-1, 1)\n    normalized_activations_step_0 = activations - mu\n\n    normalized_activations = normalized_activations_step_0\n    i = 0\n    while i < num_iters:\n        i += 1\n        logt_partition = torch.sum(exp_t(normalized_activations, t), dim=-1).view(-1, 1)\n        normalized_activations = normalized_activations_step_0 * (logt_partition ** (1.0 - t))\n\n    logt_partition = torch.sum(exp_t(normalized_activations, t), dim=-1).view(-1, 1)\n\n    return -log_t(1.0 / logt_partition, t) + mu\n\n\ndef compute_normalization(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example.\n    Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    t: Temperature 2 (< 1.0 for finite support, > 1.0 for tail heaviness).\n    num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n\n    if t < 1.0:\n        return None # not implemented as these values do not occur in the authors experiments...\n    else:\n        return compute_normalization_fixed_point(activations, t, num_iters)\n\n\ndef tempered_softmax(activations, t, num_iters=5):\n    \"\"\"Tempered softmax function.\n    Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    t: Temperature tensor > 0.0.\n    num_iters: Number of iterations to run the method.\n    Returns:\n    A probabilities tensor.\n    \"\"\"\n\n    if t == 1.0:\n        normalization_constants = torch.log(torch.sum(torch.exp(activations), dim=-1))\n    else:\n        normalization_constants = compute_normalization(activations, t, num_iters)\n\n    return exp_t(activations - normalization_constants, t)\n\n\ndef bi_tempered_logistic_loss(activations, labels, t1, t2, label_smoothing=0.0, num_iters=5):\n\n    \"\"\"Bi-Tempered Logistic Loss with custom gradient.\n    Args:\n    activations: A multi-dimensional tensor with last dimension `num_classes`.\n    labels: A tensor with shape and dtype as activations.\n    t1: Temperature 1 (< 1.0 for boundedness).\n    t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n    label_smoothing: Label smoothing parameter between [0, 1).\n    num_iters: Number of iterations to run the method.\n    Returns:\n    A loss tensor.\n    \"\"\"\n\n    if label_smoothing > 0.0:\n        num_classes = labels.shape[-1]\n        labels = (1 - num_classes / (num_classes - 1) * label_smoothing) * labels + label_smoothing / (num_classes - 1)\n\n    probabilities = tempered_softmax(activations, t2, num_iters)\n\n    temp1 = (log_t(labels + 1e-10, t1) - log_t(probabilities, t1)) * labels\n    temp2 = (1 / (2 - t1)) * (torch.pow(labels, 2 - t1) - torch.pow(probabilities, 2 - t1))\n    loss_values = temp1 - temp2\n\n    return torch.sum(loss_values, dim=-1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Definition of utilities"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))\n\n\ndef init_logger(log_file=OUTPUT_DIR+'train.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = init_logger()\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_model(path, model):\n    model.load_state_dict(torch.load(path)[\"model\"])\n    return model    \n\ndef average_snapshots(list_of_snapshots_paths, original_model):\n\n    snapshots_weights = {}\n\n    for snapshot_path in list_of_snapshots_paths:\n        model = load_model(snapshot_path, original_model)\n        snapshots_weights[snapshot_path] = dict(model.named_parameters())\n\n    params = model.named_parameters()\n    dict_params = dict(params)\n\n    N = len(snapshots_weights)\n\n    for name in dict_params.keys():\n        custom_params = None\n        for _, snapshot_params in snapshots_weights.items():\n            if custom_params is None:\n                custom_params = snapshot_params[name].data\n            else:\n                custom_params += snapshot_params[name].data\n        dict_params[name].data.copy_(custom_params/N)\n\n    model_dict = model.state_dict()\n    model_dict.update(dict_params)\n\n    model.load_state_dict(model_dict)\n    model.eval()\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Train:\n    def __init__(self, model, step_per_epoch):\n        self.model = model\n        if CFG[\"optimizer\"] == \"RAdam\":\n            self.optimizer = optim.RAdam(\n                model.parameters(),\n                lr= CFG[\"lr\"],\n                betas=(0.9, 0.999),\n                eps=1e-8,\n                weight_decay=0,\n            )\n        elif CFG[\"optimizer\"] == \"AdamW\":\n            self.optimizer = torch.optim.AdamW(\n                model.parameters(), \n                lr=CFG[\"lr\"], \n                betas=(0.9, 0.999), \n                eps=1e-08, \n                weight_decay=0.0, \n                amsgrad=False)\n        if CFG[\"scheduler\"] == \"OneCycleLR\":\n            self.scheduler = OneCycleLR(\n                self.optimizer, \n                CFG[\"lr\"],\n                epochs=CFG[\"epochs\"], \n                steps_per_epoch=step_per_epoch, \n                pct_start=0.3, \n                anneal_strategy='cos', \n                )\n        else:\n            self.scheduler = None\n        self.scaler = torch.cuda.amp.GradScaler()\n    \n    def train(self, train_loader, epoch):\n        batch_time = AverageMeter()\n        data_time = AverageMeter()\n        losses = AverageMeter()\n        scores = AverageMeter()\n        \n        self.model.train()\n        start = end = time.time()\n        global_step = 0\n        for step, (images, labels) in enumerate(train_loader):\n            # measure data loading time\n            images = images.to(device)\n            labels = labels.to(device)\n            batch_size = labels.size(0)\n            \n#             with torch.cuda.amp.autocast():\n            y_preds = self.model(images)\n            if CFG[\"loss\"] == \"bi_tempered_loss\":\n                loss = bi_tempered_logistic_loss(activations=y_preds, labels=labels, t1=CFG[\"t1\"], t2=CFG[\"t2\"], label_smoothing=CFG[\"label_smoothing\"])\n            elif CFG[\"loss\"] == \"logloss\":\n                loss = F.cross_entropy(y_preds, labels)\n            elif CFG[\"loss\"] == \"KLDivLoss\":\n                prob_m = F.log_softmax(y_preds, dim=1)\n                prob_t = F.softmax(labels)\n                loss = - (prob_m * prob_t).sum(dim=1).mean()\n            elif CFG[\"loss\"] == \"bce\":\n                loss = F.binary_cross_entropy(F.sigmoid(y_preds), labels)\n\n#             loss = loss.mean()\n            loss.backward()\n#             self.scaler.scale(loss).backward()\n            \n            if (step + 1) % CFG[\"accum_iter\"] == 0:\n#                 self.scaler.unscale_(self.optimizer)\n                grad_norm = torch.nn.utils.clip_grad_norm_(self.model.parameters(), CFG[\"max_grad_norm\"])\n#                 self.scaler.step(self.optimizer)\n#                 self.scaler.update()\n                self.optimizer.step()\n                self.optimizer.zero_grad()\n            \n            losses.update(loss.item(), batch_size)\n            if self.scheduler:\n                self.scheduler.step()\n            \n#             xm.mark_step()\n            \n            global_step += 1\n            \n            # measure elapsed time\n            batch_time.update(time.time() - end)\n            end = time.time()\n            \n            if (step + 1) % CFG[\"print_freq\"] == 0 or step == (len(train_loader)-1):\n                print('Epoch: [{0}][{1}/{2}] '\n                      'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                      'Elapsed {remain:s} '\n                      'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                      'Grad: {grad_norm:.4f}  '\n                      'LR: {lr:.6f}  '\n                      .format(\n                       epoch+1, step, len(train_loader), batch_time=batch_time,\n                       data_time=data_time, loss=losses,\n                       remain=timeSince(start, float(step+1)/len(train_loader)),\n                       grad_norm=grad_norm,\n                       lr=self.scheduler.get_lr()[0] if self.scheduler is not None else CFG[\"lr\"],\n                       ))\n                \n        return losses.avg\n    \n    def validate(self, valid_loader):\n        batch_time = AverageMeter()\n        data_time = AverageMeter()\n        losses = AverageMeter()\n        scores = AverageMeter()\n        # switch to evaluation mode\n        self.model.eval()\n        preds = []\n        start = end = time.time()\n        for step, (images, labels) in enumerate(valid_loader):\n            # measure data loading time\n            data_time.update(time.time() - end)\n#             if CFG[\"loss\"] == \"bi_tempered_loss\":\n#                 labels = torch.nn.functional.one_hot(torch.tensor(labels), num_classes=5)\n            images = images.to(device)\n            labels = labels.to(device)\n            batch_size = labels.size(0)\n            # compute loss\n            with torch.no_grad():\n                y_preds = self.model(images)\n                if CFG[\"loss\"] == \"bi_tempered_loss\":\n                    loss = bi_tempered_logistic_loss(activations=y_preds, labels=labels, t1=CFG[\"t1\"], t2=CFG[\"t2\"], label_smoothing=CFG[\"label_smoothing\"])\n                elif CFG[\"loss\"] == \"logloss\":\n                    loss = F.cross_entropy(y_preds, labels)\n                elif CFG[\"loss\"] == \"KLDivLoss\":\n                    prob_m = F.log_softmax(y_preds, dim=1)\n                    prob_t = F.softmax(labels)\n                    loss = - (prob_m * prob_t).sum(dim=1).mean()\n                elif CFG[\"loss\"] == \"bce\":\n                    loss = F.binary_cross_entropy(F.sigmoid(y_preds), labels)\n            losses.update(loss.mean().item(), batch_size)\n            # record accuracy\n            preds.append(y_preds.softmax(1).to('cpu').numpy())\n            # measure elapsed time\n            batch_time.update(time.time() - end)\n            end = time.time()\n            if (step + 1) % CFG[\"print_freq\"] == 0 or step == (len(valid_loader)-1):\n                print('EVAL: [{0}/{1}] '\n                      'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                      'Elapsed {remain:s} '\n                      'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                      .format(\n                       step, len(valid_loader), batch_time=batch_time,\n                       data_time=data_time, loss=losses,\n                       remain=timeSince(start, float(step+1)/len(valid_loader)),\n                       ))\n                \n        predictions = np.concatenate(preds)\n        return losses.avg, predictions\n    \n    def inference(self, states, test_loader):\n        self.model.to(device)\n        tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n        probs = []\n        for i, (images) in tk0:\n            images = images.to(device)\n            avg_preds = []\n            for state in states:\n                self.model.load_state_dict(state['model'])\n                self.model.eval()\n                with torch.no_grad():\n                    y_preds = self.model(images)\n                avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n            avg_preds = np.mean(avg_preds, axis=0)\n            probs.append(avg_preds)\n        probs = np.concatenate(probs)\n        return probs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5 folds and start trainig"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_tmp = pd.get_dummies(train, columns=[\"label\"])\ntrain_tmp = train_tmp.merge(train, on=\"image_id\")\ntrain_tmp = train_tmp.rename({f\"label_{i}\": f\"{i}\" for i in range(5)}, axis=1)\ntrain_tmp[\"image_id\"] = train[\"image_id\"].map(lambda x: TRAIN_DIR_2019+x)\nfolds = train_tmp.copy()\n\nextra = train_merged[train_merged[\"source\"] == 2019][[\"image_id\", \"label\"]].reset_index(drop=True).copy()\nextra_tmp = extra.copy()\nextra = pd.get_dummies(extra, columns=[\"label\"])\nextra = extra.merge(extra_tmp, on=\"image_id\")\nextra = extra.rename({f\"label_{i}\": f\"{i}\" for i in range(5)}, axis=1)\nextra[\"image_id\"] = extra[\"image_id\"].map(lambda x: TRAIN_DIR_2019+x)\n\nextra = pd.concat([extra, plants_df]).reset_index(drop=True)\nFold = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds[\"label\"])):\n    folds.loc[val_index, 'fold'] = int(n)\nfor n in range(len(extra)):\n    extra.loc[n, \"fold\"] = int(6)\nif CFG[\"use_2019\"]:\n    folds = pd.concat([folds, extra]).reset_index(drop=True)\nfolds['fold'] = folds['fold'].astype(int)\nprint(folds.groupby(['fold', \"label\"]).size())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# model & optimizer\n# ====================================================\nefmodel = EfficientNet(CFG[\"model_arch\"])\nefmodel.to(device)\nefmodel.drop_rate = 0.3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Train loop\n# ====================================================\ndef train_loop(folds, fold, efmodel):\n\n    LOGGER.info(f\"========== fold: {fold} training ==========\")\n\n    # ====================================================\n    # loader\n    # ====================================================\n    trn_idx = folds[folds['fold'] != fold].index\n    val_idx = folds[folds['fold'] == fold].index\n\n    train_folds = folds.loc[trn_idx].reset_index(drop=True)\n    valid_folds = folds.loc[val_idx].reset_index(drop=True)\n\n    train_dataset = TrainDataset(train_folds, \n                                 transform=get_transform())\n    valid_dataset = TrainDataset(valid_folds, \n                                 transform=val_transform())\n\n    train_loader = DataLoader(train_dataset, \n                              batch_size=CFG[\"train_bs\"], \n                              shuffle=True, \n                              num_workers=CFG[\"num_workers\"], pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, \n                              batch_size=CFG[\"train_bs\"], \n                              shuffle=False, \n                              num_workers=CFG[\"num_workers\"], pin_memory=True, drop_last=False)\n\n    \n    trainer = Train(efmodel, len(train_dataset) // CFG[\"train_bs\"])\n\n    best_score = 0.\n    best_loss = np.inf\n    \n    for epoch in range(CFG[\"epochs\"]):\n        \n        start_time = time.time()\n        \n        # train\n        avg_loss = trainer.train(train_loader, epoch)\n\n        # eval\n        avg_val_loss, preds = trainer.validate(valid_loader)\n        valid_labels = valid_folds[[\"label\"]].values\n\n        # scoring\n        score = accuracy_score(valid_labels, preds.argmax(1))\n\n        elapsed = time.time() - start_time\n\n        LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n        LOGGER.info(f'Epoch {epoch+1} - Accuracy: {score}')\n\n        if score > best_score:\n            best_score = score\n            LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n            torch.save({'model': efmodel.state_dict(), \n                        'preds': preds},\n                        OUTPUT_DIR+f'{CFG[\"model_arch\"]}_fold{fold}_best.pth')\n        torch.save({'model': efmodel.state_dict(), \n                    'preds': preds},\n                    OUTPUT_DIR+f'{CFG[\"model_arch\"]}_fold{fold}_snapshot{epoch}.pth')\n    \n#     trainer.model = average_snapshots([OUTPUT_DIR+f'{CFG[\"model_arch\"]}_fold{fold}_snapshot{e}.pth' for e in range(CFG[\"epochs\"] - CFG[\"model_average\"], CFG[\"epochs\"])], model)\n#     trainer.model.to(device)\n#     avg_val_loss, preds = trainer.validate(valid_loader)\n#     score = accuracy_score(valid_labels, preds.argmax(1))\n#     LOGGER.info(f\"averaged model score: {score:.4f}\")\n#     if score > best_score:\n#         LOGGER.info(\"averaged model is selected!\")\n#         torch.save({'model': model.state_dict(), \n#                     'preds': preds},\n#                      OUTPUT_DIR+f'{CFG[\"model_arch\"]}_fold{fold}_best.pth')\n    check_point = torch.load(OUTPUT_DIR+f'{CFG[\"model_arch\"]}_fold{fold}_best.pth')\n    valid_folds[[str(c) for c in range(5)]] = check_point['preds']\n    valid_folds['preds'] = check_point['preds'].argmax(1)\n\n    return valid_folds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nPrepare: 1.train  2.test  3.submission  4.folds\n\"\"\"\n\ndef get_result(result_df):\n    preds = result_df['preds'].values\n    labels = result_df[\"label\"].values\n    score = accuracy_score(labels, preds)\n    LOGGER.info(f'Score: {score:<.5f}')\n    \noof_df = pd.DataFrame()\nfor fold in TRAIN_FOLDS:\n    _oof_df = train_loop(folds, fold, efmodel)\n    oof_df = pd.concat([oof_df, _oof_df])\n    LOGGER.info(f\"========== fold: {fold} result ==========\")\nget_result(_oof_df)\n        \n# CV result\nLOGGER.info(f\"========== CV ==========\")\nget_result(oof_df)\n# save result\noof_df.to_csv(OUTPUT_DIR+'oof_df.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}