{"cells":[{"metadata":{},"cell_type":"markdown","source":"# About this notebook  \n- PyTorch resnext50_32x4d, tf_efficientnet_b3_ns application \n- I tried to apply ViT or efficientnet B5-B7 but I think if you want to use those models, it is better to apply them on TPU\n- StratifiedKFold 5 folds \n\n\n- I got some of the loss functions from the notebook: \n\nhttps://www.kaggle.com/piantic/cnn-or-transformer-pytorch-xla-tpu-for-cassava <- (Please upvote this notebook as well)\n\n- And the original notebook I used as a base for training:\n\nhttps://www.kaggle.com/yasufuminakama/cassava-resnext50-32x4d-starter-training <- (Please upvote this notebook as well)\n "},{"metadata":{},"cell_type":"markdown","source":"# Data Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\nimport pandas as pd\n\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('../input/cassava-leaf-disease-classification')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train = pd.read_csv('../input/cassava-leaf-disease-merged/merged.csv')\ntrain = pd.read_csv('../input/two-model-denoise/denoise_two_models.csv')\ntest = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nlabel_map = pd.read_json('../input/cassava-leaf-disease-classification/label_num_to_disease_map.json', \n                         orient='index')\ndisplay(train.head())\ndisplay(train.tail())\ndisplay(test.head())\ndisplay(label_map)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\ndef label_health(row):\n    #healthy\n    if row['label'] == 4:\n        return 0\n    #not healthy\n    else:\n        return 1\ntrain['healthy'] = train.apply(lambda row: label_health(row), axis=1)\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(train['label'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train = train.drop(columns=['0','1','2','3','4','preds','fold', 'correct'])\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Directory settings"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Directory settings\n# ====================================================\nimport os\n\nOUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\nTRAIN_PATH = '../input/cassava-leaf-disease-merged/train'\nTEST_PATH = '../input/cassava-leaf-disease-classification/test_images'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CFG"},{"metadata":{},"cell_type":"markdown","source":"Let's try with just 1 epoch"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# CFG\n# ====================================================\nclass CFG:\n    debug=False\n    apex=False\n    print_freq=100\n    num_workers=4\n    model_name='tf_efficientnet_b3_ns'#'resnext50_32x4d'\n    size=300\n    scheduler='CosineAnnealingWarmRestarts' # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts']\n    epochs=1\n    criterion='TaylorCrossEntropyLoss'\n    #factor=0.2 # ReduceLROnPlateau\n    #patience=4 # ReduceLROnPlateau\n    #eps=1e-6 # ReduceLROnPlateau\n    #T_max=10 # CosineAnnealingLR\n    T_0=10 # CosineAnnealingWarmRestarts\n    lr=1e-4\n    min_lr=1e-6\n    batch_size=32\n    weight_decay=1e-6\n    gradient_accumulation_steps=1\n    max_grad_norm=1000\n    seed=2021\n    target_size=5\n    target_col='label'\n    n_fold=5\n    trn_fold=[0, 1, 2, 3, 4]\n    train=True\n    inference=False\n    N=4\n    M=9\n    smoothing=0.05\n    rand_augment=True\n    \nif CFG.debug:\n    CFG.epochs = 15\n    train = train.sample(n=1000, random_state=CFG.seed).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Library"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nsys.path.append('../input/randaug')\n\nimport os\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose\n    )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport timm\nimport RandAugment\nimport warnings \nwarnings.filterwarnings('ignore')\nfrom torchvision.transforms import transforms\n\nif CFG.apex:\n    from apex import amp\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n\ndef init_logger(log_file=OUTPUT_DIR+'train.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = init_logger()\n\n\ndef seed_torch(seed=2021):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CV split"},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = train.copy()\nFold = StratifiedKFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds[CFG.target_col])):\n    folds.loc[val_index, 'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype(int)\nprint(folds.groupby(['fold', CFG.target_col]).size())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def rand_bbox(size, lam):\n    W = size[0]\n    H = size[1]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainCassavaDataset(Dataset):\n    def __init__(self, df, data_root, \n                 transforms=None, \n                 output_label=True, \n                 one_hot_label=False,\n                  \n                 fmix_params={\n                     'alpha': 1., \n                     'decay_power': 3., \n                     'shape': (CFG.size, CFG.size),\n                     'max_soft': True, \n                     'reformulate': False\n                 },\n                 do_cutmix=False,\n                 cutmix_params={\n                     'alpha': 1,\n                 }\n                ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        # self.do_fmix = do_fmix\n        self.fmix_params = fmix_params\n        self.do_cutmix = do_cutmix\n        self.cutmix_params = cutmix_params\n        \n        self.output_label = output_label\n        self.one_hot_label = one_hot_label\n        \n        if output_label == True:\n            self.labels = self.df['label'].values\n            #print(self.labels)\n            \n            if one_hot_label is True:\n                self.labels = np.eye(self.df['label'].max()+1)[self.labels]\n                #print(self.labels)\n            \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.labels[index]\n          \n        img  = get_img(\"{}/{}\".format(self.data_root, self.df.loc[index]['image_id']))\n\n        if self.transforms:\n            img = self.transforms(img)\n        '''\n        if self.do_fmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            with torch.no_grad():\n                #lam, mask = sample_mask(**self.fmix_params)\n                \n                lam = np.clip(np.random.beta(self.fmix_params['alpha'], self.fmix_params['alpha']),0.6,0.7)\n                \n                # Make mask, get mean / std\n                mask = make_low_freq_image(self.fmix_params['decay_power'], self.fmix_params['shape'])\n                mask = binarise_mask(mask, lam, self.fmix_params['shape'], self.fmix_params['max_soft'])\n    \n                fmix_ix = np.random.choice(self.df.index, size=1)[0]\n                fmix_img  = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[fmix_ix]['image_id']))\n\n                if self.transforms:\n                    fmix_img = self.transforms(fmix_img)\n\n                mask_torch = torch.from_numpy(mask)\n                \n                # mix image\n                img = mask_torch*img+(1.-mask_torch)*fmix_img\n\n                #print(mask.shape)\n\n                #assert self.output_label==True and self.one_hot_label==True\n\n                # mix target\n                rate = mask.sum()/CFG.size/CFG.size\n                target = rate*target + (1.-rate)*self.labels[fmix_ix]\n                #print(target, mask, img)\n                #assert False '''\n        \n        if self.do_cutmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            #print(img.sum(), img.shape)\n            with torch.no_grad():\n                cmix_ix = np.random.choice(self.df.index, size=1)[0]\n                cmix_img  = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[cmix_ix]['image_id']))\n                if self.transforms:\n                    cmix_img = self.transforms(cmix_img)\n                    \n                lam = np.clip(np.random.beta(self.cutmix_params['alpha'], self.cutmix_params['alpha']),0.3,0.4)\n                bbx1, bby1, bbx2, bby2 = rand_bbox((CFG.size, CFG.size), lam)\n\n                img[:, bbx1:bbx2, bby1:bby2] = cmix_img[:, bbx1:bbx2, bby1:bby2]\n\n                rate = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (CFG.size * CFG.size))\n                target = rate*target + (1.-rate)*self.labels[cmix_ix]\n                \n            #print('-', img.sum())\n            #print(target)\n            #assert False\n                            \n        # do label smoothing\n        #print(type(img), type(target))\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from RandAugment import RandAugment","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\nclass TrainDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.labels = df['label'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TRAIN_PATH}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            image = self.transform(image)\n        label = torch.tensor(self.labels[idx]).long()\n        return image, label\n    \n\nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = TrainCassavaDataset(train, data_root='../input/cassava-leaf-disease-merged/train', transforms=None)\n\nfor i in range(1):\n    image, label = train_dataset[i]\n    plt.imshow(image)\n    plt.title(f'label: {label}')\n    plt.show() \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations as A\nfrom albumentations.pytorch import ToTensorV2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Transforms\n# ====================================================\n'''\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n'''\n\ndef get_transforms(*, data):\n    \n    if data == 'train':\n        return transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.RandomResizedCrop((CFG.size, CFG.size)),\n            transforms.RandomVerticalFlip(p=0.5),\n            transforms.RandomHorizontalFlip(p=0.5),\n            RandAugment(CFG.N, CFG.M),\n            transforms.Resize((CFG.size, CFG.size)),\n            transforms.ToTensor(),\n            transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))\n        ])\n\n    elif data == 'valid':\n        return transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.Resize((CFG.size, CFG.size)),\n            transforms.ToTensor(),\n            transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))\n \n        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = TrainCassavaDataset(train, data_root='../input/cassava-leaf-disease-merged/train',transforms=get_transforms(data='train'), do_cutmix=True)\nfor i in range(3):\n    image, label = train_dataset[i]\n\n    plt.imshow(image[0])\n    plt.title(f'label: {label}')\n    plt.show() \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MODEL"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# MODEL\n# ====================================================\nclass CustomResNext(nn.Module):\n    def __init__(self, model_name=CFG.model_name, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \nclass CustomEfficientNet(nn.Module):\n    def __init__(self, model_name=CFG.model_name, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(CFG.model_name, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n''' \nclass CustomViT(nn.Module):\n    def __init__(self, model_name=CFG.model_name, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.head.in_features\n        self.model.head = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n''' ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if CFG.model_name=='tf_efficientnet_b3_ns':\n    model = CustomEfficientNet(model_name=CFG.model_name, pretrained=False)\nelse:\n    model = CustomResNext(model_name=CFG.model_name, pretrained=False)\n    \ntrain_dataset = TrainDataset(train, transform=get_transforms(data='train'))\ntrain_loader = DataLoader(train_dataset, batch_size=4, shuffle=True,\n                          num_workers=4, pin_memory=True, drop_last=True)\n\nfor image, label in train_loader:\n    output = model(image)\n    print(output)\n    break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss function"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Label Smoothing\n# ====================================================\nclass LabelSmoothingLoss(nn.Module): \n    def __init__(self, classes=5, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TaylorSoftmax(nn.Module):\n    '''\n    This is the autograd version\n    '''\n    def __init__(self, dim=1, n=2):\n        super(TaylorSoftmax, self).__init__()\n        assert n % 2 == 0\n        self.dim = dim\n        self.n = n\n\n    def forward(self, x):\n        '''\n        usage similar to nn.Softmax:\n            >>> mod = TaylorSoftmax(dim=1, n=4)\n            >>> inten = torch.randn(1, 32, 64, 64)\n            >>> out = mod(inten)\n        '''\n        fn = torch.ones_like(x)\n        denor = 1.\n        for i in range(1, self.n+1):\n            denor *= i\n            fn = fn + x.pow(i) / denor\n        out = fn / fn.sum(dim=self.dim, keepdims=True)\n        return out\n\n\nclass TaylorCrossEntropyLoss(nn.Module):\n    def __init__(self, n=2, ignore_index=-1, reduction='mean', smoothing=0.05):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n        self.lab_smooth = LabelSmoothingLoss(CFG.target_size, smoothing=smoothing)\n\n    def forward(self, logits, labels):\n        log_probs = self.taylor_softmax(logits).log()\n        loss = self.lab_smooth(log_probs, labels)\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalCosineLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, xent=.1):\n        super(FocalCosineLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n        self.xent = xent\n\n        self.y = torch.Tensor([1]).cuda()\n\n    def forward(self, input, target, reduction=\"mean\"):\n        cosine_loss = F.cosine_embedding_loss(input, F.one_hot(target, num_classes=input.size(-1)), self.y, reduction=reduction)\n\n        cent_loss = F.cross_entropy(F.normalize(input), target, reduce=False)\n        pt = torch.exp(-cent_loss)\n        focal_loss = self.alpha * (1-pt)**self.gamma * cent_loss\n\n        if reduction == \"mean\":\n            focal_loss = torch.mean(focal_loss)\n\n        return cosine_loss + self.xent * focal_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\ndef accuracy(output, target, topk=(1,)):\n    maxk = max(topk)\n    batch_size = target.size(0)\n    \n    _, pred = output.topk(maxk, 1, True, True)\n    pred = pred.t()\n    correct = pred.eq(target.view(1,-1).expand_as(pred))\n    \n    res = []\n    for k in topk:\n        correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)\n        res.append(correct_k.mul_(100.0/batch_size))\n        \n    return res\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))\n\n\ndef train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to train mode\n    model.train()\n    start = end = time.time()\n    global_step = 0\n    for step, (images, labels) in enumerate(train_loader):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        y_preds = model(images)\n        loss = criterion(y_preds, labels)\n        # record loss\n        losses.update(loss.item(), batch_size)\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        if CFG.apex:\n            with amp.scale_loss(loss, optimizer) as scaled_loss:\n                scaled_loss.backward()\n        else:\n            loss.backward()\n        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), CFG.max_grad_norm)\n        if (step + 1) % CFG.gradient_accumulation_steps == 0:\n            optimizer.step()\n            optimizer.zero_grad()\n            global_step += 1\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(train_loader)-1):\n            print('Epoch: [{0}][{1}/{2}] '\n                  'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  'Grad: {grad_norm:.4f}  '\n                  #'LR: {lr:.6f}  '\n                  .format(\n                   epoch+1, step, len(train_loader), batch_time=batch_time,\n                   data_time=data_time, loss=losses,\n                   remain=timeSince(start, float(step+1)/len(train_loader)),\n                   grad_norm=grad_norm,\n                   #lr=scheduler.get_lr()[0],\n                   ))\n    return losses.avg\n\n\ndef valid_fn(valid_loader, model, criterion, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to evaluation mode\n    model.eval()\n    preds = []\n    start = end = time.time()\n    for step, (images, labels) in enumerate(valid_loader):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        # compute loss\n        with torch.no_grad():\n            y_preds = model(images)\n        loss = criterion(y_preds, labels)\n        losses.update(loss.item(), batch_size)\n        # record accuracy\n        preds.append(y_preds.softmax(1).to('cpu').numpy())\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(valid_loader)-1):\n            print('EVAL: [{0}/{1}] '\n                  'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  .format(\n                   step, len(valid_loader), batch_time=batch_time,\n                   data_time=data_time, loss=losses,\n                   remain=timeSince(start, float(step+1)/len(valid_loader)),\n                   ))\n    predictions = np.concatenate(preds)\n    return losses.avg, predictions\n\n\ndef inference(model, states, test_loader, device):\n    model.to(device)\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state['model'])\n            model.eval()\n            with torch.no_grad():\n                y_preds = model(images)\n            avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train loop"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# ====================================================\n# Train loop\n# ====================================================\ndef train_loop(folds, fold):\n\n    LOGGER.info(f\"========== fold: {fold} training ==========\")\n\n    # ====================================================\n    # loader\n    # ====================================================\n    trn_idx = folds[folds['fold'] != fold].index\n    val_idx = folds[folds['fold'] == fold].index\n\n    train_folds = folds.loc[trn_idx].reset_index(drop=True)\n    valid_folds = folds.loc[val_idx].reset_index(drop=True)\n\n    train_dataset = TrainDataset(train_folds, \n                                 transform=get_transforms(data='train'))\n    valid_dataset = TrainDataset(valid_folds, \n                                 transform=get_transforms(data='valid'))\n\n    train_loader = DataLoader(train_dataset, \n                              batch_size=CFG.batch_size, \n                              shuffle=True, \n                              num_workers=CFG.num_workers, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, \n                              batch_size=CFG.batch_size, \n                              shuffle=False, \n                              num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n    \n    # ====================================================\n    # scheduler \n    # ====================================================\n    def get_scheduler(optimizer):\n        if CFG.scheduler=='ReduceLROnPlateau':\n            scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=CFG.factor, patience=CFG.patience, verbose=True, eps=CFG.eps)\n        elif CFG.scheduler=='CosineAnnealingLR':\n            scheduler = CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr, last_epoch=-1)\n        elif CFG.scheduler=='CosineAnnealingWarmRestarts':\n            scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr, last_epoch=-1)\n        return scheduler\n\n    # ====================================================\n    # model & optimizer\n    # ====================================================\n    if CFG.model_name == 'tf_efficientnet_b3_ns':\n        model = CustomEfficientNet(model_name=CFG.model_name, pretrained=True)   \n    else:\n        model = CustomResNext(model_name=CFG.model_name, pretrained=True)\n        \n    model.to(device)\n\n    optimizer = Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)\n    scheduler = get_scheduler(optimizer)\n\n    # ====================================================\n    # apex\n    # ====================================================\n    if CFG.apex:\n        model, optimizer = amp.initialize(model, optimizer, opt_level='O1', verbosity=0)\n\n    # ====================================================\n    # loop\n    # ====================================================\n    criterion = nn.CrossEntropyLoss()\n\n    best_score = 0.\n    best_loss = np.inf\n    \n    for epoch in range(CFG.epochs):\n        \n        start_time = time.time()\n        \n        # train\n        avg_loss = train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device)\n\n        # eval\n        avg_val_loss, preds = valid_fn(valid_loader, model, criterion, device)\n        valid_labels = valid_folds[CFG.target_col].values\n        \n        if isinstance(scheduler, ReduceLROnPlateau):\n            scheduler.step(avg_val_loss)\n        elif isinstance(scheduler, CosineAnnealingLR):\n            scheduler.step()\n        elif isinstance(scheduler, CosineAnnealingWarmRestarts):\n            scheduler.step()\n\n        # scoring\n        score = get_score(valid_labels, preds.argmax(1))\n\n        elapsed = time.time() - start_time\n\n        LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n        LOGGER.info(f'Epoch {epoch+1} - Accuracy: {score}')\n\n        if score > best_score:\n            best_score = score\n            LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n            torch.save({'model': model.state_dict(), \n                        'preds': preds,\n                       'avg_val_loss': avg_val_loss},\n                        OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.pth')\n    \n    check_point = torch.load(OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.pth')\n    valid_folds[[str(c) for c in range(5)]] = check_point['preds']\n    valid_folds['preds'] = check_point['preds'].argmax(1)\n    \n    return valid_folds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# main\n# ====================================================\ndef main():\n\n    \"\"\"\n    Prepare: 1.train  2.test  3.submission  4.folds\n    \"\"\"\n\n    def get_result(result_df):\n        preds = result_df['preds'].values\n        labels = result_df[CFG.target_col].values\n        score = get_score(labels, preds)\n        LOGGER.info(f'Score: {score:<.5f}')\n    \n    if CFG.train:\n        # train \n        oof_df = pd.DataFrame()\n        for fold in range(CFG.n_fold):\n            if fold in CFG.trn_fold:\n                _oof_df = train_loop(folds, fold)\n                oof_df = pd.concat([oof_df, _oof_df])\n                LOGGER.info(f\"========== fold: {fold} result ==========\")\n                get_result(_oof_df)\n        # CV result\n        LOGGER.info(f\"========== CV ==========\")\n        get_result(oof_df)\n        # save result\n        oof_df.to_csv(OUTPUT_DIR+'oof_df.csv', index=False)\n    \n    if CFG.inference:\n        # inference\n        model = CustomResNext(CFG.model_name, pretrained=False)\n        states = [torch.load(OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.pth') for fold in CFG.trn_fold]\n        test_dataset = TestDataset(test, transform=get_transforms(data='valid'))\n        test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, \n                                 num_workers=CFG.num_workers, pin_memory=True)\n        predictions = inference(model, states, test_loader, device)\n        # submission\n        test['label'] = predictions.argmax(1)\n        test[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Just one epoch! Chang epoch as you want"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n    main()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_df = pd.read_csv(OUTPUT_DIR+'oof_df.csv')\nprint('total length', len(result_df))\nresult_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"correct = []\nfor idx, row in result_df.iterrows():\n    if result_df.iloc[idx][1]==result_df.iloc[idx][9]:\n        correct.append('T')\n    else:\n        correct.append('F')\n        \nfor idx, row in result_df.iterrows():\n    \n        result_df['correct'] = correct","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data of each labels\nlabeled_0 = result_df[result_df['preds']==0]\nlabeled_1 = result_df[result_df['preds']==1]\nlabeled_2 = result_df[result_df['preds']==2]\nlabeled_3 = result_df[result_df['preds']==3]\nlabeled_4 = result_df[result_df['preds']==4]\n\n# low score\npoor_0 = result_df[(result_df[['0','1','2','3','4']].max(axis=1)<0.5) & (result_df['preds']==0)]\npoor_1 = result_df[(result_df[['0','1','2','3','4']].max(axis=1)<0.5) & (result_df['preds']==1)]\npoor_2 =result_df[(result_df[['0','1','2','3','4']].max(axis=1)<0.5) & (result_df['preds']==2)]\npoor_3 =result_df[(result_df[['0','1','2','3','4']].max(axis=1)<0.5) & (result_df['preds']==3)]\npoor_4 =result_df[(result_df[['0','1','2','3','4']].max(axis=1)<0.5) & (result_df['preds']==4)]\n\n# low score, incorrect result\npoor_0_wrong = result_df[(result_df[['0','1','2','3','4']].max(axis=1)<0.5) & (result_df['preds']==0)&(result_df['correct']=='F')]\npoor_1_wrong = result_df[(result_df[['0','1','2','3','4']].max(axis=1)<0.5) & (result_df['preds']==1)&(result_df['correct']=='F')]\npoor_2_wrong =result_df[(result_df[['0','1','2','3','4']].max(axis=1)<0.5) & (result_df['preds']==2)&(result_df['correct']=='F')]\npoor_3_wrong =result_df[(result_df[['0','1','2','3','4']].max(axis=1)<0.5) & (result_df['preds']==3)&(result_df['correct']=='F')]\npoor_4_wrong =result_df[(result_df[['0','1','2','3','4']].max(axis=1)<0.5) & (result_df['preds']==4)&(result_df['correct']=='F')]\n\n# high score, incorrect result\nhigh_0_wrong = result_df[(result_df[['0','1','2','3','4']].max(axis=1)>0.8) & (result_df['preds']==0)&(result_df['correct']=='F')]\nhigh_1_wrong = result_df[(result_df[['0','1','2','3','4']].max(axis=1)>0.8) & (result_df['preds']==1)&(result_df['correct']=='F')]\nhigh_2_wrong =result_df[(result_df[['0','1','2','3','4']].max(axis=1)>0.8) & (result_df['preds']==2)&(result_df['correct']=='F')]\nhigh_3_wrong =result_df[(result_df[['0','1','2','3','4']].max(axis=1)>0.8) & (result_df['preds']==3)&(result_df['correct']=='F')]\nhigh_4_wrong =result_df[(result_df[['0','1','2','3','4']].max(axis=1)>0.8) & (result_df['preds']==4)&(result_df['correct']=='F')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"noised = result_df[(result_df[['0','1','2','3','4']].max(axis=1)>0.8)&(result_df['correct']=='F')]\ndenoised = result_df[~((result_df[['0','1','2','3','4']].max(axis=1)>0.8)&(result_df['correct']=='F'))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"noised","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"denoised","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"relabeled = noised[(result_df[['0','1','2','3','4']].max(axis=1))>0.99]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"relabeled['label'] = relabeled['preds']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"relabeled","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"relabeled = denoised.append(relabeled, ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"denoised.to_csv(OUTPUT_DIR+'denoised.csv', index=False)\nrelabeled.to_csv(OUTPUT_DIR+'relabeled.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Incorrect but high scores: \",len(high_0_wrong), len(high_1_wrong), len(high_2_wrong), len(high_3_wrong), len(high_4_wrong))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# high accuracy, but incorrect result\nnot_clear_0 = len(high_0_wrong)/len(labeled_0)\nnot_clear_1 = len(high_1_wrong)/len(labeled_1)\nnot_clear_2 = len(high_2_wrong)/len(labeled_2)\nnot_clear_3 = len(high_3_wrong)/len(labeled_3)\nnot_clear_4 = len(high_4_wrong)/len(labeled_4)\nnot_clear = [not_clear_0, not_clear_1, not_clear_2, not_clear_3, not_clear_4]\nsns.barplot([0,1,2,3,4], not_clear)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# kdeplot of score of each labels\nplt.figure(figsize=(20,10))\nplt.subplot(2,3,1)\nsns.kdeplot(labeled_0['0'])\nplt.subplot(2,3,2)\nsns.kdeplot(labeled_1['1'])\nplt.subplot(2,3,3)\nsns.kdeplot(labeled_2['2'])\nplt.subplot(2,3,4)\nsns.kdeplot(labeled_3['3'])\nplt.subplot(2,3,5)\nsns.kdeplot(labeled_4['4'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# incorrect with high score - barplot labels\nplt.figure(figsize=(30,8))\nplt.subplot(1,5,1)\nsns.countplot(high_0_wrong['label'])\nplt.subplot(1,5,2)\nsns.countplot(high_1_wrong['label'])\nplt.subplot(1,5,3)\nsns.countplot(high_2_wrong['label'])\nplt.subplot(1,5,4)\nsns.countplot(high_3_wrong['label'])\nplt.subplot(1,5,5)\nsns.countplot(high_4_wrong['label'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Total length of each label: \",len(labeled_0), len(labeled_1), len(labeled_2), len(labeled_3), len(labeled_4))\nprint(\"\",len(poor_0),len(poor_1),len(poor_2),len(poor_3),len(poor_4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# low accuracy rate(there are both correct and incorrect data)\npoor_acc = {'0': len(poor_0)/len(labeled_0), '1': len(poor_1)/len(labeled_1), '2': len(poor_2)/len(labeled_2), \n            '3': len(poor_3)/len(labeled_3), '4': len(poor_4)/len(labeled_4)}\nprint(poor_acc)\nsns.barplot([0,1,2,3,4], list(poor_acc.values()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# incorrect answer rate\npoor_acc_wrong = {'0': len(poor_0_wrong)/len(labeled_0), '1': len(poor_1_wrong)/len(labeled_1), '2': len(poor_2_wrong)/len(labeled_2), \n            '3': len(poor_3_wrong)/len(labeled_3), '4': len(poor_4_wrong)/len(labeled_4)}\nprint(poor_acc_wrong)\nsns.barplot([0,1,2,3,4], list(poor_acc_wrong.values()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Incorrect but high scores"},{"metadata":{"trusted":true},"cell_type":"code","source":"high_0_wrong.head(12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,10))\nfor i in range(12):\n    plt.subplot(3, 4, i+1)\n    img = cv2.imread('../input/cassava-leaf-disease-merged/train/'+high_0_wrong.iloc[i][0])\n    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"high_1_wrong.head(12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,10))\nfor i in range(12): \n    plt.subplot(3, 4, i+1) \n    img = cv2.imread('../input/cassava-leaf-disease-merged/train/'+high_1_wrong.iloc[i][0]) \n    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,10))\nfor i in range(12):\n    plt.subplot(3, 4, i+1)\n    img = cv2.imread('../input/cassava-leaf-disease-merged/train/'+high_2_wrong.iloc[i][0])\n    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,10))\nfor i in range(12):\n    plt.subplot(3, 4, i+1)\n    img = cv2.imread('../input/cassava-leaf-disease-merged/train/'+high_3_wrong.iloc[i][0])\n    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,10))\nfor i in range(12):\n    plt.subplot(3, 4, i+1)\n    img = cv2.imread('../input/cassava-leaf-disease-merged/train/'+high_4_wrong.iloc[i][0])\n    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> ## Incorrect results"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,10))\nfor i in range(12):\n    plt.subplot(3, 4, i+1)\n    img = cv2.imread('../input/cassava-leaf-disease-merged/train/'+poor_0_wrong.iloc[i][0])\n    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,10))\nfor i in range(12):\n    plt.subplot(3, 4, i+1)\n    img = cv2.imread('../input/cassava-leaf-disease-merged/train/'+poor_1_wrong.iloc[i][0])\n    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,10))\nfor i in range(12):\n    plt.subplot(3, 4, i+1)\n    img = cv2.imread('../input/cassava-leaf-disease-merged/train/'+poor_2_wrong.iloc[i][0])\n    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,10))\nfor i in range(12):\n    plt.subplot(3, 4, i+1)\n    img = cv2.imread('../input/cassava-leaf-disease-merged/train/'+poor_4_wrong.iloc[i][0])\n    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}