{"cells":[{"metadata":{},"cell_type":"markdown","source":"> Base Pytorch Code:\nhttps://www.kaggle.com/piantic/how-to-finetuning-models-pytorch-xla-tpu\n\n> Augmentations from:\nhttps://www.kaggle.com/capiru/cassavanet-cutmix-implementation-cv-0-9#Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/fastai-vit-base-patch32-384-taylorcrossentropyloss/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys;\npackage_path = '../input/pytorch-pretrained-models'\nsys.path.append (package_path)\npackage_path = '../input/timm-pytorch-image-models/pytorch-image-models-master'\nsys.path.append (package_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    debug=False\n    device='cuda:0' # ['TPU', 'GPU', 'cpu']\n    nprocs=1 # [1, 8]\n    num_workers=4\n    model_name='vit_base_patch32_384' # ['deit_base_patch16_224', 'vit_base_patch16_384', 'resnext50_32x4d', 'tf_efficientnet_b3_ns']\n    size=384  # [64, 128, 224, 384, 512]\n    freeze_epo = 1 # after these epochs, gradually unfreeze top layers\n    gradual_unfreez_epo = 7\n    epochs = 1 # freeze_epo + gradual_unfreez_epo \n    min_unfreez_layer = 2 # allowed to unfreeze layers 11 to 5 only and not less than 5\n    max_layer_no = 11 # for ViT\n    criterion='TaylorCrossEntropyLoss' # ['CrossEntropyLoss', LabelSmoothing', 'FocalLoss' 'FocalCosineLoss', 'SymmetricCrossEntropyLoss', 'BiTemperedLoss', 'TaylorCrossEntropyLoss']\n    T_0=10 # CosineAnnealingWarmRestarts\n    lr=1e-4\n    min_lr=1e-6\n    batch_size=64 #[32, 64]\n    weight_decay=1e-6\n    gradient_accumulation_steps=1\n    max_grad_norm=1\n    tta=3 # RandAugment \n    seed=42\n    target_size=5\n    target_col='label'\n    n_fold=5\n    train_fold=[0] #, 1, 2, 3, 4]\n    # infer_fold=[0, 1, 2, 3, 4]\n    train=True\n    smoothing=0.16\n    t1=0.3 # bi-tempered-loss https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/202017\n    t2=1.0 # bi-tempered-loss https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/202017\n    freeze=False\n    model_infer_path_prefix = \"../input/fastai-vit-base-patch32-384-taylorcrossentropyloss\"\n    model_train_path_prefix = \".\"\n    train_path = '../input/cassava-leaf-disease-classification/train_images'\n    train_csv  = '../input/cassava-leaf-disease-classification/train.csv'\n    test_path  = '../input/cassava-leaf-disease-classification/test_images'\n    test_csv   = '../input/cassava-leaf-disease-classification/test.csv'\n    output_dir = 'Output/'\n    \n    IMG_MEAN = [0.485, 0.456, 0.406] #Mean for normalization Transform cassava = [0.4303, 0.4967, 0.3134] imgnet = [0.485, 0.456, 0.406]\n    IMG_STD = [0.229, 0.224, 0.225] #STD for normalization Transform cassava = [0.2142, 0.2191, 0.1954] imgnet = [0.229, 0.224, 0.225]\n    \nif CFG.debug:\n    CFG.epochs = 1\n    train = train.sample(n=1000, random_state=CFG.seed).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-04T08:37:22.528111Z","iopub.status.busy":"2021-02-04T08:37:22.527527Z","iopub.status.idle":"2021-02-04T08:37:25.046051Z","shell.execute_reply":"2021-02-04T08:37:25.045382Z"},"papermill":{"duration":2.552077,"end_time":"2021-02-04T08:37:25.046244","exception":false,"start_time":"2021-02-04T08:37:22.494167","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport pandas as pd\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nfrom   torch.nn import CrossEntropyLoss, MSELoss\nfrom   torch.nn.modules.loss import _WeightedLoss\n\nimport fastai\nfrom   fastai.callback.mixup import MixUp, CutMix\nfrom   fastai.callback import *\nfrom   fastai.callback.all import *\nfrom   fastai.callback.training import GradientClip\nfrom   fastai.callback.all import SaveModelCallback, EarlyStoppingCallback, ReduceLROnPlateau \nfrom   fastai.data.core import *\nfrom   fastai.data.load import *\nfrom   fastai.learner import Learner\nfrom   fastai.metrics import *\nfrom   fastai.optimizer import OptimWrapper \n# from   warmup_scheduler import GradualWarmupScheduler\n\nfrom   sklearn import preprocessing\nfrom   sklearn.metrics import accuracy_score\nfrom   sklearn.model_selection import StratifiedKFold\n\nimport timm\nimport albumentations as A\nfrom   albumentations.pytorch import ToTensorV2\nimport cv2\n\nfrom   tqdm import tqdm\nfrom   pprint import pprint\nfrom   functools import partial\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings (\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = torch.device ('cuda:0' if torch.cuda.is_available () else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def init_logger (log_file=CFG.output_dir+'train.log'):\n    \n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger (__name__)\n    logger.setLevel (INFO)\n    handler1 = StreamHandler ()\n    handler1.setFormatter (Formatter (\"%(message)s\"))\n    handler2 = FileHandler (filename=log_file)\n    handler2.setFormatter (Formatter (\"%(message)s\"))\n    logger.addHandler (handler1)\n    logger.addHandler (handler2)\n    return logger","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything (seed):\n    \n    random.seed (seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed (seed)\n    torch.manual_seed (seed)\n    torch.cuda.manual_seed (seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Augmentations\n# ====================================================\n\nAug_Norm    = A.Normalize(mean=CFG.IMG_MEAN, std=CFG.IMG_STD, max_pixel_value=255.0, p=1.0)\nDrop_Rand   = A.CoarseDropout(max_holes=12, max_height=int(0.11*CFG.size), max_width=int(0.11*CFG.size),\n                            min_holes=1, min_height=int(0.03*CFG.size), min_width=int(0.03*CFG.size),\n                            always_apply=False, p=0.5)\nRand_Crop   = A.RandomCrop(height= CFG.size, width = CFG.size,always_apply=True, p=1.0)\nResize_Crop = A.RandomResizedCrop(CFG.size, CFG.size,p=1.0)\ntrain_transforms = A.Compose([\n            A.Transpose(p=0.5),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.5),\n            A.HueSaturationValue(\n                hue_shift_limit=0.2, \n                sat_shift_limit=0.2, \n                val_shift_limit=0.2, \n                p=0.5\n            ),\n            A.RandomBrightnessContrast(\n                    brightness_limit=(-0.1,0.1), \n                    contrast_limit=(-0.1, 0.1), \n                    p=0.5\n                ),\n            Resize_Crop,\n            Drop_Rand,           \n            Aug_Norm,   \n            ToTensorV2(p=1.0),\n        ], p=1.)\n\nlight_transforms = A.Compose([\n            A.Transpose(p=0.5),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.5),\n            A.HueSaturationValue(\n                        hue_shift_limit=0.2, \n                        sat_shift_limit=0.2, \n                        val_shift_limit=0.2, \n                        p=0.5),\n            A.RandomBrightnessContrast(\n                            brightness_limit=(-0.1,0.1), \n                            contrast_limit=(-0.1, 0.1), \n                            p=0.5),\n            Resize_Crop,\n            Aug_Norm,   \n            ToTensorV2(p=1.0),\n        ], p=1.)\n\nheavy_transforms = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    \n    A.Resize(CFG.size, CFG.size),\n    \n    A.Transpose(p=0.5),\n    A.VerticalFlip(p=0.5),\n    #A.augmentations.transforms.ColorJitter(brightness=0.10, contrast=0.2, saturation=0.2, hue=0.00, always_apply=False, p=0.5),\n    A.ShiftScaleRotate(p=0.5),\n    A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n    A.RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n    A.augmentations.transforms.RGBShift (r_shift_limit=20, g_shift_limit=20, b_shift_limit=20, always_apply=False, p=0.5),\n    A.augmentations.transforms.ChannelDropout (channel_drop_range=(1, 1), fill_value=0, always_apply=False, p=0.5),\n    \n    A.augmentations.transforms.GridDistortion (num_steps=5, distort_limit=0.3, interpolation=1, border_mode=4, value=None, mask_value=None, always_apply=False, p=0.5),\n    A.CoarseDropout(p=0.5),\n    A.Cutout(p=0.5),\n    Aug_Norm,\n    ToTensorV2(p=1.0),])\n\nvalid_transforms = A.Compose([\n            A.CenterCrop(CFG.size, CFG.size),\n            Aug_Norm,   \n            ToTensorV2(p=1.0),\n        ], p=1.)\n\ntest_aug = A.Compose([\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.ShiftScaleRotate(p = 1.0),\n            #A.ColorJitter(brightness=0.1, contrast=0.2, saturation=0.2, hue=0.00, always_apply=False, p=1.0),\n            Rand_Crop,\n            Aug_Norm,\n            ToTensorV2(p=1.0)\n        ], p=1.)\n\nimage_net_post = A.Compose([\n            Resize_Crop,\n            Drop_Rand,\n            Aug_Norm,    \n            ToTensorV2(p=1.0)\n        ], p=1.)\n\nclass UnNormalize(object):\n    def __init__(self, mean, std):\n        self.mean = mean\n        self.std = std\n\n    def __call__(self, tensor):\n        \"\"\"\n        Args:\n            tensor (Tensor): Tensor image of size (C, H, W) to be normalized.\n        Returns:\n            Tensor: Normalized image.\n        \"\"\"\n        for t, m, s in zip(tensor, self.mean, self.std):\n            t.mul_(s).add_(m)\n            # The normalize code -> t.sub_(m).div_(s)\n        return tensor\n    \ndef get_transforms (data='train', size=CFG.size):\n    \n    if 'train' in data:\n        return light_transforms\n    elif 'valid' in data:\n        return valid_transforms\n    else: # TTA\n        return test_aug","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Datasets: Fastai DS for training and Pytorch DS for inference","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset (Datasets):\n    \n    def __init__(self, df, transform=get_transforms(), path=CFG.train_path, isLabel=True):\n        \n        super (CassavaDataset, self).__init__(df, tfms=None, n_inp=1) \n        self.file_names = df['image_id'].values\n        self.transform  = transform\n        self.path       = path\n        self.isLabel    = isLabel\n        if isLabel:\n            self.labels = df['label'].values\n        return\n        \n    def __len__(self):        \n        return len (self.items) \n\n    def __getitem__(self, idx):\n        \n        file_name = self.file_names[idx]\n        file_path = f'{self.path}/{file_name}'\n        image     = cv2.imread (file_path)\n        image     = cv2.cvtColor (image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform (image=image)\n            image = augmented['image']\n        \n        if self.isLabel:\n            label = torch.tensor (self.labels[idx]).float () # long ()\n            return image, label\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def getFolds ():\n    \n    train_folds_df = pd.read_csv (CFG.train_csv)\n    skf = StratifiedKFold (n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)\n    for n, (train_index, val_index) in enumerate (skf.split (train_folds_df, train_folds_df[CFG.target_col])):\n        train_folds_df.loc[val_index, 'fold'] = int (n)\n    train_folds_df['fold'] = train_folds_df['fold'].astype (int)\n    # print (train_folds_df.groupby (['fold', CFG.target_col]).size ())\n    return train_folds_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test your Dataset and Dataloader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TR_DATASET = CassavaDataset (getFolds ())\nfor i in range(3):\n    image, label = TR_DATASET[i]       #;print (image.shape)\n    plt.imshow (image[2])\n    plt.title (f'label: {label}')\n    plt.show () \n\ndel TR_DATASET\ngc.collect ()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Losses"},{"metadata":{},"cell_type":"markdown","source":"> Label Smoothing"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Label Smoothing\n# ====================================================\nclass LabelSmoothingLoss(nn.Module): \n    def __init__(self, classes=5, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n        \n    def forward(self, pred, target): \n        pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> FocalLoss"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, reduce=True):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduce = reduce\n\n    def forward(self, inputs, targets):\n        BCE_loss = nn.CrossEntropyLoss()(inputs, targets)\n\n        pt = torch.exp(-BCE_loss)\n        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n\n        if self.reduce:\n            return torch.mean(F_loss)\n        else:\n            return F_loss\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> FocalCosineLoss"},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalCosineLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, xent=.1):\n        super(FocalCosineLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n        self.xent = xent\n\n        self.y = torch.Tensor([1]).cuda()\n\n    def forward(self, input, target, reduction=\"mean\"):\n        cosine_loss = F.cosine_embedding_loss(input, F.one_hot(target, num_classes=input.size(-1)), self.y, reduction=reduction)\n\n        cent_loss = F.cross_entropy(F.normalize(input), target, reduce=False)\n        pt = torch.exp(-cent_loss)\n        focal_loss = self.alpha * (1-pt)**self.gamma * cent_loss\n\n        if reduction == \"mean\":\n            focal_loss = torch.mean(focal_loss)\n\n        return cosine_loss + self.xent * focal_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> SymmetricCrossEntropy"},{"metadata":{"trusted":true},"cell_type":"code","source":"class SymmetricCrossEntropy(nn.Module):\n\n    def __init__(self, alpha=0.1, beta=1.0, num_classes=5):\n        super(SymmetricCrossEntropy, self).__init__()\n        self.alpha = alpha\n        self.beta = beta\n        self.num_classes = num_classes\n\n    def forward(self, logits, targets, reduction='mean'):\n        onehot_targets = torch.eye(self.num_classes)[targets].cuda()\n        ce_loss = F.cross_entropy(logits, targets, reduction=reduction)\n        rce_loss = (-onehot_targets*logits.softmax(1).clamp(1e-7, 1.0).log()).sum(1)\n        if reduction == 'mean':\n            rce_loss = rce_loss.mean()\n        elif reduction == 'sum':\n            rce_loss = rce_loss.sum()\n        return self.alpha * ce_loss + self.beta * rce_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Bi-Tempered-Loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"def log_t(u, t):\n    \"\"\"Compute log_t for `u'.\"\"\"\n    if t==1.0:\n        return u.log()\n    else:\n        return (u.pow(1.0 - t) - 1.0) / (1.0 - t)\n\ndef exp_t(u, t):\n    \"\"\"Compute exp_t for `u'.\"\"\"\n    if t==1:\n        return u.exp()\n    else:\n        return (1.0 + (1.0-t)*u).relu().pow(1.0 / (1.0 - t))\n\ndef compute_normalization_fixed_point(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t > 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same shape as activation with the last dimension being 1.\n    \"\"\"\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations_step_0 = activations - mu\n\n    normalized_activations = normalized_activations_step_0\n\n    for _ in range(num_iters):\n        logt_partition = torch.sum(\n                exp_t(normalized_activations, t), -1, keepdim=True)\n        normalized_activations = normalized_activations_step_0 * \\\n                logt_partition.pow(1.0-t)\n\n    logt_partition = torch.sum(\n            exp_t(normalized_activations, t), -1, keepdim=True)\n    normalization_constants = - log_t(1.0 / logt_partition, t) + mu\n\n    return normalization_constants\n\ndef compute_normalization_binary_search(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t < 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (< 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations = activations - mu\n\n    effective_dim = \\\n        torch.sum(\n                (normalized_activations > -1.0 / (1.0-t)).to(torch.int32),\n            dim=-1, keepdim=True).to(activations.dtype)\n\n    shape_partition = activations.shape[:-1] + (1,)\n    lower = torch.zeros(shape_partition, dtype=activations.dtype, device=activations.device)\n    upper = -log_t(1.0/effective_dim, t) * torch.ones_like(lower)\n\n    for _ in range(num_iters):\n        logt_partition = (upper + lower)/2.0\n        sum_probs = torch.sum(\n                exp_t(normalized_activations - logt_partition, t),\n                dim=-1, keepdim=True)\n        update = (sum_probs < 1.0).to(activations.dtype)\n        lower = torch.reshape(\n                lower * update + (1.0-update) * logt_partition,\n                shape_partition)\n        upper = torch.reshape(\n                upper * (1.0 - update) + update * logt_partition,\n                shape_partition)\n\n    logt_partition = (upper + lower)/2.0\n    return logt_partition + mu\n\nclass ComputeNormalization(torch.autograd.Function):\n    \"\"\"\n    Class implementing custom backward pass for compute_normalization. See compute_normalization.\n    \"\"\"\n    @staticmethod\n    def forward(ctx, activations, t, num_iters):\n        if t < 1.0:\n            normalization_constants = compute_normalization_binary_search(activations, t, num_iters)\n        else:\n            normalization_constants = compute_normalization_fixed_point(activations, t, num_iters)\n\n        ctx.save_for_backward(activations, normalization_constants)\n        ctx.t=t\n        return normalization_constants\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        activations, normalization_constants = ctx.saved_tensors\n        t = ctx.t\n        normalized_activations = activations - normalization_constants \n        probabilities = exp_t(normalized_activations, t)\n        escorts = probabilities.pow(t)\n        escorts = escorts / escorts.sum(dim=-1, keepdim=True)\n        grad_input = escorts * grad_output\n        \n        return grad_input, None, None\n\ndef compute_normalization(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example. \n    Backward pass is implemented.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    return ComputeNormalization.apply(activations, t, num_iters)\n\ndef tempered_sigmoid(activations, t, num_iters = 5):\n    \"\"\"Tempered sigmoid function.\n    Args:\n      activations: Activations for the positive class for binary classification.\n      t: Temperature tensor > 0.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_probabilities = tempered_softmax(internal_activations, t, num_iters)\n    return internal_probabilities[..., 0]\n\n\ndef tempered_softmax(activations, t, num_iters=5):\n    \"\"\"Tempered softmax function.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature > 1.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    if t == 1.0:\n        return activations.softmax(dim=-1)\n\n    normalization_constants = compute_normalization(activations, t, num_iters)\n    return exp_t(activations - normalization_constants, t)\n\ndef bi_tempered_binary_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing = 0.0,\n        num_iters=5,\n        reduction='mean'):\n\n    \"\"\"Bi-Tempered binary logistic loss.\n    Args:\n      activations: A tensor containing activations for class 1.\n      labels: A tensor with shape as activations, containing probabilities for class 1\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A loss tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_labels = torch.stack([labels.to(activations.dtype),\n        1.0 - labels.to(activations.dtype)],\n        dim=-1)\n    return bi_tempered_logistic_loss(internal_activations, \n            internal_labels,\n            t1,\n            t2,\n            label_smoothing = label_smoothing,\n            num_iters = num_iters,\n            reduction = reduction)\n\ndef bi_tempered_logistic_loss(activations,\n        labels,\n        t1=0.6,\n        t2=1.6,\n        label_smoothing=0.2,\n        num_iters=5,\n        reduction = 'mean'):\n\n    \"\"\"Bi-Tempered Logistic Loss.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      labels: A tensor with shape and dtype as activations (onehot), \n        or a long tensor of one dimension less than activations (pytorch standard)\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing parameter between [0, 1). Default 0.0.\n      num_iters: Number of iterations to run the method. Default 5.\n      reduction: ``'none'`` | ``'mean'`` | ``'sum'``. Default ``'mean'``.\n        ``'none'``: No reduction is applied, return shape is shape of\n        activations without the last dimension.\n        ``'mean'``: Loss is averaged over minibatch. Return shape (1,)\n        ``'sum'``: Loss is summed over minibatch. Return shape (1,)\n    Returns:\n      A loss tensor.\n    \"\"\"\n\n    if len(labels.shape)<len(activations.shape): #not one-hot\n        labels_onehot = torch.zeros_like(activations)\n        labels_onehot.scatter_(1, labels[..., None], 1)\n    else:\n        labels_onehot = labels\n\n    if label_smoothing > 0:\n        num_classes = labels_onehot.shape[-1]\n        labels_onehot = ( 1 - label_smoothing * num_classes / (num_classes - 1) ) \\\n                * labels_onehot + \\\n                label_smoothing / (num_classes - 1)\n\n    probabilities = tempered_softmax(activations, t2, num_iters)\n\n    loss_values = labels_onehot * log_t(labels_onehot + 1e-10, t1) \\\n            - labels_onehot * log_t(probabilities, t1) \\\n            - labels_onehot.pow(2.0 - t1) / (2.0 - t1) \\\n            + probabilities.pow(2.0 - t1) / (2.0 - t1)\n    loss_values = loss_values.sum(dim = -1) #sum over classes\n\n    if reduction == 'none':\n        return loss_values\n    if reduction == 'sum':\n        return loss_values.sum()\n    if reduction == 'mean':\n        return loss_values.mean()\n    \n\nclass BiTemperedLogisticLoss(nn.Module): \n    def __init__(self, t1=0.6, t2=1.6, smoothing=0.2): \n        super(BiTemperedLogisticLoss, self).__init__() \n        self.t1 = t1\n        self.t2 = t2\n        self.smoothing = smoothing\n    def forward(self, logit_label, truth_label):\n        loss_label = bi_tempered_logistic_loss(\n            logit_label, truth_label,\n            t1=self.t1, t2=self.t2,\n            label_smoothing=self.smoothing,\n            reduction='none'\n        )\n        \n        loss_label = loss_label.mean()\n        return loss_label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> TaylorCrossEntropyLoss with LabelSmoothing"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TaylorSoftmax(nn.Module):\n    '''\n    This is the autograd version\n    '''\n    def __init__(self, dim=1, n=2):\n        super(TaylorSoftmax, self).__init__()\n        assert n % 2 == 0\n        self.dim = dim\n        self.n = n\n\n    def forward(self, x):\n        '''\n        usage similar to nn.Softmax:\n            >>> mod = TaylorSoftmax(dim=1, n=4)\n            >>> inten = torch.randn(1, 32, 64, 64)\n            >>> out = mod(inten)\n        '''\n        fn = torch.ones_like(x)\n        denor = 1.\n        for i in range(1, self.n+1):\n            denor *= i\n            fn = fn + x.pow(i) / denor\n        out = fn / fn.sum(dim=self.dim, keepdims=True)\n        return out\n\n\nclass TaylorCrossEntropyLoss(nn.Module):\n\n    def __init__(self, n=2, ignore_index=-1, reduction='mean', smoothing=0.2):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n        self.lab_smooth = LabelSmoothingLoss(CFG.target_size, smoothing=smoothing)\n\n    def forward(self, logits, labels):\n        labels = torch.round (labels).long ()\n        log_probs = self.taylor_softmax(logits).log()\n        loss = self.lab_smooth(log_probs, labels)\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_criterion ():\n    \n    if CFG.criterion=='CrossEntropyLoss':\n        criterion = nn.CrossEntropyLoss()\n    elif CFG.criterion=='LabelSmoothing':\n        criterion = LabelSmoothingLoss(classes=CFG.target_size, smoothing=CFG.smoothing)\n    elif CFG.criterion=='FocalLoss':\n        criterion = FocalLoss().to(device)\n    elif CFG.criterion=='FocalCosineLoss':\n        criterion = FocalCosineLoss()\n    elif CFG.criterion=='SymmetricCrossEntropyLoss':\n        criterion = SymmetricCrossEntropy().to(device)\n    elif CFG.criterion=='BiTemperedLoss':\n        criterion = BiTemperedLogisticLoss(t1=CFG.t1, t2=CFG.t2, smoothing=CFG.smoothing)\n    elif CFG.criterion=='TaylorCrossEntropyLoss':\n        criterion = TaylorCrossEntropyLoss(smoothing=CFG.smoothing)\n    return criterion","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Models"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# MODEL\n# ====================================================\nclass CustomResNext(nn.Module):\n    def __init__(self, model_name=CFG.model_name, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \n    def freeze(self):\n        # To freeze the residual layers\n        for param in self.model.parameters():\n            param.requires_grad = False\n\n        for param in self.model.fc.parameters():\n            param.requires_grad = True\n    \n    def unfreeze(self):\n        # Unfreeze all layers\n        for param in self.model.parameters():\n            param.requires_grad = True\n            \n\nclass CustomEfficientNet(nn.Module):\n    def __init__(self, model_name=CFG.model_name, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(CFG.model_name, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n        \n    def freeze(self):\n        # To freeze the residual layers\n        for param in self.model.parameters():\n            param.requires_grad = False\n\n        for param in self.model.classifier.parameters():\n            param.requires_grad = True\n    \n    def unfreeze(self):\n        # Unfreeze all layers\n        for param in self.model.parameters():\n            param.requires_grad = True\n    \n    \nclass CustomDeiT(nn.Module):\n    def __init__(self, model_name=CFG.model_name, pretrained=False):\n        super().__init__()\n        self.model = torch.hub.load('facebookresearch/deit:main', model_name, pretrained=pretrained)\n        n_features = self.model.head.in_features\n        self.model.head = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \n    def freeze(self):\n        # To freeze the residual layers\n        for param in self.model.parameters():\n            param.requires_grad = False\n\n        for param in self.model.head.parameters():\n            param.requires_grad = True\n    \n    def unfreeze(self):\n        # Unfreeze all layers\n        for param in self.model.parameters():\n            param.requires_grad = True\n\n    \nclass CustomViT(nn.Module):\n    \n    def __init__(self, model_name=CFG.model_name, pretrained=False, \n                 min_unfreez_layer=CFG.min_unfreez_layer, max_layer_no=CFG.max_layer_no):\n        \n        super().__init__()\n        self.model      = timm.create_model(model_name, pretrained=pretrained)\n        n_features      = self.model.head.in_features\n        self.model.head = nn.Linear(n_features, CFG.target_size)\n        self.min_unfreez_layer = min_unfreez_layer\n        self.max_layer_no      = max_layer_no\n        return\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \n    def freeze(self):\n        # To freeze the residual layers\n        for param in self.model.parameters():\n            param.requires_grad = False\n\n        for param in self.model.head.parameters():\n            param.requires_grad = True\n        return\n    \n    def unfreeze(self):\n        # Unfreeze all layers\n        for param in self.model.parameters():\n            param.requires_grad = True\n        return\n    \n    def unfreeze_layer (self, layer_no=11):\n        # unfreeze a particular layer\n        if layer_no >= self.min_unfreez_layer and layer_no <= self.max_layer_no:\n            for param in self.model.blocks[layer_no].parameters ():\n                param.requires_grad = False\n        return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_state (model_path, model):\n    \n    state_dict = None\n    try:  # single GPU model_file\n        model.load_state_dict (torch.load (model_path, map_location=torch.device ('cpu')), strict=True)\n        state_dict = torch.load (model_path)['model']\n    except:  # multi GPU model_file\n        state_dict = torch.load (model_path, map_location=torch.device ('cpu'))\n        state_dict = {k[7:] if k.startswith ('module.') else k: state_dict[k] for k in state_dict.keys ()}\n        model.load_state_dict (state_dict)\n    return state_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def getModel (fold, isTrain=True):\n    \n    model = None\n    if 'deit_' in CFG.model_name:\n        model = CustomDeiT (model_name=CFG.model_name, pretrained=isTrain)\n    elif 'vit_' in CFG.model_name:\n        model = CustomViT (model_name=CFG.model_name, pretrained=False)  # TODO pretrained=isTrain\n    elif 'resnext' in CFG.model_name:\n        model = CustomResNext (CFG.model_name, pretrained=False)\n    elif 'efficientnet' in CFG.model_name:\n        model = CustomEfficientNet (CFG.model_name, pretrained=isTrain)        \n    if isTrain:\n        \n        # TODO: remove this\n        model_path = f'{CFG.model_infer_path_prefix}/{CFG.model_name}_fold{fold}.pth'\n        load_state (model_path, model)\n        torch.save (model.state_dict (), f'{CFG.model_train_path_prefix}/{CFG.model_name}_fold{fold}.pth')\n    else:\n        \n        # TODO: change CFG.model_train_path_prefix to CFG.model_infer_path_prefix\n        model_path = f'{CFG.model_train_path_prefix}/{CFG.model_name}_fold{fold}.pth'\n        load_state (model_path, model)\n        # torch.save (model.state_dict (), f'{CFG.model_train_path_prefix}/{CFG.model_name}_fold{fold}.pth')\n    if CFG.freeze:        \n        model.freeze ()\n    else:\n        model.unfreeze ()\n    return model","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-04T10:51:50.123379Z","iopub.status.busy":"2021-02-04T10:51:50.122532Z","iopub.status.idle":"2021-02-04T10:51:50.133568Z","shell.execute_reply":"2021-02-04T10:51:50.134006Z"},"papermill":{"duration":0.087782,"end_time":"2021-02-04T10:51:50.134186","exception":false,"start_time":"2021-02-04T10:51:50.046404","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from fastai.imports import *\nfrom fastai.torch_core import *\nfrom fastai.learner import *\n    \n@patch\n@delegates (subplots)\ndef plot_metrics (self: Recorder, nrows=None, ncols=None, figsize=None, **kwargs):\n    \n    metrics = np.stack(self.values)\n    names = self.metric_names[1:-1]\n    n = len(names) - 1\n    if nrows is None and ncols is None:\n        nrows = int(math.sqrt(n))\n        ncols = int(np.ceil(n / nrows))\n    elif nrows is None: nrows = int(np.ceil (n / ncols))\n    elif ncols is None: ncols = int(np.ceil (n / nrows))\n    figsize = figsize or (ncols * 6, nrows * 4)\n    fig, axs = subplots (nrows, ncols, figsize=figsize, **kwargs)\n    axs = [ax if i < n else ax.set_axis_off() for i, ax in enumerate (axs.flatten())][:n]\n    for i, (name, ax) in enumerate (zip (names, [axs[0]] + axs)):\n        ax.plot (metrics[:, i], color='#1f77b4' if i == 0 else '#ff7f0e', label='valid' if i > 0 else 'train')\n        ax.set_title (name if i > 1 else 'losses')\n        ax.legend (loc='best')\n    plt.show ()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@delegates (torch.optim.AdamW.__init__)\ndef pytorch_AdamW (param_groups, **kwargs):\n    return OptimWrapper (torch.optim.AdamW ([{'params': ps, **kwargs} for ps in param_groups]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class UnfreezeCallback (Callback):\n    \n    def __init__(self, freeze_epo=CFG.freeze_epo, max_layer_no=CFG.max_layer_no):\n        super().__init__()\n        self.freeze_epo   = freeze_epo\n        self.max_layer_no = max_layer_no\n        return\n        \n    def before_epoch (self): \n        if self.epoch >= self.freeze_epo:\n            self.learn.model.unfreeze_layer (self.max_layer_no - (self.epoch-self.freeze_epo))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_fold_loop (fold, train_df=getFolds ()):\n\n    LOGGER.info(f\"========== fold: {fold} training ==========\")\n\n    # ====================================================\n    # loader\n    # ====================================================\n    trn_idx = train_df[train_df['fold'] != fold].index\n    val_idx = train_df[train_df['fold'] == fold].index\n\n    train_folds_df = train_df.loc[trn_idx].reset_index (drop=True)\n    valid_folds_df = train_df.loc[val_idx].reset_index (drop=True)    \n    loss_func = get_criterion ()\n    model     = getModel (fold, isTrain=True)\n    # device  = torch.device (CFG.device)\n    model     = model.to (DEVICE)\n    modelfile = f'{CFG.model_train_path_prefix}/{CFG.model_name}_fold{fold}'\n    callbacks = [\n        EarlyStoppingCallback (monitor='accuracy', min_delta=0.0001, patience=12),\n        SaveModelCallback     (monitor='accuracy', fname=modelfile),\n        ReduceLROnPlateau     (monitor='accuracy', min_delta=0.0001, factor=2.0, min_lr=1e-8, patience=1),\n        GradientClip (CFG.max_grad_norm),\n        # UnfreezeCallback (),\n        CutMix (1.0)\n    ]            \n    size          = CFG.size\n    train_dataset = CassavaDataset (train_folds_df, transform=get_transforms ('train', size))\n    valid_dataset = CassavaDataset (valid_folds_df, transform=get_transforms ('valid', size))\n    train_datlder = DataLoader (train_dataset, batch_size=CFG.batch_size, device=DEVICE)\n    valid_datlder = DataLoader (valid_dataset, batch_size=CFG.batch_size, device=DEVICE)\n    dls           = DataLoaders (train_datlder, valid_datlder, device=DEVICE)\n    learn         = Learner (dls, model, loss_func=loss_func, model_dir=f'{CFG.model_train_path_prefix}',\n                     opt_func=partial (pytorch_AdamW, lr=0.007, betas=(0.9, 0.999), eps=1e-08, weight_decay=0.01),\n                     metrics=[accuracy, error_rate])   #fbeta, auc_roc_score\n    # print (learn.summary ())\n    \n    lr_min, _ = learn.lr_find (start_lr=1e-6, end_lr=1e-4, num_it=100) \n    print ('lr_min =', lr_min)\n    lr        = lr_min\n    learn.fit_one_cycle (CFG.epochs, lr, wd=CFG.weight_decay, cbs=callbacks)\n    learn.recorder.plot_metrics ()\n    # learn.recorder.plot_lr ()\n        \n    valid_scores = learn.validate (dl=valid_datlder)\n    return valid_scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_main ():\n    \n    if CFG.train:\n        # train \n        valid_scores = []\n        for fold in range (CFG.n_fold):\n            if fold in CFG.train_fold:\n                valid_scores_fold = train_fold_loop (fold)\n                valid_scores_fold = np.array (valid_scores_fold).reshape ((1, -1))\n                valid_scores.append (valid_scores_fold)\n                \n        LOGGER.info (f\"========== CV ==========\")\n        valid_scores = np.vstack (valid_scores)\n        valid_scores = np.mean (valid_scores, axis=0)\n        print (\"CV Scores =\", valid_scores)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train Main"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"gc.collect ()\n# model_names = timm.list_models (pretrained=True)\nmodel_names = timm.list_models ('vit_*', pretrained=True)\npprint (model_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p /kaggle/working/Output/\n!touch /kaggle/working/Output/train.log","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"LOGGER = init_logger ()\nseed_everything (seed=CFG.seed)\ntrain_main ()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.074102,"end_time":"2021-02-04T10:51:52.173717","exception":false,"start_time":"2021-02-04T10:51:52.099615","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_metrics (labels, preds):\n    \n    # preds   = pred_pr.argmax (-1)             # ;print ('labels.shape=', labels.shape, 'preds.shape=', preds.shape, 'pred_logits.shape=', pred_logits.shape)\n    precision, recall, f1, _ = precision_recall_fscore_support (labels, preds, average='macro')\n    acc     = accuracy_score (labels, preds)\n    mcc     = matthews_corrcoef (labels, preds)   # matthews correlation coefficient\n    # softmax = nn.Softmax (dim=1)\n    # pred_pr = softmax (torch.tensor (pred_logits))\n    # auc     = roc_auc_score (labels, pred_pr[:, 1])\n    metrics = {\n        'mcc'      : mcc,\n        'accuracy' : acc,\n        'f1'       : f1,\n        'precision': precision,\n        'recall'   : recall,\n        #'auc'     : auc\n    }\n    return metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TorchCassavaDataset (torch.utils.data.Dataset):\n    \n    def __init__(self, df, transform=get_transforms('valid'), path=CFG.test_path, isLabel=False):\n        \n        super ().__init__()\n        self.df         = df\n        self.file_names = df['image_id'].values\n        self.transform  = transform\n        self.path       = path\n        self.isLabel    = isLabel\n        if isLabel:\n            self.labels = df['label'].values\n        return\n        \n    def __len__(self):        \n        return self.df.shape[0]\n\n    def __getitem__(self, idx):\n        \n        file_name = self.file_names[idx]\n        file_path = f'{self.path}/{file_name}'\n        image     = cv2.imread (file_path)\n        image     = cv2.cvtColor (image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform (image=image)\n            image     = augmented['image']\n        \n        if self.isLabel:\n            label = torch.tensor (self.labels[idx]).long ()\n            return image, label\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_one_loop (model, data_loader, device):\n    model.eval ()\n    image_preds_all = []\n    pbar = tqdm (enumerate (data_loader), total=len (data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to (device).float ()\n        \n        image_preds = model (imgs) \n        image_preds_all += [torch.softmax (image_preds, 1).detach ().cpu ().numpy ()]\n    \n    image_preds_all = np.concatenate (image_preds_all, axis=0)\n    return image_preds_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def infer ():\n    \n    CFG.freeze = True\n    tst_preds_avg = []    \n    seed_everything (CFG.seed)\n    test_df = pd.DataFrame ()\n    test_df['image_id'] = list (os.listdir (CFG.test_path))\n    test_ds = TorchCassavaDataset (test_df)             #;print ('len (test_ds) =', len (test_ds)) ;print ('test_ds[10].shape =', test_ds[10].shape)\n    tst_loader = torch.utils.data.DataLoader (\n        test_ds, \n        batch_size=CFG.batch_size,\n        num_workers=CFG.num_workers,\n        shuffle=False,\n        pin_memory=False,\n    )                                            #;print ('len (tst_loader)=', len (tst_loader))\n    device = DEVICE  # torch.device (CFG.device)\n    for fold in CFG.train_fold:\n        \n        print ('Inference fold {} started'.format (fold))\n        model     = getModel (fold, isTrain=False).to (device)\n        tst_preds = []        \n        with torch.no_grad ():\n            for _ in range (CFG.tta):\n                \n                tst_preds += [1/CFG.tta * inference_one_loop (model, tst_loader, device)]\n        \n        tst_preds_avg.append (np.mean (tst_preds, axis=0))\n        del model\n        torch.cuda.empty_cache ()\n        gc.collect ()\n    \n    test_df['label'] = np.argmax (np.mean (tst_preds_avg, axis=0), axis=-1)\n    print (test_df.head ())\n    test_df.to_csv ('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.073737,"end_time":"2021-02-04T10:55:21.412066","exception":false,"start_time":"2021-02-04T10:55:21.338329","status":"completed"},"scrolled":true,"tags":[],"trusted":true},"cell_type":"code","source":"LOGGER = init_logger ()\nseed_everything (seed=CFG.seed)\ninfer ()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-04T10:55:21.261218Z","iopub.status.busy":"2021-02-04T10:55:21.26039Z","iopub.status.idle":"2021-02-04T10:55:21.26318Z","shell.execute_reply":"2021-02-04T10:55:21.263656Z"},"papermill":{"duration":0.08381,"end_time":"2021-02-04T10:55:21.26381","exception":false,"start_time":"2021-02-04T10:55:21.18","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print ('Done !')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}