{"cells":[{"metadata":{},"cell_type":"markdown","source":"\nThis notebook is derived from https://www.kaggle.com/debarshichanda/cassava-bitempered-logistic-loss.\n\ncomments made in areas that were not initially clear to me or for the benefit of others.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ver 22\n# crop_size = 256\n# LB - 0.865\n# ver 25\n# 512\n# center crop only\n# LB - 0.885\n# ver 26\n# original settings  whoops, not really.  original settings did not do center crop in training.\n# ver 27\n# train:  crop: randomresizecrop(512)\n# t1 = 0.8, t2 = 1.4\n# LB - 0.888\n# ver 28\n# t1 = 0.6, t2 = 1.6\n# ver 29\n# batch_size = 32 - barfed - out of CUDA menory\n# ver 31\n# batch_size = 16\n# # of epochs = 14\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# verifying GPU?\n!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\n\n# install modules for loading and using the pre-trained model\n# lots of model-goodness - https://github.com/rwightman/pytorch-image-models\n\n# if internet available can just do:\n# !pip install timm\n# !pip install pretrainedmodels\n\n# if internet not available\n# upload these two modules as input and pip will install from there\n!pip install ../input/timmmodels/dist/timm-0.3.4.tar\n!pip install ../input/pretrainedmodels/pretrainedmodels-0.7.4\n\n# also need to be able to load the actual weights from the pretrained model\n# this is available as a standard kaggle dataset \"timm-pretrained-efficientnet\"\n# make the folder then copy the file to it\nif not os.path.exists('/root/.cache/torch/hub/checkpoints/'):\n        os.makedirs('/root/.cache/torch/hub/checkpoints/')\n!cp ../input/timm-pretrained-efficientnet/efficientnet/tf_efficientnet_b4_ns-d6313a46.pth /root/.cache/torch/hub/checkpoints/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nimport copy\nimport time\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport torchvision\nfrom torchvision import models\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.cuda import amp\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.utils import class_weight\n\nfrom tqdm.notebook import tqdm\nfrom collections import defaultdict\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\nimport pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT_DIR = \"../input/cassava-leaf-disease-classification\"\nTRAIN_DIR = \"../input/cassava-leaf-disease-classification/train_images\"\nTEST_DIR = \"../input/cassava-leaf-disease-classification/test_images\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    model_name = 'tf_efficientnet_b4_ns'\n    img_size = 512\n    scheduler = 'CosineAnnealingWarmRestarts'\n    T_max = 10\n    T_0 = 10\n    lr = 1e-4\n    min_lr = 1e-6\n    batch_size = 16\n    weight_decay = 1e-6\n    seed = 42\n    num_classes = 5\n    num_epochs = 14\n    n_fold = 5\n    smoothing = 0.2\n    t1 = 0.6\n    t2 = 1.6\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    savemodel = True\n    savemodelpath = '/kaggle/working/bitemp-32.pth'\n    dosubmission = True\n    crop_size = 512\n    crop_p = 1.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed = 42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(f\"{ROOT_DIR}/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=CFG.n_fold)\nfor fold, ( _, val_) in enumerate(skf.split(X=df, y=df.label)):\n    df.loc[val_ , \"kfold\"] = int(fold)\n    \ndf['kfold'] = df['kfold'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaLeafDataset(nn.Module):\n    def __init__(self, root_dir, df, transforms=None):\n        self.root_dir = root_dir\n        self.df = df\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_path = os.path.join(self.root_dir, self.df.iloc[index, 0])\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        label = self.df.iloc[index, 1]\n        \n        if self.transforms:\n            img = self.transforms(image=img)[\"image\"]\n            \n        return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_transforms = {\n    \"train\": A.Compose([\n        A.RandomResizedCrop(CFG.img_size, CFG.img_size),\n        # A.CenterCrop(CFG.crop_size, CFG.crop_size, p=CFG.crop_p),\n        A.Transpose(p=0.5),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.ShiftScaleRotate(p=0.5),\n        A.HueSaturationValue(\n                hue_shift_limit=0.2, \n                sat_shift_limit=0.2, \n                val_shift_limit=0.2, \n                p=0.5\n            ),\n        A.RandomBrightnessContrast(\n                brightness_limit=(-0.1,0.1), \n                contrast_limit=(-0.1, 0.1), \n                p=0.5\n            ),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        A.CoarseDropout(p=0.5),\n        A.Cutout(p=0.5),\n        ToTensorV2()], p=1.),\n    \n    \"valid\": A.Compose([\n        A.CenterCrop(CFG.crop_size, CFG.crop_size, p=CFG.crop_p),\n        A.Resize(CFG.img_size, CFG.img_size),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        ToTensorV2()], p=1.)\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Code taken from https://github.com/fhopfmueller/bi-tempered-loss-pytorch/blob/master/bi_tempered_loss_pytorch.py\n\ndef log_t(u, t):\n    \"\"\"Compute log_t for `u'.\"\"\"\n    if t==1.0:\n        return u.log()\n    else:\n        return (u.pow(1.0 - t) - 1.0) / (1.0 - t)\n\ndef exp_t(u, t):\n    \"\"\"Compute exp_t for `u'.\"\"\"\n    if t==1:\n        return u.exp()\n    else:\n        return (1.0 + (1.0-t)*u).relu().pow(1.0 / (1.0 - t))\n\ndef compute_normalization_fixed_point(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t > 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same shape as activation with the last dimension being 1.\n    \"\"\"\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations_step_0 = activations - mu\n\n    normalized_activations = normalized_activations_step_0\n\n    for _ in range(num_iters):\n        logt_partition = torch.sum(\n                exp_t(normalized_activations, t), -1, keepdim=True)\n        normalized_activations = normalized_activations_step_0 * \\\n                logt_partition.pow(1.0-t)\n\n    logt_partition = torch.sum(\n            exp_t(normalized_activations, t), -1, keepdim=True)\n    normalization_constants = - log_t(1.0 / logt_partition, t) + mu\n\n    return normalization_constants\n\ndef compute_normalization_binary_search(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t < 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (< 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations = activations - mu\n\n    effective_dim = \\\n        torch.sum(\n                (normalized_activations > -1.0 / (1.0-t)).to(torch.int32),\n            dim=-1, keepdim=True).to(activations.dtype)\n\n    shape_partition = activations.shape[:-1] + (1,)\n    lower = torch.zeros(shape_partition, dtype=activations.dtype, device=activations.device)\n    upper = -log_t(1.0/effective_dim, t) * torch.ones_like(lower)\n\n    for _ in range(num_iters):\n        logt_partition = (upper + lower)/2.0\n        sum_probs = torch.sum(\n                exp_t(normalized_activations - logt_partition, t),\n                dim=-1, keepdim=True)\n        update = (sum_probs < 1.0).to(activations.dtype)\n        lower = torch.reshape(\n                lower * update + (1.0-update) * logt_partition,\n                shape_partition)\n        upper = torch.reshape(\n                upper * (1.0 - update) + update * logt_partition,\n                shape_partition)\n\n    logt_partition = (upper + lower)/2.0\n    return logt_partition + mu\n\nclass ComputeNormalization(torch.autograd.Function):\n    \"\"\"\n    Class implementing custom backward pass for compute_normalization. See compute_normalization.\n    \"\"\"\n    @staticmethod\n    def forward(ctx, activations, t, num_iters):\n        if t < 1.0:\n            normalization_constants = compute_normalization_binary_search(activations, t, num_iters)\n        else:\n            normalization_constants = compute_normalization_fixed_point(activations, t, num_iters)\n\n        ctx.save_for_backward(activations, normalization_constants)\n        ctx.t=t\n        return normalization_constants\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        activations, normalization_constants = ctx.saved_tensors\n        t = ctx.t\n        normalized_activations = activations - normalization_constants \n        probabilities = exp_t(normalized_activations, t)\n        escorts = probabilities.pow(t)\n        escorts = escorts / escorts.sum(dim=-1, keepdim=True)\n        grad_input = escorts * grad_output\n        \n        return grad_input, None, None\n\ndef compute_normalization(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example. \n    Backward pass is implemented.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    return ComputeNormalization.apply(activations, t, num_iters)\n\ndef tempered_sigmoid(activations, t, num_iters = 5):\n    \"\"\"Tempered sigmoid function.\n    Args:\n      activations: Activations for the positive class for binary classification.\n      t: Temperature tensor > 0.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_probabilities = tempered_softmax(internal_activations, t, num_iters)\n    return internal_probabilities[..., 0]\n\n\ndef tempered_softmax(activations, t, num_iters=5):\n    \"\"\"Tempered softmax function.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature > 1.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    if t == 1.0:\n        return activations.softmax(dim=-1)\n\n    normalization_constants = compute_normalization(activations, t, num_iters)\n    return exp_t(activations - normalization_constants, t)\n\ndef bi_tempered_binary_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing = 0.0,\n        num_iters=5,\n        reduction='mean'):\n\n    \"\"\"Bi-Tempered binary logistic loss.\n    Args:\n      activations: A tensor containing activations for class 1.\n      labels: A tensor with shape as activations, containing probabilities for class 1\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A loss tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_labels = torch.stack([labels.to(activations.dtype),\n        1.0 - labels.to(activations.dtype)],\n        dim=-1)\n    return bi_tempered_logistic_loss(internal_activations, \n            internal_labels,\n            t1,\n            t2,\n            label_smoothing = label_smoothing,\n            num_iters = num_iters,\n            reduction = reduction)\n\ndef bi_tempered_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing=0.0,\n        num_iters=5,\n        reduction = 'mean'):\n\n    \"\"\"Bi-Tempered Logistic Loss.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      labels: A tensor with shape and dtype as activations (onehot), \n        or a long tensor of one dimension less than activations (pytorch standard)\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing parameter between [0, 1). Default 0.0.\n      num_iters: Number of iterations to run the method. Default 5.\n      reduction: ``'none'`` | ``'mean'`` | ``'sum'``. Default ``'mean'``.\n        ``'none'``: No reduction is applied, return shape is shape of\n        activations without the last dimension.\n        ``'mean'``: Loss is averaged over minibatch. Return shape (1,)\n        ``'sum'``: Loss is summed over minibatch. Return shape (1,)\n    Returns:\n      A loss tensor.\n    \"\"\"\n\n    if len(labels.shape)<len(activations.shape): #not one-hot\n        labels_onehot = torch.zeros_like(activations)\n        labels_onehot.scatter_(1, labels[..., None], 1)\n    else:\n        labels_onehot = labels\n\n    if label_smoothing > 0:\n        num_classes = labels_onehot.shape[-1]\n        labels_onehot = ( 1 - label_smoothing * num_classes / (num_classes - 1) ) \\\n                * labels_onehot + \\\n                label_smoothing / (num_classes - 1)\n\n    probabilities = tempered_softmax(activations, t2, num_iters)\n\n    loss_values = labels_onehot * log_t(labels_onehot + 1e-10, t1) \\\n            - labels_onehot * log_t(probabilities, t1) \\\n            - labels_onehot.pow(2.0 - t1) / (2.0 - t1) \\\n            + probabilities.pow(2.0 - t1) / (2.0 - t1)\n    loss_values = loss_values.sum(dim = -1) #sum over classes\n\n    if reduction == 'none':\n        return loss_values\n    if reduction == 'sum':\n        return loss_values.sum()\n    if reduction == 'mean':\n        return loss_values.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, optimizer, scheduler, num_epochs, dataloaders, dataset_sizes, device, fold):\n    start = time.time()\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    history = defaultdict(list)\n    scaler = amp.GradScaler()\n\n    for epoch in range(1,num_epochs+1):\n        print('Epoch {}/{}'.format(epoch, num_epochs))\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train','valid']:\n            if(phase == 'train'):\n                model.train() # Set model to training mode\n            else:\n                model.eval() # Set model to evaluation mode\n            \n            running_loss = 0.0\n            running_corrects = 0.0\n            \n            # Iterate over data\n            for inputs,labels in tqdm(dataloaders[phase]):\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    with amp.autocast():\n                        outputs = model(inputs)\n                        _, preds = torch.max(outputs,1)\n                        loss = bi_tempered_logistic_loss(outputs, labels, t1=CFG.t1, t2=CFG.t2, label_smoothing=CFG.smoothing)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        scaler.scale(loss).backward()\n                        scaler.step(optimizer)\n                        scaler.update()\n\n\n                running_loss += loss.item()*inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data).double().item()\n\n            \n            epoch_loss = running_loss/dataset_sizes[phase]\n            epoch_acc = running_corrects/dataset_sizes[phase]\n\n            history[phase + ' loss'].append(epoch_loss)\n            history[phase + ' acc'].append(epoch_acc)\n\n            if phase == 'train' and scheduler != None:\n                scheduler.step()\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n                phase, epoch_loss, epoch_acc))\n            \n            # deep copy the model\n            if phase=='valid' and epoch_acc >= best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n                PATH = f\"Fold{fold}_{best_acc}_epoch{epoch}.bin\"\n                # torch.save(model.state_dict(), PATH)\n\n        print()\n\n    end = time.time()\n    time_elapsed = end - start\n    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    print(\"Best Accuracy \",best_acc)\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_fold(model, optimizer, scheduler, device, fold, num_epochs=10):\n    valid_df = df[df.kfold == fold]\n    train_df = df[df.kfold != fold]\n    \n    train_data = CassavaLeafDataset(TRAIN_DIR, train_df, transforms=data_transforms[\"train\"])\n    valid_data = CassavaLeafDataset(TRAIN_DIR, valid_df, transforms=data_transforms[\"valid\"])\n    \n    dataset_sizes = {\n        'train' : len(train_data),\n        'valid' : len(valid_data)\n    }\n    \n    train_loader = DataLoader(dataset=train_data, batch_size=CFG.batch_size, num_workers=4, pin_memory=True, shuffle=True)\n    valid_loader = DataLoader(dataset=valid_data, batch_size=CFG.batch_size, num_workers=4, pin_memory=True, shuffle=False)\n    \n    dataloaders = {\n        'train' : train_loader,\n        'valid' : valid_loader\n    }\n\n    model, history = train_model(model, optimizer, scheduler, num_epochs, dataloaders, dataset_sizes, device, fold)\n    \n    return model, history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = timm.create_model(CFG.model_name, pretrained=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_features = model.classifier.in_features\n# for param in model.parameters():\n#     param.requires_grad = False\nmodel.classifier = nn.Linear(num_features, CFG.num_classes)\nmodel.to(CFG.device);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = optim.Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fetch_scheduler(optimizer):\n    if CFG.scheduler == 'CosineAnnealingLR':\n        scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr)\n    elif CFG.scheduler == 'CosineAnnealingWarmRestarts':\n        scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr)\n    elif CFG.scheduler == None:\n        return None\n        \n    return scheduler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scheduler = fetch_scheduler(optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model, history = run_fold(model, optimizer, scheduler, device=CFG.device, fold=0, num_epochs=CFG.num_epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# this will save the full model to the configured path for use in inference from another notebook\nif(CFG.savemodel):\n    torch.save(model, CFG.savemodelpath)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# do the inference if configured\nif(CFG.dosubmission):\n    t_df = pd.read_csv(f\"{ROOT_DIR}/sample_submission.csv\")\n    T_DIR = TEST_DIR\n    t_data = CassavaLeafDataset(T_DIR, t_df, transforms=data_transforms[\"valid\"])\n    t_loader = DataLoader(dataset=t_data, batch_size=1, num_workers=4, pin_memory=True, shuffle=False)\n    \n    # make a copy of the dataframe used to read the inputs\n    # not sure about modifying dataframe used in the dataloader\n    submit_df = pd.DataFrame(t_df).copy(deep=True)\n    \n    for i, (inputs, label) in enumerate(t_loader):\n        inputs = inputs.to(CFG.device)\n        # make the prediction\n        outputs = model(inputs).detach().cpu().numpy()\n        # choose the largest to store as the label\n        pred_label = np.argmax(outputs)\n        # add that to the dataframe which will be saved as the submission\n        submit_df.iloc[i] = [t_df.iloc[i]['image_id'], pred_label]\n        \n    # save the submit dataframe to the submission file\n    submit_df.to_csv(\"/kaggle/working/submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}