{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Installing necessary libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"! /opt/conda/bin/python3.7 -m pip install -q --upgrade pip\n! pip install -q timm catalyst\n! pip install -q --upgrade wandb\n! pip install -q pytorch-gradcam","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We are going to use `wandb` for tracking our model's performance. If you don't have a `wandb` account, go to [this](wandb.ai) link, create an account using either google or github account. Then go to `wandb.ai/[your_username]` -> `Create New Project`. Give a cute little name to your project. Open your project page. You'll find some line like this:\n\n`wandb login e1da498db2dd649a76a04c6e4743e5a4f95a2ae0`\n\nCopy and paste this line to the next cell."},{"metadata":{"trusted":true},"cell_type":"code","source":"! wandb login e1da498db2dd649a76a04c6e4743e5a4f95a2ae0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Config\nThis section contains configuration parameters for my classification pipeline."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport cv2\nimport pandas as pd\nimport albumentations as A\nfrom albumentations.augmentations.transforms import Equalize, Posterize, Downscale\nfrom albumentations import (\n    PadIfNeeded, HorizontalFlip, VerticalFlip, CenterCrop,    \n    RandomCrop, Resize, Crop, Compose, HueSaturationValue,\n    Transpose, RandomRotate90, ElasticTransform, GridDistortion, \n    OpticalDistortion, RandomSizedCrop, Resize, CenterCrop,\n    VerticalFlip, HorizontalFlip, OneOf, CLAHE, Normalize,\n    RandomBrightnessContrast, Cutout, RandomGamma, ShiftScaleRotate ,\n    GaussNoise, Blur, MotionBlur, GaussianBlur, \n)\n\nSEED = 24\nn_epochs = 30\ndevice = 'cuda:0'\ndata_dir = '../input/cassava-leaf-disease-merged/'\nloss_thr = 1e6\nimg_path = f'{data_dir}/train'\ndf = pd.read_csv(f'{data_dir}/merged.csv')\ndf['path'] = df['image_id'].map(lambda x: f\"{img_path}/{x}\")\nencoder_model = 'gluon_seresnext50_32x4d'\nfold = 0\nmodel_name= f'SE-Resnext50_fold{fold}' # Will come up with a better name later\nmodel_dir = 'model_dir'\nhistory_dir = 'history_dir'\nload_model = False\nimg_dim = 320\nbatch_size = 32\naccum_step = 1\nlearning_rate = 2.00e-3\nnum_workers = 4\nmixed_precision = True\npatience = 3\nbalanced_sampler = False\ntrain_aug = A.Compose([A.CenterCrop(p=0.3, height=int(0.7*img_dim), width=int(0.7*img_dim)),\nA.augmentations.transforms.RandomCrop(int(0.7*img_dim), int(0.7*img_dim), p=0.3),\nA.augmentations.transforms.Rotate(limit=30, interpolation=1, border_mode=4, value=None, mask_value=None, always_apply=False, p=0.5),\nA.augmentations.transforms.Resize(img_dim, img_dim, interpolation=1, always_apply=True, p=0.6),\nCutout(num_holes=8, max_h_size=20, max_w_size=20, fill_value=0, always_apply=False, p=0.2),\nA.RandomBrightnessContrast(brightness_limit=0.3, contrast_limit=0.3, brightness_by_max=True, always_apply=False, p=0.3),\nA.augmentations.transforms.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=20, always_apply=False, p=0.4),\n# A.HorizontalFlip(p=0.5),\nA.VerticalFlip(p=0.5),                    \nOneOf([\n        GaussNoise(var_limit=0.1),\n        Blur(),\n        GaussianBlur(blur_limit=3),\n        # RandomGamma(p=0.7),\n        ], p=0.3),\nA.HorizontalFlip(p=0.3), Normalize(always_apply=True)],)\nval_aug = Compose([Normalize(always_apply=True)])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.label.hist()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fixing Seed"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport torch\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Sample Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib import pyplot as plt\n\n%matplotlib inline\nfig = plt.figure(figsize=(60, 60))\nfor class_id in sorted(df['label'].unique()):\n    for i, (idx, row) in enumerate(df.loc[df['label'] == class_id].sample(3, random_state=SEED).iterrows()):\n        ax = fig.add_subplot(5, 5, class_id * 5 + i + 1, xticks=[], yticks=[])\n        path= row['path']\n        image = cv2.imread(path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (img_dim, img_dim))\n        plt.imshow(image)\n        ax.set_title('Label: %s Name: %s' % (row['label'], row['image_id']), fontsize=30)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Stratification"},{"metadata":{"trusted":true},"cell_type":"code","source":"df1 = df[df['source']==2020]\ndf2 = df[df['source']==2019]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split, KFold, StratifiedKFold\nskf = StratifiedKFold(n_splits=5, random_state=SEED)\nX = df1['path']\ny = df1['label']\ntrain_idx = []\nval_idx = []\n\ndf1['fold'] = np.nan\ndf2['fold'] = np.nan\ndf2['fold'] = df2['fold'].map(lambda x: fold)\n\ndf1= df1.sample(frac=1, random_state=SEED).reset_index(drop=True)\n#split data\nfor i, (_, test_index) in enumerate(skf.split(X, y)):\n    df1.loc[test_index, 'fold'] = i\n    \ndf1['fold'] = df1['fold'].astype('int')\n\nvalid_df = df1[df1['fold']==fold]\ntrain_df1 = df1[df1['fold']!=fold]\ntrain_df = pd.concat([train_df1, df2])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset,DataLoader\n\ndef onehot(size, target, one_hot=False):\n    if not one_hot: return target\n    else:\n        vec = torch.zeros(size, dtype=torch.float32)\n        vec[target] = 1.\n        return vec\n\n\nclass LeafDataset(Dataset):\n    def __init__(self, df, dim=256, transforms=None):\n        super().__init__()\n        self.image_ids = df.path.tolist()\n        try:\n            self.labels = df.label.tolist()\n        except:\n            self.labels = None\n        self.transforms = transforms\n        self.dim = dim\n        \n    def __getitem__(self, idx):\n        image_id = self.image_ids[idx]\n        image = cv2.imread(image_id, cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (self.dim, self.dim))\n        \n        if self.transforms is not None:\n            aug = self.transforms(image=image)\n            image = aug['image'].reshape(self.dim, self.dim, 3).transpose(2, 0, 1)\n        else:\n            image = image.reshape(self.dim, self.dim, 3).transpose(2, 0, 1)\n        if self.labels is not None:\n            target = self.labels[idx]\n            return image_id, image, onehot(5, target, False)\n        else:\n            return image_id, image\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def get_labels(self):\n        return list(self.labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch import nn\nfrom torch.nn import *\nfrom torch.nn import functional as F\nfrom torchvision import models\nimport timm\n\nclass Resne_t(nn.Module):\n\n    def __init__(self, model_name):\n        super().__init__()\n        self.backbone = timm.create_model(model_name, pretrained=True)\n        self.in_features = self.backbone.fc.in_features\n        self.output = nn.Sequential(nn.Linear(self.in_features, 128), nn.Linear(128, 5))\n\n    def forward(self, x):\n        x = self.backbone.conv1(x)\n        x = self.backbone.bn1(x)\n        try:\n            x = self.backbone.act1(x)\n        except:\n            x = self.backbone.relu(x)\n        x = self.backbone.maxpool(x)\n\n        x = self.backbone.layer1(x)\n        x = self.backbone.layer2(x)\n        \n        x = self.backbone.layer3(x)\n        x = self.backbone.layer4(x)\n        x = self.backbone.global_pool(x)\n        x = x.view(x.size(0), -1)\n        x = self.output(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"import logging\nlogging.basicConfig(level=logging.ERROR)\nimport wandb\nfrom functools import partial\nfrom collections import Counter\nimport gc\nimport time\nimport pandas as pd\nfrom torch import optim\nfrom catalyst.data.sampler import BalanceClassSampler\n\nwandb.init(project=\"quantum_ai_demo\")\nwandb.run.name= model_name\n\nm_p = mixed_precision\nif m_p:\n  scaler = torch.cuda.amp.GradScaler() \n\nnp.random.seed(SEED)\n\ntrain_ds = LeafDataset(train_df, img_dim, train_aug)\nif balanced_sampler:\n  print('Using Balanced Sampler....')\n  train_loader = torch.utils.data.DataLoader(train_ds,batch_size=batch_size, sampler=BalanceClassSampler(labels=train_ds.get_labels(), mode=\"upsampling\"), shuffle=False, num_workers=4)\nelse:\n  train_loader = torch.utils.data.DataLoader(train_ds, batch_size=batch_size, shuffle=True, num_workers=num_workers, pin_memory=True)\n\nval_ds = LeafDataset(valid_df, img_dim, val_aug)\nvalid_loader = torch.utils.data.DataLoader(\nval_ds, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True)\n\nos.makedirs(model_dir, exist_ok=True)\nos.makedirs(history_dir, exist_ok=True)\n\nresult = pd.DataFrame(columns=['name', 'prediction', 'label', 'difference'])\nif os.path.exists(f'{history_dir}/history_{model_name}_{img_dim}.csv'):\n    history = pd.read_csv(f'{history_dir}/history_{model_name}_{img_dim}.csv')\nelse:\n    history = pd.DataFrame(columns=['train_loss','train_time','val_loss','val_cat_acc', 'val_time'])\n\nmodel = Resne_t(encoder_model).to(device)\nwandb.watch(model)\ncriterion = nn.CrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_model(valid_loss, valid_acc, best_valid_loss, best_valid_acc, best_state, savepath):\n    if valid_loss<best_valid_loss:\n        print(f'Validation loss has decreased from:  {best_valid_loss:.4f} to: {valid_loss:.4f}. Saving checkpoint')\n        torch.save(best_state, savepath+'_loss.pth')\n        best_valid_loss = valid_loss\n    if valid_acc>best_valid_acc:\n        print(f'Validation Accuracy score has increased from:  {best_valid_acc:.4f} to: {valid_acc:.4f}. Saving checkpoint')\n        torch.save(best_state, savepath + '_acc.pth')\n        best_valid_acc = valid_acc\n    else:\n        torch.save(best_state, savepath + '_last.pth')\n    return best_valid_loss, best_valid_acc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Confusion Matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib agg\nimport itertools\nfrom sklearn.metrics import confusion_matrix\nimport seaborn as sns\n\n\ndef plot_confusion_matrix(predictions, actual_labels, labels):\n    cm = confusion_matrix(predictions, actual_labels, labels)\n    # Normalise\n    cmn = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n    fig, ax = plt.subplots(figsize=(12,12))\n    sns.heatmap(cmn, annot=True, fmt='.2f', xticklabels=labels, yticklabels=labels)\n    plt.ylabel('Actual')\n    plt.xlabel('Predicted')\n    plt.savefig('confusion_matrix.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Grad-CAM Visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"from gradcam import GradCAM, GradCAMpp\n\ndef visualize_cam(mask, img, alpha=1.0, beta=0.15):\n    \n    \"\"\"\n    Courtesy: https://github.com/vickyliin/gradcam_plus_plus-pytorch/blob/master/gradcam/utils.py\n    Make heatmap from mask and synthesize GradCAM result image using heatmap and img.\n    Args:\n        mask (torch.tensor): mask shape of (1, 1, H, W) and each element has value in range [0, 1]\n        img (torch.tensor): img shape of (1, 3, H, W) and each pixel value is in range [0, 1]\n    Return:\n        heatmap (torch.tensor): heatmap img shape of (3, H, W)\n        result (torch.tensor): synthesized GradCAM result of same shape with heatmap.\n    \"\"\"\n    heatmap = (255 * mask.squeeze()).type(torch.uint8).cpu().numpy()\n    heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_MAGMA)\n    heatmap = torch.from_numpy(heatmap).permute(2, 0, 1).float().div(255)\n    b, g, r = heatmap.split(1)\n    heatmap = torch.cat([r, g, b]) * alpha\n\n    result = heatmap+img.cpu()*beta\n    result = result.div(result.max()).squeeze()\n\n    return heatmap, result\n\n\ndef grad_cam_gen(model, img, device = 'cuda'):     \n    configs = [dict(model_type='resnet', arch=model, layer_name='layer4_2_se_module_fc2')]\n    for config in configs:\n        config['arch'].to(device).eval()\n\n    cams = [\n    [cls.from_config(**config) for cls in (GradCAM, GradCAMpp)]\n        for config in configs]\n\n    for _, gradcam_pp in cams:\n        mask_pp, _ = gradcam_pp(img)\n        heatmap_pp, result_pp = visualize_cam(mask_pp, img)\n        result_pp = result_pp.cpu().numpy()\n        #convert image back to Height,Width,Channels\n        heatmap_pp = np.transpose(heatmap_pp, (1,2,0))\n        result_pp = np.transpose(result_pp, (1,2,0))\n        return result_pp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib agg\ndef plot_heatmap(model):\n    fig = plt.figure(figsize=(70, 56))\n    for class_id in sorted(valid_df['label'].unique()):\n        for i, (idx, row) in enumerate(valid_df.loc[valid_df['label'] == class_id].sample(5, random_state=SEED).iterrows()):\n            ax = fig.add_subplot(5, 5, class_id * 5 + i + 1, xticks=[], yticks=[])\n            path=f\"{row['path']}\"\n            image = cv2.imread(path, cv2.IMREAD_COLOR)\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            image = cv2.resize(image, (img_dim, img_dim))\n            aug = val_aug(image=image)\n            image = aug['image'].reshape(img_dim, img_dim, 3).transpose(2, 0, 1)\n            image = torch.FloatTensor(image)\n            prediction = torch.argmax(model(torch.unsqueeze(image.to(device), dim=0)))\n            prediction = prediction.data.cpu().numpy()\n            image = grad_cam_gen(model.backbone, torch.unsqueeze(image, dim=0).cuda())\n            ax.set_title('Label: %s Prediction: %s' % (row['label'], prediction), fontsize=30)\n            plt.imshow(image)\n            plt.savefig('heatmap.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_val(epoch, dataloader, optimizer, pretrained=None, train=True, mode='train', record=True):\n    global m_p\n    global result\n    global batch_size\n    global accum_step\n    t1 = time.time()\n    running_loss = 0\n    epoch_samples = 0\n    pred = []\n    lab = []\n    if pretrained:\n        model.load_state_dict(pretrained)\n    if train:\n        model.train()\n        print(\"Initiating train phase ...\")\n    else:\n        model.eval()\n        print(\"Initiating val phase ...\")\n    for idx, (_, img, labels) in enumerate(dataloader):\n        with torch.set_grad_enabled(train):\n            img = img.to(device, dtype=torch.float32)\n            labels = torch.LongTensor(labels).to(device)\n            epoch_samples += len(img)\n            optimizer.zero_grad()\n            with torch.cuda.amp.autocast(m_p):\n                if m_p:\n                    img = img.half()\n                else:\n                    img = img.float()\n                outputs = model(img)\n\n                loss = criterion(outputs, labels).sum()\n                running_loss += loss.item()*len(img)\n                loss = loss/accum_step\n      \n                if train:\n                     if m_p:\n                         scaler.scale(loss).backward()\n                         if (idx+1) % accum_step == 0:\n                             scaler.step(optimizer)\n                             scaler.update() \n                             optimizer.zero_grad()\n                     else:\n                         loss.backward()\n                         if (idx+1) % accum_step == 0:\n                             optimizer.step()\n                             optimizer.zero_grad()\n\n        elapsed = int(time.time() - t1)\n        eta = int(elapsed / (idx+1) * (len(dataloader)-(idx+1)))\n        pred.append(torch.argmax(outputs, dim=1).detach().cpu().numpy())\n        lab.append(labels.cpu().numpy())\n        if train:\n            msg = f\"Epoch: {epoch} Progress: [{idx}/{len(dataloader)}] loss: {(running_loss/epoch_samples):.4f} Time: {elapsed}s ETA: {eta} s\"\n        else:\n            msg = f'Epoch {epoch} Progress: [{idx}/{len(dataloader)}] loss: {(running_loss/epoch_samples):.4f} Time: {elapsed}s ETA: {eta} s'\n        wandb.log({\"Train Loss\": running_loss/epoch_samples})\n        print(msg, end= '\\r')\n    cat_acc = (np.concatenate(pred)==np.concatenate(lab)).mean()\n    history.loc[epoch, f'{mode}_loss'] = running_loss/epoch_samples\n    history.loc[epoch, f'{mode}_time'] = elapsed\n    if mode=='val' or mode=='test':\n        lr_reduce_scheduler.step(cat_acc)\n        msg = f'{mode} Loss: {running_loss/epoch_samples:.4f} \\n {mode} Categorical Accuracy: {cat_acc:.4f}'\n        print(msg)\n        wandb.log({f\"{mode} Loss\": running_loss/epoch_samples, f\"{mode} Categorical Accuracy\":cat_acc})\n        plot_confusion_matrix(np.concatenate(lab), np.concatenate(pred), [i for i in range(5)])\n        plot_heatmap(model)\n        conf = cv2.imread('./confusion_matrix.png', cv2.IMREAD_COLOR)\n        conf = cv2.cvtColor(conf, cv2.COLOR_BGR2RGB)\n        wandb.log({\"Confusion_Matrix\": [wandb.Image(conf, caption=\"Confusion Matrix\")]})\n        heatmap = cv2.imread('./heatmap.png', cv2.IMREAD_COLOR)\n        heatmap = cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB)\n        wandb.log({\"Class Activation Mapping\": [wandb.Image(heatmap, caption=\"Heatmap\")]})\n        history.loc[epoch, f'{mode}_loss'] = running_loss/epoch_samples\n        history.loc[epoch, f'{mode}_cat_acc'] = cat_acc\n        # NaN check\n        if running_loss/epoch_samples > loss_thr or running_loss!=running_loss:\n            print('\\033[91mMixed Precision\\033[0m rendering nan value. Forcing \\033[91mMixed Precision\\033[0m to be False ...')\n            m_p = False\n            batch_size = batch_size//2\n            accum_step = accum_step*2\n            print('Loading last best model ...')\n            tmp = torch.load(os.path.join(model_dir, model_name+'_loss.pth'))\n            model.load_state_dict(tmp['model'])\n            optimizer.load_state_dict(tmp['optim'])\n            lr_reduce_scheduler.load_state_dict(tmp['scheduler'])\n            del tmp\n            \n        if record:\n            history.to_csv(f'{history_dir}/history_{model_name}_{img_dim}.csv', index=False)\n        return running_loss/epoch_samples, cat_acc\n\n\nplist = [ \n        {'params': model.backbone.parameters(),  'lr': learning_rate/100},\n        {'params': model.output.parameters(),  'lr': learning_rate}\n    ]\noptimizer = optim.Adam(plist, lr=learning_rate)\nlr_reduce_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=patience, verbose=True, threshold=1e-4, threshold_mode='rel', cooldown=0, min_lr=1e-7, eps=1e-08)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def main():\n  prev_epoch_num = 0\n  best_valid_loss = np.inf\n  best_valid_acc = 0.0\n\n  if load_model:\n    tmp = torch.load(os.path.join(model_dir, model_name+'_acc.pth'))\n    model.load_state_dict(tmp['model'])\n    optimizer.load_state_dict(tmp['optim'])\n    lr_reduce_scheduler.load_state_dict(tmp['scheduler'])\n    scaler.load_state_dict(tmp['scaler'])\n    prev_epoch_num = tmp['epoch']\n    best_valid_loss = tmp['best_loss']\n    best_valid_loss, best_valid_acc = train_val(prev_epoch_num+1, valid_loader, optimizer=optimizer, train=False, mode='val')\n    del tmp\n    print('Model Loaded!')\n  \n  for epoch in range(prev_epoch_num, n_epochs):\n    torch.cuda.empty_cache()\n    print(gc.collect())\n\n    train_val(epoch, train_loader, optimizer=optimizer, train=True, mode='train')\n    valid_loss, valid_acc = train_val(epoch, valid_loader, optimizer=optimizer, train=False, mode='val')\n    print(\"#\"*20)\n    print(f\"Epoch {epoch} Report:\")\n    print(f\"Validation Loss: {valid_loss :.4f} Validation ACC: {valid_acc :.4f}\")\n    best_state = {'model': model.state_dict(), 'optim': optimizer.state_dict(), 'scheduler':lr_reduce_scheduler.state_dict(), \n          'scaler': scaler.state_dict(),\n    'best_loss':valid_loss, 'best_acc':valid_acc, 'epoch':epoch}\n    best_valid_loss, best_valid_acc = save_model(valid_loss, valid_acc, best_valid_loss, best_valid_acc, best_state, os.path.join(model_dir, model_name))\n    print(\"#\"*20)\n   \nif __name__== '__main__':\n  main()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}