{"cells":[{"metadata":{},"cell_type":"markdown","source":"### This kernel shows the use of different regularization/data augmentation techniques with different models. For fast results, I've only trained the models on 5000 images and validated the model on 50 images. Refere previous versions of this kernel for those implementations. Here is the list of versions: \n\n- v19: Pre-trained EfficientNet-B1 with GridMask + AugMix\n- v17: Pre-trained EfficientNet-B1 with GridMask\n- v14: ResNet101 with GridMask + Cutmix + Mixup (Changed train/validation size to 10k/200 images)\n- v13: ResNet101 with GridMask\n- v11: ResNet152 with GridMask\n- v10: ResNet50 with Cutmix and Mixup\n- v9: ResNet101 with Cutmix and Mixup\n- v7, v6, v5: ResNet18 with Cutmix and mixup with different augmentation probabilities\n\n### Stay tuned for more combinations with larger training and validation datasets!\n\nReferences: \n- I've refered the PyTorch source for implementations of ResNets: https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py\n- Implementation of GridMask: https://www.kaggle.com/haqishen/gridmask\n- Implementation of CutMix/Mixup: https://www.kaggle.com/c/bengaliai-cv19/discussion/126504\n\n### Feel free to comment if you find any bug."},{"metadata":{},"cell_type":"markdown","source":"### Here, for GridMask + CutMix + Mixup, again there are two possible variations: First Applying GridMask and then applying CutMix/MixUp and the other is vice versa. This kernel shows the former one. I'm not sure which one works better yet."},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --upgrade efficientnet-pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport random\n\nimport cv2\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.auto import tqdm\n\nimport torch\nfrom torch import nn\nimport albumentations\nfrom torchvision import transforms\nfrom torch.optim import lr_scheduler\nimport torch.utils.data as data_utils\nfrom torch.utils.data import Dataset, DataLoader\nfrom albumentations.core.transforms_interface import DualTransform\n\nfrom efficientnet_pytorch import EfficientNet\n\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\nfrom albumentations.augmentations import functional as F\nfrom PIL import Image, ImageOps, ImageEnhance\n\n\nfrom PIL import Image\n\nfrom sklearn.model_selection import train_test_split\n\nfrom matplotlib import pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Constants\nSEED = 1996\nBATCH_SIZE = 256 \nDIM = (128, 128)\nSIZE = 128\nHEIGHT = 137 \nWIDTH = 236\nPARTIAL_SIZE = 5000\nVAL_SIZE = 0.01\n\n# Set random seeds\nrandom.seed(SEED)\nnp.random.seed(SEED)\n\n# load files\nIMG_PATH = '../input/grapheme-imgs-128x128/'\ntrain_df = pd.read_csv('../input/bengaliai-cv19/train.csv')\ntest_df = pd.read_csv('../input/bengaliai-cv19/test.csv')\ntrain_df['filename'] = train_df.image_id.apply(lambda filename: os.path.join(IMG_PATH + filename + '.png'))\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\ntrain_df = train_df.drop(['grapheme'], axis=1)\n\n# top 5 samples\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: Partial data. Remove for actual training\ntrain_df = train_df[0:PARTIAL_SIZE]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, val_df = train_test_split(train_df, test_size=VAL_SIZE, random_state=SEED, shuffle=False, stratify=None)\nval_df = val_df.reset_index(drop=True)\nprint(train_df.shape, val_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_df.tail()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Reference for GridMask: https://www.kaggle.com/haqishen/gridmask\nReference for AugMix: https://www.kaggle.com/haqishen/augmix-based-on-albumentations"},{"metadata":{},"cell_type":"markdown","source":"## GridMask Regularization"},{"metadata":{"trusted":true},"cell_type":"code","source":"class GridMask(DualTransform):\n    \"\"\"GridMask augmentation for image classification and object detection.\n\n    Args:\n        num_grid (int): number of grid in a row or column.\n        fill_value (int, float, lisf of int, list of float): value for dropped pixels.\n        rotate ((int, int) or int): range from which a random angle is picked. If rotate is a single int\n            an angle is picked from (-rotate, rotate). Default: (-90, 90)\n        mode (int):\n            0 - cropout a quarter of the square of each grid (left top)\n            1 - reserve a quarter of the square of each grid (left top)\n            2 - cropout 2 quarter of the square of each grid (left top & right bottom)\n\n    Targets:\n        image, mask\n\n    Image types:\n        uint8, float32\n\n    Reference:\n    |  https://arxiv.org/abs/2001.04086\n    |  https://github.com/akuxcw/GridMask\n    \"\"\"\n\n    def __init__(self, num_grid=3, fill_value=0, rotate=0, mode=0, always_apply=False, p=0.5):\n        super(GridMask, self).__init__(always_apply, p)\n        if isinstance(num_grid, int):\n            num_grid = (num_grid, num_grid)\n        if isinstance(rotate, int):\n            rotate = (-rotate, rotate)\n        self.num_grid = num_grid\n        self.fill_value = fill_value\n        self.rotate = rotate\n        self.mode = mode\n        self.masks = None\n        self.rand_h_max = []\n        self.rand_w_max = []\n\n    def init_masks(self, height, width):\n        if self.masks is None:\n            self.masks = []\n            n_masks = self.num_grid[1] - self.num_grid[0] + 1\n            for n, n_g in enumerate(range(self.num_grid[0], self.num_grid[1] + 1, 1)):\n                grid_h = height / n_g\n                grid_w = width / n_g\n                this_mask = np.ones((int((n_g + 1) * grid_h), int((n_g + 1) * grid_w))).astype(np.uint8)\n                for i in range(n_g + 1):\n                    for j in range(n_g + 1):\n                        this_mask[\n                             int(i * grid_h) : int(i * grid_h + grid_h / 2),\n                             int(j * grid_w) : int(j * grid_w + grid_w / 2)\n                        ] = self.fill_value\n                        if self.mode == 2:\n                            this_mask[\n                                 int(i * grid_h + grid_h / 2) : int(i * grid_h + grid_h),\n                                 int(j * grid_w + grid_w / 2) : int(j * grid_w + grid_w)\n                            ] = self.fill_value\n                \n                if self.mode == 1:\n                    this_mask = 1 - this_mask\n\n                self.masks.append(this_mask)\n                self.rand_h_max.append(grid_h)\n                self.rand_w_max.append(grid_w)\n\n    def apply(self, image, mask, rand_h, rand_w, angle, **params):\n        h, w = image.shape[:2]\n        mask = F.rotate(mask, angle) if self.rotate[1] > 0 else mask\n        mask = mask[:,:,np.newaxis] if image.ndim == 3 else mask\n        image *= mask[rand_h:rand_h+h, rand_w:rand_w+w].astype(image.dtype)\n        return image\n\n    def get_params_dependent_on_targets(self, params):\n        img = params['image']\n        height, width = img.shape[:2]\n        self.init_masks(height, width)\n\n        mid = np.random.randint(len(self.masks))\n        mask = self.masks[mid]\n        rand_h = np.random.randint(self.rand_h_max[mid])\n        rand_w = np.random.randint(self.rand_w_max[mid])\n        angle = np.random.randint(self.rotate[0], self.rotate[1]) if self.rotate[1] > 0 else 0\n\n        return {'mask': mask, 'rand_h': rand_h, 'rand_w': rand_w, 'angle': angle}\n\n    @property\n    def targets_as_params(self):\n        return ['image']\n\n    def get_transform_init_args_names(self):\n        return ('num_grid', 'fill_value', 'rotate', 'mode')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## AugMix Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def int_parameter(level, maxval):\n    \"\"\"Helper function to scale `val` between 0 and maxval .\n    Args:\n    level: Level of the operation that will be between [0, `PARAMETER_MAX`].\n    maxval: Maximum value that the operation can have. This will be scaled to\n      level/PARAMETER_MAX.\n    Returns:\n    An int that results from scaling `maxval` according to `level`.\n    \"\"\"\n    return int(level * maxval / 10)\n\n\ndef float_parameter(level, maxval):\n    \"\"\"Helper function to scale `val` between 0 and maxval.\n    Args:\n    level: Level of the operation that will be between [0, `PARAMETER_MAX`].\n    maxval: Maximum value that the operation can have. This will be scaled to\n      level/PARAMETER_MAX.\n    Returns:\n    A float that results from scaling `maxval` according to `level`.\n    \"\"\"\n    return float(level) * maxval / 10.\n\n\ndef sample_level(n):\n    return np.random.uniform(low=0.1, high=n)\n\n\ndef autocontrast(pil_img, _):\n    return ImageOps.autocontrast(pil_img)\n\n\ndef equalize(pil_img, _):\n    return ImageOps.equalize(pil_img)\n\n\ndef posterize(pil_img, level):\n    level = int_parameter(sample_level(level), 4)\n    return ImageOps.posterize(pil_img, 4 - level)\n\n\ndef rotate(pil_img, level):\n    degrees = int_parameter(sample_level(level), 30)\n    if np.random.uniform() > 0.5:\n        degrees = -degrees\n    return pil_img.rotate(degrees, resample=Image.BILINEAR)\n\n\ndef solarize(pil_img, level):\n    level = int_parameter(sample_level(level), 256)\n    return ImageOps.solarize(pil_img, 256 - level)\n\n\ndef shear_x(pil_img, level):\n    level = float_parameter(sample_level(level), 0.3)\n    if np.random.uniform() > 0.5:\n        level = -level\n    return pil_img.transform(pil_img.size,\n                           Image.AFFINE, (1, level, 0, 0, 1, 0),\n                           resample=Image.BILINEAR)\n\n\ndef shear_y(pil_img, level):\n    level = float_parameter(sample_level(level), 0.3)\n    if np.random.uniform() > 0.5:\n        level = -level\n    return pil_img.transform(pil_img.size,\n                           Image.AFFINE, (1, 0, 0, level, 1, 0),\n                           resample=Image.BILINEAR)\n\n\ndef translate_x(pil_img, level):\n    level = int_parameter(sample_level(level), pil_img.size[0] / 3)\n    if np.random.random() > 0.5:\n        level = -level\n    return pil_img.transform(pil_img.size,\n                           Image.AFFINE, (1, 0, level, 0, 1, 0),\n                           resample=Image.BILINEAR)\n\n\ndef translate_y(pil_img, level):\n    level = int_parameter(sample_level(level), pil_img.size[0] / 3)\n    if np.random.random() > 0.5:\n        level = -level\n    return pil_img.transform(pil_img.size,\n                           Image.AFFINE, (1, 0, 0, 0, 1, level),\n                           resample=Image.BILINEAR)\n\n\n# operation that overlaps with ImageNet-C's test set\ndef color(pil_img, level):\n    level = float_parameter(sample_level(level), 1.8) + 0.1\n    return ImageEnhance.Color(pil_img).enhance(level)\n\n\n# operation that overlaps with ImageNet-C's test set\ndef contrast(pil_img, level):\n    level = float_parameter(sample_level(level), 1.8) + 0.1\n    return ImageEnhance.Contrast(pil_img).enhance(level)\n\n\n# operation that overlaps with ImageNet-C's test set\ndef brightness(pil_img, level):\n    level = float_parameter(sample_level(level), 1.8) + 0.1\n    return ImageEnhance.Brightness(pil_img).enhance(level)\n\n\n# operation that overlaps with ImageNet-C's test set\ndef sharpness(pil_img, level):\n    level = float_parameter(sample_level(level), 1.8) + 0.1\n    return ImageEnhance.Sharpness(pil_img).enhance(level)\n\n\naugmentations = [\n    autocontrast, equalize, posterize, rotate, solarize, shear_x, shear_y,\n    translate_x, translate_y\n]\n\naugmentations_all = [\n    autocontrast, equalize, posterize, rotate, solarize, shear_x, shear_y,\n    translate_x, translate_y, color, contrast, brightness, sharpness\n]\n\ndef normalize(image):\n    \"\"\"Normalize input image channel-wise to zero mean and unit variance.\"\"\"\n    return image - 127\n\ndef apply_op(image, op, severity):\n    #   image = np.clip(image, 0, 255)\n    pil_img = Image.fromarray(image)  # Convert to PIL.Image\n    pil_img = op(pil_img, severity)\n    return np.asarray(pil_img)\n\ndef augment_and_mix(image, severity=3, width=3, depth=-1, alpha=1.):\n    \"\"\"Perform AugMix augmentations and compute mixture.\n    Args:\n    image: Raw input image as float32 np.ndarray of shape (h, w, c)\n    severity: Severity of underlying augmentation operators (between 1 to 10).\n    width: Width of augmentation chain\n    depth: Depth of augmentation chain. -1 enables stochastic depth uniformly\n      from [1, 3]\n    alpha: Probability coefficient for Beta and Dirichlet distributions.\n    Returns:\n    mixed: Augmented and mixed image.\n    \"\"\"\n    ws = np.float32(\n      np.random.dirichlet([alpha] * width))\n    m = np.float32(np.random.beta(alpha, alpha))\n\n    mix = np.zeros_like(image).astype(np.float32)\n    for i in range(width):\n        image_aug = image.copy()\n        depth = depth if depth > 0 else np.random.randint(1, 4)\n        for _ in range(depth):\n            op = np.random.choice(augmentations)\n            image_aug = apply_op(image_aug, op, severity)\n        # Preprocessing commutes since all coefficients are convex\n        mix += ws[i] * image_aug\n#         mix += ws[i] * normalize(image_aug)\n\n    mixed = (1 - m) * image + m * mix\n#     mixed = (1 - m) * normalize(image) + m * mix\n    return mixed\n\n\nclass RandomAugMix(ImageOnlyTransform):\n\n    def __init__(self, severity=3, width=3, depth=-1, alpha=1., always_apply=False, p=0.5):\n        super().__init__(always_apply, p)\n        self.severity = severity\n        self.width = width\n        self.depth = depth\n        self.alpha = alpha\n\n    def apply(self, image, **params):\n        image = augment_and_mix(\n            image,\n            self.severity,\n            self.width,\n            self.depth,\n            self.alpha\n        )\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GraphemesDataset(Dataset):\n    def __init__(self, train_df, dim, transform):\n        self.train_df = train_df\n        self.dim = dim\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.train_df)\n    \n    @staticmethod\n    def to_grayscale(rgb_image):\n        return np.dot(rgb_image[... , :3] , [0.299 , 0.587, 0.114])\n    \n    def __getitem__(self, index):\n        # load the image file using cv2\n        image = cv2.imread(os.path.join(IMG_PATH + self.train_df['image_id'][index] + '.png'))\n        image = cv2.resize(image,  self.dim)\n        \n        if self.transform:\n            res = self.transform(image=image)\n            image = res['image'].astype(np.float32)\n        else:\n            image = image.astype(np.float32)\n\n        image = self.to_grayscale(image)\n        image /= 255\n        image = image.reshape(-1, SIZE, SIZE)\n        \n        target_root = self.train_df['grapheme_root'].iloc[index]\n        target_vowel = self.train_df['vowel_diacritic'].iloc[index]\n        target_consonant = self.train_df['consonant_diacritic'].iloc[index]\n        \n        return image, target_root, target_vowel, target_consonant","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_train = albumentations.Compose([\n    albumentations.OneOf([\n        GridMask(num_grid=3, mode=0, rotate=15),\n        GridMask(num_grid=3, mode=2, rotate=15),\n    ], p=0.7),\n    RandomAugMix(severity=4, width=3, alpha=1.0, p=0.7),\n])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = GraphemesDataset(train_df, DIM, transform=transforms_train)\ntrain_loader = data_utils.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_images(images):\n    fig, ax = plt.subplots(nrows=3, ncols=4, figsize=(16, 8))\n    count=0\n    for row in ax:\n        for col in row:\n            col.imshow(images[count].reshape(SIZE, SIZE).cpu().detach().numpy().astype(np.float64))\n            count += 1\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_images(next(iter(train_loader))[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate the validation dataset\nval_dataset = GraphemesDataset(val_df, DIM,transform=None)\nval_loader = data_utils.DataLoader(val_dataset, batch_size=int(PARTIAL_SIZE*VAL_SIZE), shuffle=False)\nX_val, Y_val_root, Y_val_vowel, Y_val_consonant = next(iter(val_loader))\nX_val.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_val = X_val.to(device, dtype=torch.float)\nY_val_root = Y_val_root.to(device)\nY_val_vowel = Y_val_vowel.to(device)\nY_val_consonant = Y_val_consonant.to(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Modeling"},{"metadata":{"trusted":true},"cell_type":"code","source":"class EfficientNetWrapper(nn.Module):\n    def __init__(self):\n        super(EfficientNetWrapper, self).__init__()\n        \n        # Load imagenet pre-trained model \n        self.effNet = EfficientNet.from_pretrained('efficientnet-b1', in_channels=1).to(device)\n        \n        # Appdend output layers based on our date\n        self.fc_root = nn.Linear(in_features=1000, out_features=168)\n        self.fc_vowel = nn.Linear(in_features=1000, out_features=11)\n        self.fc_consonant = nn.Linear(in_features=1000, out_features=7)\n        \n    def forward(self, X):\n        output = self.effNet(X)\n        output_root = self.fc_root(output)\n        output_vowel = self.fc_vowel(output)\n        output_consonant = self.fc_consonant(output)\n        \n        return output_root, output_vowel, output_consonant","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%capture\nmodel = EfficientNetWrapper().to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LEARNING_RATE = 0.02\nEPOCHS = 150\nCUTMIX_ALPHA = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = nn.DataParallel(model)\noptimizer = torch.optim.SGD(model.parameters(), lr=LEARNING_RATE)\n# lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5, verbose=True, threshold=0.0001, threshold_mode='rel', cooldown=0, min_lr=0.00001, eps=1e-08)\ncriterion = nn.CrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clear_cache():\n    gc.collect()\n    torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_accuracy(root_preds, target_root, vowel_pred, target_vowel, consonant_pred, target_consonant):\n    assert len(root_preds) == len(target_root) and len(vowel_pred) == len(target_vowel) and len(consonant_pred) == len(target_consonant)\n    \n    total = len(target_root) + len(target_vowel) + len(target_consonant)\n    _, predicted_root = torch.max(root_preds.data, axis=1)\n    _, predicted_vowel = torch.max(vowel_pred.data, axis=1)\n    _, predicted_consonant = torch.max(consonant_pred.data, axis=1)\n    \n    del root_preds\n    del vowel_pred\n    del consonant_pred\n    torch.cuda.empty_cache()\n\n    correct = (predicted_root == target_root).sum().item() + (predicted_vowel == target_vowel).sum().item() + (predicted_consonant == target_consonant).sum().item()\n    \n    del target_root\n    del target_vowel\n    del target_consonant\n    torch.cuda.empty_cache()\n    return correct / total","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"total_steps = len(train_loader)\nval_acc_list = []\nplot_flag = True\nfor epoch in range(EPOCHS):\n    for i, (x_train, target_root, target_vowel, target_consonant) in enumerate(train_loader):\n        x_train = x_train.to(device, dtype=torch.float)\n        target_root = target_root.to(device)\n        target_vowel = target_vowel.to(device)\n        target_consonant = target_consonant.to(device)\n\n        if plot_flag:\n            plot_flag = False\n            plot_images(x_train)\n        \n        # Forward pass\n        root_preds, vowel_pred, consonant_pred = model(x_train)\n        loss = (criterion(root_preds, target_root) + criterion(vowel_pred, target_vowel) + criterion(consonant_pred, target_consonant)) / 3\n        \n        del x_train\n        clear_cache()\n        \n        # Backpropagate\n        optimizer.zero_grad()  # Reason: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch\n        loss.backward()\n        optimizer.step()\n    \n#     lr_scheduler.step(loss.item())\n    del root_preds\n    del target_root\n    del vowel_pred\n    del target_vowel\n    del consonant_pred\n    del target_consonant\n    clear_cache()\n\n    # Calculate validation accuracy after each epoch\n    # Predict on validation set\n    root_val_preds, vowel_val_pred, consonant_val_pred = model(X_val)\n\n    val_acc = get_accuracy(root_val_preds, Y_val_root, vowel_val_pred, Y_val_vowel, consonant_val_pred, Y_val_consonant)\n    val_acc_list.append(val_acc)\n\n    del root_val_preds\n    del vowel_val_pred\n    del consonant_val_pred\n    clear_cache()\n\n    print('Epoch [{}/{}], Loss: {:.4f}, Validation accuracy: {:.2f}%'\n          .format(epoch + 1, EPOCHS, loss.item(), val_acc * 100))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.style.use('ggplot')\nplt.figure()\nplt.plot(np.arange(0, EPOCHS), val_acc_list, label='val_accuracy')\n\nplt.title('Accuracy')\nplt.xlabel('# of epochs')\nplt.ylabel('Accuracy')\nplt.legend(loc='upper right')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}