{"cells":[{"metadata":{},"cell_type":"markdown","source":"# SIIM-ISIC SE_ResNeXT50\n\nHello everyone,\n\nThis notebook is a baseline for future experiments with SE_ResNeXT50. \n\nHere are the tips/tricks present in the training pipeline: \n- Use of meta-features\n- Differential learning rates for SE_ResNeXT and head for meta-features\n- Use of awesome Alex Shonenkov's dataset\n- BalanceClassSampler\n- HairAugmentation\n- SoftMarginFocalLoss\n\nCredits go to:\n- Alex Shonenkov for his great starter notebook: https://www.kaggle.com/shonenkov/training-cv-melanoma-starter\n- Roman's hair augmentation: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/159176\n\nThis notebook is also inspired from my EDA notebook: https://www.kaggle.com/rftexas/siim-isic-melanoma-analysis-eda-efficientnetb1","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Installing dependencies","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install -q pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os, re, random, gc\nfrom tqdm import tqdm\nfrom collections import OrderedDict\nfrom glob import glob\n\nfrom datetime import datetime\nimport time\n\nimport numpy as np\nimport pandas as pd\n\nimport torch \nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch.optim import Adam, lr_scheduler\nfrom torch.utils.data.sampler import WeightedRandomSampler\nfrom torch.utils.data import Dataset\n\nfrom catalyst.data.sampler import BalanceClassSampler\n\nimport pretrainedmodels\n\nimport albumentations\nfrom albumentations.pytorch.transforms import ToTensorV2\n\nimport cv2\n\nfrom sklearn.metrics import roc_auc_score\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Config","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainConfig:\n    num_workers = 8\n    batch_size = 256\n    \n    lr_cnn = 1e-3\n    lr_meta = 1e-2\n    \n    num_epochs = 8\n    seed = 2020\n    \n    verbose = True\n    verbose_step = 1\n    \n    step_scheduler = True\n    \n    scheduler_params = dict(\n        mode='min',\n        factor=0.5,\n        patience=1, \n        verbose=False,\n        threshold=0.0001,\n        threshold_mode='abs',\n        cooldown=0,\n        min_lr=1e-8,\n        eps=1e-8\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = '../input/melanoma-merged-external-data-512x512-jpeg/'\nTRAIN_DATA_PATH = DATA_PATH + '512x512-dataset-melanoma/512x512-dataset-melanoma/'\nTEST_CSV = '/kaggle/input/siim-isic-melanoma-classification/test.csv'\n\nWIDTH = 128\nHEIGHT = 128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading data\n\ndf = pd.read_csv(DATA_PATH + 'folds.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(TrainConfig.seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter(object):\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n    \n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RocAucMeter(object):\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.y_true = np.array([0, 1])\n        self.y_pred = np.array([0.5, 0.5])\n        self.score = 0\n        \n    def update(self, y_true, y_pred):\n        y_true = y_true.cpu().numpy().argmax(axis=1).astype(int)\n        y_pred = 1 - F.softmax(y_pred, dim=1).data.cpu().numpy()[:, 0]\n        self.y_true = np.hstack((self.y_true, y_true))\n        self.y_pred = np.hstack((self.y_pred, y_pred))\n        self.score = roc_auc_score(self.y_true, self.y_pred)\n        \n    @property\n    def avg(self):\n        return self.score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class HairAugmentation(albumentations.ImageOnlyTransform):\n    def __init__(self, \n                 max_hairs:int = 4, \n                 hairs_folder: str = \"/kaggle/input/melanoma-hairs\", \n                 p=0.5):\n        \n        super().__init__(p=p)\n        self.max_hairs = max_hairs\n        self.hairs_folder = hairs_folder\n    \n    def apply(self, img, **params):\n        n_hairs = random.randint(0, self.max_hairs)\n\n        if not n_hairs:\n            return img\n\n        height, width, _ = img.shape  # target image width and height\n        hair_images = [im for im in os.listdir(self.hairs_folder) if 'png' in im]\n\n        for _ in range(n_hairs):\n            hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))\n            hair = cv2.flip(hair, random.choice([-1, 0, 1]))\n            hair = cv2.rotate(hair, random.choice([0, 1, 2]))\n            \n            h_height, h_width, _ = hair.shape  # hair image width and height\n            hair = cv2.resize(hair, (int(h_width*0.8), int(h_height*0.8)))\n            \n            h_height, h_width, _ = hair.shape  # hair image width and height\n            roi_ho = random.randint(0, img.shape[0] - hair.shape[0])\n            roi_wo = random.randint(0, img.shape[1] - hair.shape[1])\n            roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]\n\n            img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)\n            ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)\n            mask_inv = cv2.bitwise_not(mask)\n            img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv).astype(np.float32)\n            hair_fg = cv2.bitwise_and(hair, hair, mask=mask).astype(np.float32)\n\n            dst = cv2.add(img_bg, hair_fg)\n            img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst\n\n        return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transforms():\n\n    return albumentations.Compose([\n\n        HairAugmentation(p=0.5),\n\n        albumentations.ShiftScaleRotate(p=0.9),       \n        albumentations.HorizontalFlip(p=0.5),\n        albumentations.VerticalFlip(p=0.5),\n\n\n        albumentations.OneOf([\n\n            albumentations.CLAHE(p=0.5),\n            albumentations.RandomBrightnessContrast(p=0.9),\n            albumentations.HueSaturationValue(p=0.5),\n\n        ]),\n\n        albumentations.OneOf([\n\n            albumentations.GridDistortion(p=0.5),\n            albumentations.ElasticTransform(p=0.5),\n\n        ]),\n\n        albumentations.CoarseDropout(p=0.5),\n\n        albumentations.Resize(width=WIDTH, height=HEIGHT, p=1.0),\n        \n        albumentations.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225],\n            p=1.0\n        ),\n        \n        ToTensorV2(p=1.0),\n\n    ])\n    \n\ndef get_valid_transforms():\n\n    return albumentations.Compose([\n\n        albumentations.Resize(width=WIDTH, height=HEIGHT, p=1.0),\n        \n        albumentations.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225],\n            p=1.0\n        ),\n        \n        ToTensorV2(p=1.0),\n\n    ])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Generating meta-features","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(TEST_CSV)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# One-hot encoding of anatom_site_general_challenge feature\nconcat = pd.concat([df['anatom_site_general_challenge'], \n                    test_df['anatom_site_general_challenge']], ignore_index=True)\ndummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')\ndf = pd.concat([df, dummies.iloc[:df.shape[0]]], axis=1)\ntest_df = pd.concat([test_df, dummies.iloc[df.shape[0]:].reset_index(drop=True)], axis=1)\n\n# Sex features\ndf['sex'] = df['sex'].map({'male': 1, 'female': 0})\ntest_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})\ndf['sex'] = df['sex'].fillna(-1)\ntest_df['sex'] = test_df['sex'].fillna(-1)\n\n# Age features\ndf['age_approx'] /= df['age_approx'].max()\ntest_df['age_approx'] /= test_df['age_approx'].max()\ndf['age_approx'] = df['age_approx'].fillna(-99)\ntest_df['age_approx'] = test_df['age_approx'].fillna(-99)\n\ndf['patient_id'] = df['patient_id'].fillna(-999)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"meta_features = ['sex', 'age_approx'] + [col for col in df.columns if 'site_' in col]\nmeta_features.remove('anatom_site_general_challenge')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.to_csv('test.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset class","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    def __init__(self, \n                 image_ids, \n                 targets, \n                 meta_features, \n                 augmentations=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.targets = targets\n        self.meta_features = meta_features\n        self.augmentations = augmentations\n    \n    def __len__(self):\n        return len(self.image_ids)\n    \n    def __getitem__(self, item):\n        \n        # Image\n        path = TRAIN_DATA_PATH + self.image_ids[item] + '.jpg'\n        image = cv2.imread(path)\n                        \n        if self.augmentations:\n            augmented = self.augmentations(image=image)\n            image = augmented['image']\n                                            \n        # Meta-features\n        patient_info = np.array(self.meta_features.iloc[item].values, dtype=np.float32)\n        \n        return {\n            'image': image,\n            'target': self.one_hot(2, self.targets[item]),\n            'meta': torch.tensor(patient_info, dtype=torch.float),\n        }\n    \n    def get_targets(self):\n        return list(self.targets)\n    \n    @staticmethod\n    def one_hot(size, target):\n        tensor = torch.zeros(size, dtype=torch.float32)\n        tensor[target] = 1.\n        return tensor","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class SoftMarginFocalLoss(nn.Module):\n    def __init__(self, margin=0.2, gamma=2):\n        super(SoftMarginFocalLoss, self).__init__()\n        self.gamma = gamma\n        self.margin = margin\n                \n        self.weight_pos = 2\n        self.weight_neg = 1\n    \n    def forward(self, inputs, targets):\n        em = np.exp(self.margin)\n        \n        log_pos = -F.logsigmoid(inputs)\n        log_neg = -F.logsigmoid(-inputs)\n        \n        log_prob = targets*log_pos + (1-targets)*log_neg\n        prob = torch.exp(-log_prob)\n        margin = torch.log(em + (1-em)*prob)\n        \n        weight = targets*self.weight_pos + (1-targets)*self.weight_neg\n        loss = self.margin + weight * (1 - prob) ** self.gamma * log_prob\n        \n        loss = loss.mean()\n        \n        return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaModel(nn.Module):\n    def __init__(self, n_meta_features):\n        super(MelanomaModel, self).__init__()\n        \n        self.encoder = pretrainedmodels.__dict__[\"se_resnext50_32x4d\"](pretrained=None)\n        self.encoder.load_state_dict(\n            torch.load(\n                \"../input/pretrained-model-weights-pytorch/se_resnext50_32x4d-a260b3a4.pth\"\n            )\n        ) \n        self.dropout = nn.Dropout(0.3)\n        self.head = nn.Linear(2048+250, 2, bias=True)\n        \n        self.n_meta_features = n_meta_features\n        \n        self.meta = nn.Sequential(OrderedDict([\n            ('meta_l1', nn.Linear(self.n_meta_features, 500, bias=True)),\n            ('meta_bn1', nn.BatchNorm1d(500)),\n            ('meta_a1', nn.ReLU()),\n            ('meta_d1', nn.Dropout(p=0.2)),\n            ('meta_l2', nn.Linear(500, 250, bias=True)),  \n            ('meta_bn2', nn.BatchNorm1d(250)),\n            ('meta_a2', nn.ReLU()),\n            ('meta_d2', nn.Dropout(p=0.2)),\n        ]))\n    \n    def forward(self, image, meta_features):\n        batch_size, _, _, _ = image.shape\n        \n        cnn_features = self.encoder.features(image)\n        cnn_features = F.adaptive_avg_pool2d(cnn_features, 1).reshape(batch_size, -1)\n        \n        meta_features = self.meta(meta_features)\n        \n        features = torch.cat((cnn_features, meta_features), dim=1)\n        logit = self.head(self.dropout(features))\n        \n        return logit","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fitter","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Fitter:\n    def __init__(self, model, device, config):\n        self.config = config\n        self.model = model\n        self.device = device\n        \n        self.epoch = 0\n        \n        self.base_dir = './'\n        self.log_path = f'{self.base_dir}/log.txt'\n        self.best_loss = float('inf')\n            \n        param_optimizer = list(model.named_parameters())\n        image_parameters = [p for n, p in param_optimizer if 'meta_' not in n]\n        meta_parameters = [p for n, p in param_optimizer if 'meta_' in n]\n    \n        self.optimizer = torch.optim.Adam([\n            \n            {'params': image_parameters, 'lr': config.lr_cnn},\n            {'params': meta_parameters, 'lr': config.lr_meta},\n            \n        ], lr=config.lr_cnn)\n    \n        self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            self.optimizer,\n            **self.config.scheduler_params\n        )\n        \n        \n        self.criterion = SoftMarginFocalLoss().to(self.device)\n        self.log(f'Fitter prepared. Training on {self.device}')\n    \n    def fit(self, train_loader, valid_loader):\n        for epoch in range(self.config.num_epochs):\n            \n            if self.config.verbose:\n                lr_cnn = self.optimizer.param_groups[0]['lr']\n                lr_meta = self.optimizer.param_groups[1]['lr']\n                timestamp = datetime.utcnow().isoformat()\n                self.log(f'\\n{timestamp}\\nLR_encoder: {lr_cnn}\\nLR_meta: {lr_meta}')\n            \n            t = time.time()\n            train_loss, train_auc = self.train_one_epoch(train_loader)\n            self.log(f'[RESULT]: Train. Epoch: {self.epoch}, ' + \\\n                     f'loss: {train_loss.avg:.5f}, auc: {train_auc.avg:.5f}, ' + \\\n                     f'time: {(time.time() - t):.5f}')\n            self.save(f'{self.base_dir}/last-checkpoint.bin')\n            \n            t = time.time()\n            val_loss, val_auc = self.validation_one_epoch(valid_loader)\n            self.log(f'[RESULT]: Val. Epoch: {self.epoch}, ' + \\\n                     f'val_loss: {val_loss.avg:.5f}, val_auc: {val_auc.avg:.5f}, ' + \\\n                     f'time: {(time.time() - t):.5f}')\n            \n            if self.config.step_scheduler:\n                self.scheduler.step(val_loss.avg)\n            \n            if val_loss.avg < self.best_loss:\n                self.best_loss = val_loss.avg\n                self.model.eval()\n                self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin'))[:-3]:\n                    os.remove(path)\n            \n            self.epoch += 1 \n    \n    def train_one_epoch(self, train_loader):\n        self.model.train()\n        \n        loss_score = AverageMeter()\n        auc_score = RocAucMeter()\n        \n        t = time.time()\n        \n        for step, data in enumerate(train_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Train Step {step}/{len(train_loader)}, ' + \\\n                        f'loss: {loss_score.avg:.5f}, auc: {auc_score.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n                \n            images = data['image']\n            meta_features = data['meta']\n            targets = data['target']\n                \n            images = images.to(self.device)\n            meta_features = meta_features.to(self.device)\n            targets = targets.to(self.device).float()\n                \n            batch_size = images.shape[0]\n            self.model.zero_grad()\n                \n            outputs = self.model(images, meta_features)\n                \n            loss = self.criterion(outputs, targets)\n            loss.backward()\n                \n            auc_score.update(targets, outputs)\n            loss_score.update(loss.detach().item(), batch_size)\n                \n            self.optimizer.step()\n        \n        return loss_score, auc_score\n\n    def validation_one_epoch(self, valid_loader):\n        self.model.eval()\n        \n        loss_score = AverageMeter()\n        auc_score = RocAucMeter()\n        \n        t = time.time()\n        \n        for step, data in enumerate(valid_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Val Step {step}/{len(valid_loader)}, ' + \\\n                        f'loss: {loss_score.avg:.5f}, auc: {auc_score.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            \n            images = data['image']\n            meta_features = data['meta']\n            targets = data['target']\n            \n            images = images.to(self.device)\n            meta_features = meta_features.to(self.device)\n            targets = targets.to(self.device).float()\n            \n            batch_size = images.shape[0]\n            \n            with torch.no_grad():\n                outputs = self.model(images, meta_features)\n                loss = self.criterion(outputs, targets)\n                auc_score.update(targets, outputs)\n                loss_score.update(loss.detach().item(), batch_size)\n        \n        return loss_score, auc_score\n    \n    def save(self, path):\n        self.model.eval()\n        \n        torch.save({\n            'model_state_dict': self.model.state_dict(),\n            'optimizer_state_dict': self.optimizer.state_dict(),\n            'scheduler_state_dict': self.scheduler.state_dict(),\n            'best_loss': self.best_loss,\n            'epoch': self.epoch,\n        }, path)\n    \n    def load(self, path):\n        checkpoint = torch.load(path)\n        self.model.load_state_dict(checkpoint['model_state_dict'])\n        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n        self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n        self.best_summary_loss = checkpoint['best_loss']\n        self.epoch = checkpoint['epoch'] + 1\n        \n    def log(self, message):\n        if self.config.verbose:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Engine","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_fold(fold):\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model = MelanomaModel(len(meta_features)).to(device)\n    \n    # Selecting fold\n    train_df = df[df['fold'] != fold].reset_index(drop=True)\n    valid_df = df[(df['fold'] == fold) & (df['source'] == 'ISIC20')].reset_index(drop=True)\n    \n    # Loading data\n    train_dataset = MelanomaDataset(\n        image_ids=train_df['image_id'],\n        targets=train_df['target'],\n        meta_features=train_df[meta_features],\n        augmentations=get_train_transforms(),\n    )\n    \n    valid_dataset = MelanomaDataset(\n        image_ids=valid_df['image_id'],\n        targets=valid_df['target'],\n        meta_features=valid_df[meta_features],\n        augmentations=get_valid_transforms(),\n    )\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        sampler=BalanceClassSampler(labels=train_dataset.get_targets(), mode=\"downsampling\"),\n        batch_size=TrainConfig.batch_size,\n        pin_memory=True,\n        drop_last=True,\n        num_workers=TrainConfig.num_workers\n    )\n    \n    valid_loader = torch.utils.data.DataLoader(\n        valid_dataset,\n        batch_size=TrainConfig.batch_size,\n        num_workers=TrainConfig.num_workers,\n        shuffle=False,\n        pin_memory=True,\n        drop_last=False,\n    )\n    \n    fitter = Fitter(\n        model=model,\n        device=device,\n        config=TrainConfig\n    )\n    \n    fitter.fit(train_loader, valid_loader)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"run_fold(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"run_fold(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"run_fold(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"run_fold(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"run_fold(4)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}