{"cells":[{"metadata":{},"cell_type":"markdown","source":"\n**V. Update**\n\nAdded TTA from ALASKA2: from [Ivan](https://www.kaggle.com/demesgal/train-inference-gpu-baseline-tta/notebook)\n\n---\n\n# Note\n\nThis Notebook is fully taken from [@shonenkov](https://www.kaggle.com/shonenkov), all credit goes to him; I've just used it to check out **GhostNet** archtecture, initialized with the official pre-trained weights. (+ also beautified code anyway :p)\n\n- [[PyTorch] GhostNet-Pretrained Weights](https://www.kaggle.com/ipythonx/ghostnetpretrained)\n\n\n---\n\n### Main ideas\n- Using External Data\n- StratifyGroupKFold\n- Focal Loss / Label Smoothing\n- BalanceClassSampler\n- SimpleAugs\n- 512x512 image size\n- GhostNet","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Dependencies","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import torch, cv2, os, time, random, cv2, sklearn, warnings, sys\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nfrom skimage import io\nimport albumentations as A\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\n\nfrom torch import nn\nfrom glob import glob\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\nwarnings.filterwarnings(\"ignore\") \nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning) \n\nSEED = 42\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# External data\n\nI have prepared kernel with merging data. Don't forget to read [this kernel](https://www.kaggle.com/shonenkov/merge-external-data) ;)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = '../input/melanoma-merged-external-data-512x512-jpeg'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# StratifyGroupKFold\n\nI think group by patient_id is very important. Also I think that stratify by sex, target, source, anatom_site_general_challenge also useful.\nCode with getting folds you can find [here](https://www.kaggle.com/shonenkov/merge-external-data)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_folds = pd.read_csv(f'{DATA_PATH}/folds.csv', index_col='image_id')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose([\n            A.RandomSizedCrop(min_max_height=(400, 400), height=512, width=512, p=0.5),\n            A.RandomRotate90(p=0.5),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.Resize(height=512, width=512, p=1),\n            A.Cutout(num_holes=8, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),\n            ToTensorV2(p=1.0),                  \n        ], p=1.0)\n\ndef get_valid_transforms():\n    return A.Compose([\n            A.Resize(height=512, width=512, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_ROOT_PATH = f'{DATA_PATH}/512x512-dataset-melanoma/512x512-dataset-melanoma'\n\ndef onehot(size, target):\n    vec = torch.zeros(size, dtype=torch.float32)\n    vec[target] = 1.\n    return vec\n\nclass DatasetRetriever(Dataset):\n\n    def __init__(self, image_ids, labels, transforms=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.labels = labels\n        self.transforms = transforms\n\n    def __getitem__(self, idx: int):\n        image_id = self.image_ids[idx]\n        image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = image.astype(np.float32) / 255.0\n\n        label = self.labels[idx]\n\n        if self.transforms:\n            sample = {'image': image}\n            sample = self.transforms(**sample)\n            image = sample['image']\n\n        target = onehot(2, label)\n        return image, target\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\n    def get_labels(self):\n        return list(self.labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Metrics","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\nclass RocAucMeter(object):\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.y_true = np.array([0,1])\n        self.y_pred = np.array([0.5,0.5])\n        self.score = 0\n\n    def update(self, y_true, y_pred):\n        y_true = y_true.cpu().numpy().argmax(axis=1).clip(min=0, max=1).astype(int)\n        # y_pred = 1 - nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:,0]\n        y_pred = nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:,1]\n        self.y_true = np.hstack((self.y_true, y_true))\n        self.y_pred = np.hstack((self.y_pred, y_pred))\n        self.score = sklearn.metrics.roc_auc_score(self.y_true, self.y_pred)\n\n    @property\n    def avg(self):\n        return self.score\n\n    \nclass APScoreMeter(RocAucMeter):\n    def __init__(self):\n        super(APScoreMeter, self).__init__()\n\n    def update(self, y_true, y_pred):\n        y_true = y_true.cpu().numpy().argmax(axis=1).clip(min=0, max=1).astype(int)\n        y_pred = nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:,1]\n        self.y_true = np.hstack((self.y_true, y_true))\n        self.y_pred = np.hstack((self.y_pred, y_pred))\n        self.score = sklearn.metrics.average_precision_score(self.y_true, self.y_pred)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, logits=False, reduce=True):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.logits = logits\n        self.reduce = reduce\n\n    def forward(self, inputs, targets):\n        if self.logits:\n            BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets)\n        else:\n            BCE_loss = F.binary_cross_entropy(inputs, targets)\n\n        pt = torch.exp(-BCE_loss)\n        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n\n        if self.reduce:\n            return torch.mean(F_loss)\n        else:\n            return F_loss\n        \nclass LabelSmoothing(nn.Module):\n    def __init__(self, smoothing = 0.1):\n        super(LabelSmoothing, self).__init__()\n        self.confidence = 1.0 - smoothing\n        self.smoothing = smoothing\n\n    def forward(self, x, target):\n        if self.training:\n            x = x.float()\n            target = target.float()\n            logprobs = torch.nn.functional.log_softmax(x, dim = -1)\n\n            nll_loss = -logprobs * target\n            nll_loss = nll_loss.sum(-1)\n    \n            smooth_loss = -logprobs.mean(dim=-1)\n\n            loss = self.confidence * nll_loss + self.smoothing * smooth_loss\n\n            return loss.mean()\n        else:\n            return torch.nn.functional.cross_entropy(x, target)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Net","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sys.path.insert(0, \"/kaggle/input/ghostnetpretrained\")\nfrom ghost_net import ghost_net\n\nnet = ghost_net().cuda()\nnet.load_state_dict(torch.load('../input/ghostnetpretrained/ghostnet_pretrained.pth'))\nnet.classifier[4] = nn.Linear(1280, 2).cuda()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fitter","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Fitter:\n    def __init__(self, model, device, config, folder):\n        self.config = config\n        self.epoch = 0\n\n        self.base_dir = f'./{folder}'\n        if not os.path.exists(self.base_dir):\n            os.makedirs(self.base_dir)\n\n        self.log_path = f'{self.base_dir}/log.txt'\n        self.best_score = 0\n        self.best_loss = 10**5\n        self.best_ap = 0\n        \n        self.model = model\n        self.device = device\n\n        param_optimizer = list(self.model.named_parameters())\n        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n        optimizer_grouped_parameters = [\n            {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n            {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n        ] \n\n        self.optimizer = torch.optim.AdamW(self.model.parameters(), lr=config.lr)\n        self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)\n\n#         self.criterion = FocalLoss(logits=True).to(self.device)\n        self.criterion = LabelSmoothing().to(self.device)\n        self.log(f'Fitter prepared. Device is {self.device}')\n\n    def fit(self, train_loader, validation_loader):\n        for e in range(self.config.n_epochs):\n            if self.config.verbose:\n                lr = self.optimizer.param_groups[0]['lr']\n                timestamp = datetime.utcnow().isoformat()\n                self.log(f'\\n{timestamp}\\nLR: {lr}')\n\n            t = time.time()\n            summary_loss, roc_auc_scores, ap_scores = self.train_one_epoch(train_loader)\n            self.log(f'[RESULT]: Train. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, roc_auc: {roc_auc_scores.avg:.5f}, ap: {ap_scores.avg:.5f}, time: {(time.time() - t):.5f}')\n\n            t = time.time()\n            summary_loss, roc_auc_scores, ap_scores = self.validation(validation_loader)\n\n            self.log(f'[RESULT]: Val. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, roc_auc: {roc_auc_scores.avg:.5f}, ap: {ap_scores.avg:.5f}, time: {(time.time() - t):.5f}')\n            if summary_loss.avg < self.best_loss:\n                self.best_loss = summary_loss.avg\n                self.save_model(f'{self.base_dir}/best-loss-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-loss-checkpoint-*epoch.bin'))[:-2]:\n                    os.remove(path)\n                    \n            if roc_auc_scores.avg > self.best_score:\n                self.best_score = roc_auc_scores.avg\n                self.save_model(f'{self.base_dir}/best-score-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-score-checkpoint-*epoch.bin'))[:-2]:\n                    os.remove(path)\n                    \n            if ap_scores.avg > self.best_ap:\n                self.best_ap = ap_scores.avg\n                self.save_model(f'{self.base_dir}/best-ap-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-ap-checkpoint-*epoch.bin'))[:-2]:\n                    os.remove(path)\n\n            if self.config.validation_scheduler:\n                self.scheduler.step(metrics=summary_loss.avg)\n\n            self.epoch += 1\n\n    def validation(self, val_loader):\n        self.model.eval()\n        summary_loss = AverageMeter()\n        roc_auc_scores = RocAucMeter()\n        ap_scores = APScoreMeter()\n        t = time.time()\n        for step, (images, targets) in enumerate(val_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Val Step {step}/{len(val_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, roc_auc: {roc_auc_scores.avg:.5f}, ap: {ap_scores.avg:.5f} ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            with torch.no_grad():\n                targets = targets.to(self.device).float()\n                batch_size = images.shape[0]\n                images = images.to(self.device).float()\n                outputs = self.model(images)\n                loss = self.criterion(outputs, targets)\n                roc_auc_scores.update(targets, outputs)\n                ap_scores.update(targets, outputs)\n                summary_loss.update(loss.detach().item(), batch_size)\n\n        return summary_loss, roc_auc_scores, ap_scores\n\n    def train_one_epoch(self, train_loader):\n        self.model.train()\n        summary_loss = AverageMeter()\n        roc_auc_scores = RocAucMeter()\n        ap_scores = APScoreMeter()\n        t = time.time()\n        for step, (images, targets) in enumerate(train_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Train Step {step}/{len(train_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, roc_auc: {roc_auc_scores.avg:.5f}, ap: {ap_scores.avg:.5f} ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            \n            targets = targets.to(self.device).float()\n            images = images.to(self.device).float()\n            batch_size = images.shape[0]\n\n            self.optimizer.zero_grad()\n            outputs = self.model(images)\n            loss = self.criterion(outputs, targets)\n            loss.backward()\n            \n            roc_auc_scores.update(targets, outputs)\n            ap_scores.update(targets, outputs)\n            summary_loss.update(loss.detach().item(), batch_size)\n\n            self.optimizer.step()\n\n            if self.config.step_scheduler:\n                self.scheduler.step()\n\n        return summary_loss, roc_auc_scores, ap_scores\n    \n    def save_model(self, path):\n        self.model.eval()\n        torch.save(self.model.state_dict(),path)\n\n    def save(self, path):\n        self.model.eval()\n        torch.save({\n            'model_state_dict': self.model.state_dict(),\n            'optimizer_state_dict': self.optimizer.state_dict(),\n            'scheduler_state_dict': self.scheduler.state_dict(),\n            'best_score': self.best_score,\n            'best_ap': self.best_ap,\n            'best_loss': self.best_loss,\n            'epoch': self.epoch,\n        }, path)\n\n    def load(self, path):\n        checkpoint = torch.load(path)\n        self.model.load_state_dict(checkpoint['model_state_dict'])\n        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n        self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n        self.best_score = checkpoint['best_score']\n        self.best_ap = checkpoint['best_ap']\n        self.best_loss = checkpoint['best_loss']\n        self.epoch = checkpoint['epoch']\n        \n    def log(self, message):\n        if self.config.verbose:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainGlobalConfig:\n    num_workers = 2\n    batch_size = 10\n    n_epochs = 15\n    lr = 0.00003\n\n    # -------------------\n    verbose = True\n    verbose_step = 1\n    # -------------------\n\n    # --------------------\n    step_scheduler = False  # do scheduler.step after optimizer.step\n    validation_scheduler = True  # do scheduler.step after validation stage loss\n\n    \n    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n    scheduler_params = dict(\n        mode='min',\n        factor=0.8,\n        patience=1,\n        verbose=False, \n        threshold=0.0001,\n        threshold_mode='abs',\n        cooldown=0, \n        min_lr=1e-8,\n        eps=1e-08\n    )\n    # --------------------","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Save all states for \"honest\" training of folds","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fitter = Fitter(model=net, device=torch.device('cuda:0'), config=TrainGlobalConfig, folder='base_state')\nBASE_STATE_PATH = f'{fitter.base_dir}/base_state.bin'\nfitter.save(BASE_STATE_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from catalyst.data.sampler import BalanceClassSampler\n\ndef train_fold(fold_number):\n    train_dataset = DatasetRetriever(\n        image_ids=df_folds[df_folds['fold'] != fold_number].index.values,\n        labels=df_folds[df_folds['fold'] != fold_number].target.values,\n        transforms=get_train_transforms(),\n    )\n\n    df_val = df_folds[(df_folds['fold'] == fold_number) & (df_folds['source'] == 'ISIC20')]\n\n    validation_dataset = DatasetRetriever(\n        image_ids=df_val.index.values,\n        labels=df_val.target.values,\n        transforms=get_valid_transforms(),\n    )\n\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        sampler=BalanceClassSampler(labels=train_dataset.get_labels(), mode=\"downsampling\"),\n        batch_size=TrainGlobalConfig.batch_size,\n        pin_memory=False,\n        drop_last=True,\n        num_workers=TrainGlobalConfig.num_workers,\n    )\n    val_loader = torch.utils.data.DataLoader(\n        validation_dataset, \n        batch_size=TrainGlobalConfig.batch_size,\n        num_workers=TrainGlobalConfig.num_workers,\n        shuffle=False,\n        sampler=SequentialSampler(validation_dataset),\n        pin_memory=False,\n    )\n\n    fitter = Fitter(model=net, device=torch.device('cuda:0'), config=TrainGlobalConfig, folder=f'fold{fold_number}')\n    fitter.load(BASE_STATE_PATH)\n    fitter.fit(train_loader, val_loader)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training\n\nSharing the training logs.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# for fold_number in range(1): # range(5)\n#     train_fold(fold_number=fold_number)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"file = open('../input/melanomaghostnet/log.txt', 'r')\nfor line in file.readlines():\n    print(line[:-1])\nfile.close()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inference and Test Time Augmentation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint_path = '../input/melanomaghostnet/best-ap-checkpoint-045epoch.bin'\ncheckpoint = torch.load(checkpoint_path)\nnet.load_state_dict(checkpoint);\nnet.eval();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_transforms(mode):\n    if mode == 0:\n        return A.Compose([\n                A.Resize(height=512, width=512, p=1.0),\n                ToTensorV2(p=1.0),\n            ], p=1.0)\n    elif mode == 1:\n        return A.Compose([\n                A.HorizontalFlip(p=1),\n                A.Resize(height=512, width=512, p=1.0),\n                ToTensorV2(p=1.0),\n            ], p=1.0)    \n    elif mode == 2:\n        return A.Compose([\n                A.VerticalFlip(p=1),\n                A.Resize(height=512, width=512, p=1.0),\n                ToTensorV2(p=1.0),\n            ], p=1.0)\n    else:\n        return A.Compose([\n                A.HorizontalFlip(p=1),\n                A.VerticalFlip(p=1),\n                A.Resize(height=512, width=512, p=1.0),\n                ToTensorV2(p=1.0),\n            ], p=1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_ROOT_PATH = f'{DATA_PATH}/512x512-test/512x512-test'\n\nclass DatasetRetriever(Dataset):\n    def __init__(self, image_ids, transforms=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.transforms = transforms\n\n    def __getitem__(self, idx: int):\n        image_id = self.image_ids[idx]\n        image = cv2.imread(f'{TEST_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = image.astype(np.float32) / 255.0\n        if self.transforms:\n            sample = {'image': image}\n            sample = self.transforms(**sample)\n            image = sample['image']\n        return image_id, image\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv(f'../input/siim-isic-melanoma-classification/test.csv', index_col='image_name')\nresults = []\n\nfor mode in range(0, 4):\n    test_dataset = DatasetRetriever(\n        image_ids=df_test.index.values,\n        transforms=get_test_transforms(mode),\n    )\n\n    test_loader = torch.utils.data.DataLoader(\n        test_dataset, \n        batch_size=8,\n        num_workers=2,\n        shuffle=False,\n        drop_last=False,\n    )\n    \n    result = {'image_name': [], 'target': []}\n    for step, (image_names, images) in enumerate(test_loader):\n        print(step, end='\\r')\n        y_pred = net(images.cuda())\n        y_pred = nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:,1]\n\n        result['image_name'].extend(image_names)\n        result['target'].extend(y_pred)\n        \n    results.append(result)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submissions = []\nfor mode in range(4):\n    submission = pd.DataFrame(results[mode])\n    submissions.append(submission)\n    \nfor mode in range(4):\n    submissions[mode].to_csv(f'submission_{mode}.csv', index=False)\n    \nsubmissions[0]['target'] = (submissions[0]['target']*3 + submissions[1]['target'] + \n                           submissions[2]['target'] + submissions[3]['target'])/6\nsubmissions[0].to_csv(f'submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submissions[0].head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}