{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":25954,"databundleVersionId":2091745,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":13757335,"sourceType":"datasetVersion","datasetId":8754473},{"sourceId":13757349,"sourceType":"datasetVersion","datasetId":8754483},{"sourceId":13757358,"sourceType":"datasetVersion","datasetId":8754488},{"sourceId":13757400,"sourceType":"datasetVersion","datasetId":8754515}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\nimport torch\nfrom torch import nn, optim\nfrom  torch.utils.data import Dataset, DataLoader\nimport torchvision.models as models\nfrom matplotlib import pyplot as plt\nimport os, random, gc\nimport json\nfrom  ast import literal_eval\nfrom sklearn.metrics import label_ranking_average_precision_score\nfrom tqdm.notebook import tqdm\nimport joblib\nimport seaborn as sns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-17T03:41:56.255969Z","iopub.execute_input":"2025-11-17T03:41:56.256279Z","iopub.status.idle":"2025-11-17T03:42:06.987505Z","shell.execute_reply.started":"2025-11-17T03:41:56.256254Z","shell.execute_reply":"2025-11-17T03:42:06.986551Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-17T03:42:13.592327Z","iopub.execute_input":"2025-11-17T03:42:13.592890Z","iopub.status.idle":"2025-11-17T03:42:13.605237Z","shell.execute_reply.started":"2025-11-17T03:42:13.592867Z","shell.execute_reply":"2025-11-17T03:42:13.603916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/birdclef-2021/train_metadata.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-17T03:42:25.448126Z","iopub.execute_input":"2025-11-17T03:42:25.448472Z","iopub.status.idle":"2025-11-17T03:42:26.012856Z","shell.execute_reply.started":"2025-11-17T03:42:25.448447Z","shell.execute_reply":"2025-11-17T03:42:26.012047Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels = df['primary_label']\nlabel_counts = train_labels.value_counts()\ntop_100_counts = label_counts[:100]\n\nplt.figure(figsize=(15, 7))\nsns.barplot(x=top_100_counts.index, y=top_100_counts.values)\nplt.xticks(rotation=90)\nplt.xlabel(\"Метка класса\")\nplt.ylabel(\"Количество аудио\")\nplt.title(\"Топ 100 классов по количеству данных в обучающем наборе\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-17T03:42:28.044235Z","iopub.execute_input":"2025-11-17T03:42:28.044595Z","iopub.status.idle":"2025-11-17T03:42:29.252434Z","shell.execute_reply.started":"2025-11-17T03:42:28.044541Z","shell.execute_reply":"2025-11-17T03:42:29.251417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"NUM_CLASSES = 397\nSR = 32000\nDURATION = 7\nMAX_READ_SAMPLES = 5\nDATA_ROOT = Path('/kaggle/input/birdclef-2021')\nMEL_PATHS = sorted(Path('/kaggle/input').glob('mels-birds-train*/rich_train_metadata.csv'))\nTRAIN_LABEL_PATHS = sorted(Path('/kaggle/input').glob('mels-birds-train*/LABEL_IDS.json'))\nMODEL_ROOT = Path('.')\nTRAIN_BATCH_SIZE = 100\nTRAIN_NUM_WORKERS = 2\nVAL_BATCH_SIZE = 128\nVAL_NUM_WORKERS = 2\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('Device:', DEVICE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T22:23:00.970635Z","iopub.execute_input":"2025-11-16T22:23:00.971189Z","iopub.status.idle":"2025-11-16T22:23:01.096200Z","shell.execute_reply.started":"2025-11-16T22:23:00.971166Z","shell.execute_reply":"2025-11-16T22:23:01.095573Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_df(mel_paths=MEL_PATHS, train_label_paths=TRAIN_LABEL_PATHS):\n  df_list = []\n  LABEL_IDS = {}\n    \n  for file_path in mel_paths:\n    temp = pd.read_csv(str(file_path), index_col=0)\n    temp['impath'] = temp.apply(\n        lambda row: file_path.parent/'audio_images/{}/{}.npy'.format(row.primary_label, row.filename), \n        axis=1\n    ) \n    df_list.append(temp)\n\n  df = pd.concat(df_list, ignore_index=True)\n  df['secondary_labels'] = df['secondary_labels'].apply(literal_eval)\n\n  for file_path in train_label_paths:\n    with open(str(file_path)) as f:\n      LABEL_IDS.update(json.load(f))\n\n  return LABEL_IDS, df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T22:23:02.691328Z","iopub.execute_input":"2025-11-16T22:23:02.691794Z","iopub.status.idle":"2025-11-16T22:23:02.697082Z","shell.execute_reply.started":"2025-11-16T22:23:02.691768Z","shell.execute_reply":"2025-11-16T22:23:02.696522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_efficientnet_model(name, num_classes=NUM_CLASSES):\n    if 'efficientnet' in name:\n        model = models.efficientnet_b2(weights=models.EfficientNet_B2_Weights.DEFAULT)\n    else:\n        raise RuntimeError('Незнакомая модель')\n    if hasattr(model, 'fc'):\n        nb_ft = model.fc.in_features\n        model.fc = nn.Linear(nb_ft, num_classes)\n    elif hasattr(model, '_fc'): \n        nb_ft = model._fc.in_features\n        model._fc = nn.Linear(nb_ft, num_classes)\n    elif hasattr(model, 'classifier'):\n        if isinstance(model.classifier, nn.Sequential):\n            for layer in reversed(model.classifier):\n                if hasattr(layer, 'in_features'):\n                    nb_ft = layer.in_features\n                    break\n            model.classifier = nn.Linear(nb_ft, num_classes)\n        else:\n            nb_ft = model.classifier.in_features\n            model.classifier = nn.Linear(nb_ft, num_classes)\n    elif hasattr(model, 'last_linear'):\n        nb_ft = model.last_linear.in_features\n        model.last_linear = nn.Linear(nb_ft, num_classes)\n\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T22:23:10.662241Z","iopub.execute_input":"2025-11-16T22:23:10.662533Z","iopub.status.idle":"2025-11-16T22:23:10.668878Z","shell.execute_reply.started":"2025-11-16T22:23:10.662514Z","shell.execute_reply":"2025-11-16T22:23:10.668332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_data(df):\n    def load_row(row):\n        return row.filename, np.load(str(row.impath))[:MAX_READ_SAMPLES]\n\n    pool = joblib.Parallel(4)\n    mapper = joblib.delayed(load_row)\n    tasks = [mapper(row) for row in df.itertuples(False)]\n    res = pool(tqdm(tasks))\n    res = dict(res)\n    return res","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T22:23:13.369947Z","iopub.execute_input":"2025-11-16T22:23:13.370636Z","iopub.status.idle":"2025-11-16T22:23:13.374987Z","shell.execute_reply.started":"2025-11-16T22:23:13.370610Z","shell.execute_reply":"2025-11-16T22:23:13.374342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BirdClefDataset(Dataset):\n    def __init__(self, audio_image_store, meta, sr=SR, is_train=True, num_classes=NUM_CLASSES, duration=DURATION):\n        self.audio_image_store = audio_image_store\n        self.meta = meta.copy().reset_index(drop=True)\n        self.sr = sr\n        self.is_train = is_train\n        self.num_classes = num_classes\n        self.duration = duration\n        self.audio_length = self.duration*self.sr\n    \n    @staticmethod\n    def normalize(image):\n        image = image.astype(\"float32\", copy=False) / 255.0\n        image = np.stack([image, image, image])\n        return image\n\n    def __len__(self):\n        return len(self.meta)\n    \n    def __getitem__(self, idx):\n        row = self.meta.iloc[idx]\n        image = self.audio_image_store[row.filename]\n\n        image = image[np.random.choice(len(image))]\n        image = self.normalize(image)\n        \n        t = np.zeros(self.num_classes, dtype=np.float32) + 0.0025\n        t[row.label_id] = 0.995\n        \n        return image, t","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T22:23:14.934838Z","iopub.execute_input":"2025-11-16T22:23:14.935525Z","iopub.status.idle":"2025-11-16T22:23:14.941539Z","shell.execute_reply.started":"2025-11-16T22:23:14.935500Z","shell.execute_reply":"2025-11-16T22:23:14.940793Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@torch.no_grad()\ndef evaluate(net, criterion, val_laoder):\n    net.eval()\n    os, y = [], []\n    val_laoder = tqdm(val_laoder, leave = False, total=len(val_laoder))\n\n    for icount, (xb, yb) in  enumerate(val_laoder):\n        y.append(yb.to(DEVICE))\n        xb = xb.to(DEVICE)\n        o = net(xb)\n        os.append(o)\n\n    y = torch.cat(y)\n    o = torch.cat(os)\n\n    l = criterion(o, y).item()\n    \n    o = o.sigmoid()\n    y = (y > 0.5)*1.0\n\n    lrap = label_ranking_average_precision_score(y.cpu().numpy(), o.cpu().numpy())\n\n    o = (o > 0.5)*1.0\n\n    prec = ((o*y).sum()/(1e-6 + o.sum())).item()\n    rec = ((o*y).sum()/(1e-6 + y.sum())).item()\n    f1 = 2*prec*rec/(1e-6+prec+rec)\n\n    return l, lrap, f1, rec, prec","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AutoSave:\n    def __init__(self, top_k=3, metric=\"f1_val\", mode=\"max\", root=None, name=\"ckpt\", save_best=True):\n        self.top_k = top_k\n        self.logs = []\n        self.metric = metric\n        self.mode = mode\n        self.root = Path(root or MODEL_ROOT)\n        assert self.root.exists()\n        self.name = name\n        self.save_best = save_best\n\n        self.top_models = []\n        self.top_metrics = []\n        self.best_metric = -float('inf') if mode == \"max\" else float('inf')\n        self.best_epoch = -1\n\n    def log(self, model, metrics):\n        metric = metrics[self.metric]\n        rank = self.rank(metric)\n\n        self.top_metrics.insert(rank+1, metric)\n        if len(self.top_metrics) > self.top_k:\n            self.top_metrics.pop(0)\n\n        self.logs.append(metrics)\n        \n        # Сохраняем в топ-K\n        self.save(model, metric, rank, metrics[\"epoch\"])\n        \n        # Отдельно сохраняем лучшую модель\n        if self.save_best:\n            self.save_best_model(model, metric, metrics[\"epoch\"])\n\n    def save_best_model(self, model, metric, epoch):\n        \"\"\"Сохраняет лучшую модель отдельно\"\"\"\n        is_better = (self.mode == \"max\" and metric > self.best_metric) or \\\n                   (self.mode == \"min\" and metric < self.best_metric)\n        \n        if is_better:\n            self.best_metric = metric\n            self.best_epoch = epoch\n            \n            # Удаляем предыдущую лучшую модель\n            best_pattern = f\"{self.name}_best_*.pth\"\n            for old_best in self.root.glob(best_pattern):\n                old_best.unlink()\n            \n            # Сохраняем новую лучшую (без лишних символов)\n            best_name = f\"{self.name}_best_epoch{epoch:02d}_{metric:.4f}.pth\"\n            best_path = self.root / best_name\n            \n            torch.save({\n                'model_state_dict': model.state_dict(),\n                'epoch': epoch,\n                'metric': metric,\n                'metric_name': self.metric,\n                'logs': self.logs\n            }, best_path.as_posix())\n            \n            print(f\"🏆 NEW BEST! Epoch {epoch}, {self.metric}: {metric:.4f}\")\n            print(f\"💾 Saved: {best_name}\")\n\n    def save(self, model, metric, rank, epoch):\n        \"\"\"Сохраняет модель в топ-K\"\"\"\n        # Более простое имя файла\n        name = f\"{self.name}_epoch{epoch:02d}_{metric:.4f}.pth\"\n        path = self.root / name\n\n        old_model = None\n        self.top_models.insert(rank+1, name)\n        if len(self.top_models) > self.top_k:\n            old_model = self.root / self.top_models[0]\n            self.top_models.pop(0)      \n\n        torch.save({\n            'model_state_dict': model.state_dict(),\n            'epoch': epoch,\n            'metric': metric,\n            'metric_name': self.metric\n        }, path.as_posix())\n\n        if old_model and old_model.exists():\n            old_model.unlink()\n            print(f\"🗑️ Removed: {old_model.name}\")\n\n        self.to_json()\n\n    def rank(self, val):\n        if self.mode == \"max\":\n            for i, top_val in enumerate(self.top_metrics):\n                if val <= top_val:\n                    return i - 1\n            return len(self.top_metrics) - 1\n        else:\n            for i, top_val in enumerate(self.top_metrics):\n                if val >= top_val:\n                    return i - 1\n            return len(self.top_metrics) - 1\n\n    def to_json(self):\n        log_name = f\"{self.name}_logs.json\"\n        log_path = self.root / log_name\n        with log_path.open(\"w\") as f:\n            json.dump(self.logs, f, indent=2)\n\n    def get_best_model_info(self):\n        if self.best_epoch == -1:\n            return None\n        return {\n            'epoch': self.best_epoch,\n            'metric': self.best_metric,\n            'metric_name': self.metric\n        }","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def one_fold(model_name, fold, train_set, val_set, epochs=50, save=True, save_root=None, patience=7):\n    save_root = Path(save_root) or MODEL_ROOT\n    saver = AutoSave(\n        root=save_root, \n        name=f\"birdclef_{model_name}_fold{fold}\", \n        metric=\"f1_val\",\n        mode=\"max\",\n        top_k=3,\n        save_best=True\n    )\n    \n    net = get_efficientnet_model(model_name).to(DEVICE)\n    criterion = nn.BCEWithLogitsLoss()\n    optimizer = optim.AdamW(net.parameters(), lr=1e-3, weight_decay=1e-4)\n    scheduler_cosine = optim.lr_scheduler.CosineAnnealingLR(optimizer, eta_min=1e-6, T_max=epochs)\n    scheduler_plateau = optim.lr_scheduler.ReduceLROnPlateau(\n        optimizer, mode='max', factor=0.5, patience=1, min_lr=1e-6, threshold=0.002, threshold_mode='rel'\n    )\n    \n    train_data = BirdClefDataset(audio_image_store, meta=df.iloc[train_set].reset_index(drop=True),\n                             sr=SR, duration=DURATION, is_train=True)\n    train_loader = DataLoader(train_data, batch_size=TRAIN_BATCH_SIZE, num_workers=TRAIN_NUM_WORKERS, \n                            shuffle=True, pin_memory=True, drop_last=True)\n    \n    val_data = BirdClefDataset(audio_image_store, meta=df.iloc[val_set].reset_index(drop=True),  \n                             sr=SR, duration=DURATION, is_train=False)\n    val_loader = DataLoader(val_data, batch_size=VAL_BATCH_SIZE, num_workers=VAL_NUM_WORKERS, shuffle=False)\n    \n    best_f1 = 0\n    patience_counter = 0\n    \n    for epoch in range(epochs):\n        print(f\"\\n--> [EPOCH {epoch:02d}]\")\n        net.train()\n\n        (l, l_val), (lrap, lrap_val), (f1, f1_val), (rec, rec_val), (prec, prec_val) = one_epoch(\n            net=net,\n            criterion=criterion,\n            optimizer=optimizer,\n            train_loader=train_loader,\n            val_loader=val_loader\n        )\n\n        scheduler_plateau.step(f1_val)\n        scheduler_cosine.step()\n        \n        current_lr = optimizer.param_groups[0]['lr']\n        print(\n            \"[{epoch:02d}] loss: {loss} lrap: {lrap} f1: {f1} rec: {rec} prec: {prec} lr: {lr:.2e}\".format(\n                epoch=epoch,\n                loss=\"({:.4f}, {:.4f})\".format(l, l_val),\n                prec=\"({:.3f}, {:.3f})\".format(prec, prec_val),\n                rec=\"({:.3f}, {:.3f})\".format(rec, rec_val),\n                f1=\"({:.3f}, {:.3f})\".format(f1, f1_val),\n                lrap=\"({:.3f}, {:.3f})\".format(lrap, lrap_val),\n                lr=current_lr\n            )\n        )\n\n        if save:\n            metrics = {\n                \"loss\": l, \"lrap\": lrap, \"f1\": f1, \"rec\": rec, \"prec\": prec,\n                \"loss_val\": l_val, \"lrap_val\": lrap_val, \"f1_val\": f1_val, \"rec_val\": rec_val, \"prec_val\": prec_val,\n                \"epoch\": epoch, \"lr\": current_lr\n            }\n            saver.log(net, metrics)\n\n        # Ранняя остановка\n        if f1_val > best_f1:\n            best_f1 = f1_val\n            patience_counter = 0\n            print(f\"🎯 New best F1: {best_f1:.4f}\")\n        else:\n            patience_counter += 1\n            print(f\"⏳ No improvement: {patience_counter}/{patience}\")\n            \n        if patience_counter >= patience:\n            print(f\"🛑 Early stopping at epoch {epoch}\")\n            best_info = saver.get_best_model_info()\n            if best_info:\n                print(f\"🏆 Best model: epoch {best_info['epoch']}, {best_info['metric_name']}: {best_info['metric']:.4f}\")\n            break\n    \n    best_info = saver.get_best_model_info()\n    if best_info:\n        print(f\"\\n🎉 Training finished! Best model: epoch {best_info['epoch']}, {best_info['metric_name']}: {best_info['metric']:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def one_epoch(net, criterion, optimizer, train_loader, val_loader):\n    net.train()\n    l, lrap, prec, rec, f1, icount = 0., 0., 0., 0., 0., 0\n    train_loader_tqdm = tqdm(train_loader, leave=False)\n    epoch_bar = train_loader_tqdm\n    \n    for (xb, yb) in epoch_bar:\n        _l, _lrap, _f1, _rec, _prec = one_step(xb, yb, net, criterion, optimizer)\n        l += _l\n        lrap += _lrap\n        f1 += _f1\n        rec += _rec\n        prec += _prec\n        icount += 1\n            \n        if hasattr(epoch_bar, \"set_postfix\") and not icount % 10:\n            epoch_bar.set_postfix(\n                loss=\"{:.6f}\".format(l/icount),\n                lrap=\"{:.3f}\".format(lrap/icount),\n                prec=\"{:.3f}\".format(prec/icount),\n                rec=\"{:.3f}\".format(rec/icount),\n                f1=\"{:.3f}\".format(f1/icount),\n            )\n    \n    l /= icount\n    lrap /= icount\n    f1 /= icount\n    rec /= icount\n    prec /= icount\n    \n    l_val, lrap_val, f1_val, rec_val, prec_val = evaluate(net, criterion, val_loader)\n    \n    return (l, l_val), (lrap, lrap_val), (f1, f1_val), (rec, rec_val), (prec, prec_val)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train(model_name, epochs=20, save=True, n_splits=5, seed=177, save_root=None, suffix=\"\", folds=None):\n  gc.collect()\n  torch.cuda.empty_cache()\n\n  save_root = save_root or MODEL_ROOT/f\"{model_name}{suffix}\"\n  save_root.mkdir(exist_ok=True, parents=True)\n  \n  fold_bar = tqdm(df.reset_index().groupby(\"fold\").index.apply(list).items(), total=df.fold.max()+1)\n  \n  for fold, val_set in fold_bar:\n      if folds and not fold in folds:\n        continue\n      \n      print(f\"\\n [FOLD {fold}]\")\n      fold_bar.set_description(f\"[FOLD {fold}]\")\n      train_set = np.setdiff1d(df.index, val_set)\n        \n      one_fold(model_name, fold=fold, train_set=train_set , val_set=val_set , epochs=epochs, save=save, save_root=save_root)\n    \n      gc.collect()\n      torch.cuda.empty_cache()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"LABEL_IDS, df = get_df()\naudio_image_store = load_data(df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T22:23:17.752455Z","iopub.execute_input":"2025-11-16T22:23:17.752715Z","iopub.status.idle":"2025-11-16T22:26:32.287034Z","shell.execute_reply.started":"2025-11-16T22:23:17.752695Z","shell.execute_reply":"2025-11-16T22:26:32.286192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ds = BirdClefDataset(audio_image_store, meta=df, sr=SR, duration=DURATION, is_train=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T22:26:41.634719Z","iopub.execute_input":"2025-11-16T22:26:41.634995Z","iopub.status.idle":"2025-11-16T22:26:41.699492Z","shell.execute_reply.started":"2025-11-16T22:26:41.634974Z","shell.execute_reply":"2025-11-16T22:26:41.698857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x, y = ds[np.random.choice(len(ds))]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T22:26:45.324103Z","iopub.execute_input":"2025-11-16T22:26:45.324365Z","iopub.status.idle":"2025-11-16T22:26:45.340163Z","shell.execute_reply.started":"2025-11-16T22:26:45.324347Z","shell.execute_reply":"2025-11-16T22:26:45.339398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"try:\n    train('efficientnet', epochs=30, suffix=f\"_sr{SR}_d{DURATION}_v1_v1\", folds=[3])\nexcept Exception as e:\n    raise ValueError() from  e","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-17T00:18:51.275039Z","iopub.execute_input":"2025-11-17T00:18:51.275330Z","iopub.status.idle":"2025-11-17T02:13:50.837961Z","shell.execute_reply.started":"2025-11-17T00:18:51.275295Z","shell.execute_reply":"2025-11-17T02:13:50.837070Z"}},"outputs":[],"execution_count":null}]}