{"cells":[{"metadata":{"trusted":true,"_uuid":"a7fe0e6e0364e1e7e0570b0563dc5be69632430a"},"cell_type":"code","source":"FOLD = 0\n\nimport os\nimport random\nimport time\nimport math\nimport requests\nimport glob\nimport gc\n\nimport ast\n\nimport numpy as np\nimport pandas as pd\n\nimport mlcrate as mlc\n\nimport os\n\nimport cv2\n\nfrom sklearn.model_selection import train_test_split, KFold, StratifiedKFold\nfrom sklearn.metrics import fbeta_score, accuracy_score, precision_score, recall_score, f1_score\n\nfrom skimage.transform import resize\n\nfrom PIL import Image, ImageDraw\n\nfrom tqdm import tqdm, tqdm_notebook\n\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch import optim\nfrom torch.optim import Optimizer\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.utils.checkpoint as checkpoint\n\nimport torchvision\nfrom torchvision import transforms, utils\n\nimport torchtext\nimport torchtext.data as data\n\nfrom torch.nn.utils.rnn import pad_sequence\n\nimport spacy\nfrom spacy.lang.en import English\n\nSEED = 1337\n\nNOTIFY_EACH_EPOCH = False\n\nWORKERS = 0\nBATCH_SIZE = 512\n\nN_SPLITS = 10\n\nrandom.seed(SEED)\nos.environ['PYTHONHASHSEED'] = str(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed(SEED)\ntorch.backends.cudnn.deterministic = True\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\ndef get_n_params(model):\n    pp=0\n    for p in list(model.parameters()):\n        nn=1\n        for s in list(p.size()):\n            nn = nn*s\n        pp += nn\n    return pp\n\n# from https://github.com/floydhub/save-and-resume\ndef save_checkpoint(state):\n    \"\"\"Save checkpoint if a new best is achieved\"\"\"\n    print (\" Saving checkpoint\")\n\n    filename = f'./checkpoint-{state[\"epoch\"]}.pt.tar'\n    torch.save(state, filename)\n    \ndef initialize(model, path=None, optimizer=None):   \n    if path == None:\n        checkpoints = glob.glob('./*.pt.tar')\n        path = checkpoints[np.argmax([int(checkpoint.split('checkpoint-')[1].split('.')[0]) for checkpoint in checkpoints])]\n    \n    checkpoint = torch.load(path)\n\n    model.load_state_dict(checkpoint['model'])\n\n    print(f' Loaded checkpoint {path} | Trained for {checkpoint[\"epoch\"] + 1} epochs')\n    \n    if optimizer:\n        optimizer.load_state_dict(checkpoint['optimizer'])\n          \n        epoch = checkpoint['epoch'] + 1\n        train_iteration = checkpoint['train_iteration']\n        val_iteration = checkpoint['val_iteration']\n\n        return model, optimizer, epoch, train_iteration, val_iteration\n    else:\n        return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"321593e69c05e023da01c4019576ba09e34cafd1"},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/sample_submission.csv')\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3aa1be8c22876fb7e946a6255bd9c89ef19d92be"},"cell_type":"code","source":"# From https://www.kaggle.com/spirosrap/bilstm-attention-kfold-clr-extra-features-capsule/notebook\ndef add_features(df):\n    df2 = df.copy(deep=True)\n    df2['question_text'] = df2['question_text'].apply(lambda x:str(x))\n    df2['total_length'] = df2['question_text'].apply(len)\n    df2['capitals'] = df2['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))\n    df2['caps_vs_length'] = df2.apply(lambda row: float(row['capitals'])/float(row['total_length']), axis=1)\n    df2['num_words'] = df2.question_text.str.count('\\S+')\n    df2['num_unique_words'] = df2['question_text'].apply(lambda comment: len(set(w for w in comment.split())))\n    df2['words_vs_unique'] = df2['num_unique_words'] / df2['num_words']  \n\n    return np.concatenate((df2['caps_vs_length'].values.reshape(-1, 1), df2['words_vs_unique'].values.reshape(-1, 1)), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4e3d795b5e36576dc98d5319faae7887b4a4d26","scrolled":false},"cell_type":"code","source":"kfold = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\n\ntrain_idx, val_idx = list(kfold.split(train))[FOLD]\nx_train, x_val = train.iloc[train_idx], train.iloc[val_idx]\n# x_train, x_val = train.loc[:1000], train.loc[1000:2000]\n\ntest = x_train.iloc[:56000+376000]\nx_train = x_train.iloc[56000+376000:]\ntest_1 = test.iloc[:56000]\ntest_2 = test.iloc[56000:]\n\nx_train_meta = add_features(x_train)\nx_val_meta = add_features(x_val)\ntest_1_meta = add_features(test_1)\ntest_2_meta = add_features(test_2)\n\nx_train = x_train.reset_index()\nx_val = x_val.reset_index()\nx_test_1 = test_1.reset_index()\nx_test_2 = test_2.reset_index()\n\nx_train.to_csv('train.csv')\nx_val.to_csv('val.csv')\nx_test_1.to_csv('test_1.csv')\nx_test_2.to_csv('test_2.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eaa2946399f83ea0f77eafd2e02991b283eddab3"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\','•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c7e4854b60b1ff433abe2c2082542f58acb43a5"},"cell_type":"code","source":"nlp = English()\ndef tokenize(sentence):\n#     sentence = sentence.split()\n#     return sentence\n\n    sentence = str(sentence)\n    for punct in puncts:\n        sentence = sentence.replace(punct, f' {punct} ')\n        \n    x = nlp(sentence)\n    return [token.text for token in x]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d52cc97f05e431151f7f488fa39a3ae014610b88","scrolled":false,"_kg_hide-output":false,"_kg_hide-input":false},"cell_type":"code","source":"%%time\n# from http://mlexplained.com/2018/02/08/a-comprehensive-tutorial-to-torchtext\nindex_field = data.Field(sequential=False, use_vocab=False, batch_first=True)\nquestion_field = data.Field(tokenize=tokenize, lower=True, batch_first=True, include_lengths=True)\ntarget_field = data.Field(sequential=False, use_vocab=False, batch_first=True)\n\ntrain_fields = [\n    ('id', index_field),\n    ('index', None),\n    ('qid', None),\n    ('question_text', question_field),\n    ('target', target_field)\n]\n\ntest_fields = [\n    ('id', index_field),\n    ('index', None),\n    ('qid', None),\n    ('question_text', question_field)\n]\n\ntrain_dataset, val_dataset = data.TabularDataset.splits('./', train='train.csv', validation='val.csv', format='CSV', skip_header=True, fields=train_fields)\ntest_1_dataset = data.TabularDataset('./test_1.csv', format='CSV', skip_header=True, fields=train_fields)\ntest_2_dataset = data.TabularDataset('./test_2.csv', format='CSV', skip_header=True, fields=train_fields)\n\nquestion_field.build_vocab(train_dataset, val_dataset, max_size=95000)\n\ntrain_dataloader, val_dataloader = data.BucketIterator.splits((train_dataset, val_dataset), (BATCH_SIZE, BATCH_SIZE), sort_key=lambda x: len(x.question_text), sort_within_batch=True)\n# test_1_dataloader = data.BucketIterator(test_1_dataset, 512, sort_key=lambda x: len(x.question_text), sort_within_batch=True)\n# test_2_dataloader = data.BucketIterator(test_2_dataset, 512, sort_key=lambda x: len(x.question_text), sort_within_batch=True)\ntest_1_dataloader = data.BucketIterator(test_1_dataset, 512, sort=False, shuffle=False)\ntest_2_dataloader = data.BucketIterator(test_2_dataset, 512, sort=False, shuffle=False)\n\nprint(f'Train Dataset: {len(train_dataset)}')\nprint(f'Val Dataset: {len(val_dataset)}')\nprint(f'Test 1 Dataset: {len(test_1_dataset)}')\nprint(f'Test 2 Dataset: {len(test_2_dataset)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dda7d32355775e045d56ae70f793e55bacbc0651"},"cell_type":"code","source":"# from https://discuss.pytorch.org/t/self-attention-on-words-and-masking/5671/4\nclass SelfAttention(nn.Module):\n    def __init__(self, hidden_size, batch_first=False):\n        super(SelfAttention, self).__init__()\n\n        self.hidden_size = hidden_size\n        self.batch_first = batch_first\n\n        self.att_weights = nn.Parameter(torch.Tensor(1, hidden_size), requires_grad=True)\n        nn.init.xavier_uniform_(self.att_weights.data)\n\n    def get_mask(self):\n        pass\n\n    def forward(self, inputs, lengths):\n        if self.batch_first:\n            batch_size, max_len = inputs.size()[:2]\n        else:\n            max_len, batch_size = inputs.size()[:2]\n            \n        # apply attention layer\n        weights = torch.bmm(inputs,\n                            self.att_weights  # (1, hidden_size)\n                            .permute(1, 0)  # (hidden_size, 1)\n                            .unsqueeze(0)  # (1, hidden_size, 1)\n                            .repeat(batch_size, 1, 1) # (batch_size, hidden_size, 1)\n                            )\n    \n        attentions = torch.softmax(F.relu(weights.squeeze()), dim=-1)\n        \n        # create mask based on the sentence lengths\n        mask = torch.ones(attentions.size(), requires_grad=True).cuda()\n        for i, l in enumerate(lengths):  # skip the first sentence\n            if l < max_len:\n                mask[i, l:] = 0\n\n        # apply mask and renormalize attention scores (weights)\n        masked = attentions * mask\n        _sums = masked.sum(-1).unsqueeze(-1)  # sums per row\n        \n        attentions = masked.div(_sums)\n\n        # apply attention weights\n        weighted = torch.mul(inputs, attentions.unsqueeze(-1).expand_as(inputs))\n\n        # get the final fixed vector representations of the sentences\n        representations = weighted.sum(1).squeeze()\n\n        return representations, attentions\n\nclass net(nn.Module):\n    def __init__(self, embedding):\n        super(net, self).__init__()\n                \n        self.embedding = nn.Embedding.from_pretrained(embedding)\n        \n        self.skip_lstm = nn.LSTM(input_size=300, hidden_size=128, num_layers=1, batch_first=True, bidirectional=True)        \n        self.skip_attention = SelfAttention(128*2, batch_first=True)\n\n        self.lstm = nn.LSTM(input_size=300, hidden_size=128, num_layers=1, batch_first=True, bidirectional=True)\n        self.attention = SelfAttention(128*2, batch_first=True)\n        \n        self.avg_pool = nn.AdaptiveAvgPool1d(1)\n        self.max_pool = nn.AdaptiveMaxPool1d(1)\n        \n        self.fc = nn.Linear(128*2*4+2, 1)\n        self.logit = nn.Linear(1, 1)\n\n    def forward(self, x, x_meta, x_len):\n        x = self.embedding(x)\n        x = nn.utils.rnn.pack_padded_sequence(x, x_len, batch_first=True)\n\n        skip_lstm, _ = self.skip_lstm(x)\n        lstm, _ = self.lstm(x)\n        \n        lstm, lstm_lengths = nn.utils.rnn.pad_packed_sequence(lstm, batch_first=True)\n        skip_lstm, skip_lstm_lengths = nn.utils.rnn.pad_packed_sequence(skip_lstm, batch_first=True)\n        \n        skip_attention, _ = self.skip_attention(skip_lstm, skip_lstm_lengths)\n        attention, _ = self.attention(lstm, lstm_lengths)\n        \n        avg_pool = self.avg_pool(lstm.transpose(1, 2))\n        max_pool = self.max_pool(lstm.transpose(1, 2))\n                                \n        x = torch.cat([\n            skip_attention.view(-1, 128*2),\n            attention.view(-1, 128*2),\n            avg_pool.view(-1, 128*2),\n            max_pool.view(-1, 128*2),\n            x_meta.view(-1, 2)\n        ], dim=1)\n                \n        x = self.fc(x)\n        x = self.logit(x).view(-1)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5240b2b0f9ca907d5137255c97ced13ef75c5bfd"},"cell_type":"code","source":"def choose_threshold(val_preds, y_val):\n    thresholds = np.arange(0.1, 0.501, 0.01)\n\n    val_scores = []\n    for threshold in thresholds:\n        threshold = np.round(threshold, 2)\n        f1 = f1_score(y_val, (val_preds > threshold).astype(int))\n        val_scores.append(f1)\n\n    best_val_f1 = np.max(val_scores)\n    best_threshold = np.round(thresholds[np.argmax(val_scores)], 2)\n    \n    return best_threshold, best_val_f1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85135cc5c4b635e7636f7951235b3ffe56597dce"},"cell_type":"code","source":"def test(net, question_field, test_dataloader, test_meta):\n    model = net(question_field.vocab.vectors).to(device)\n    model = initialize(model)\n\n    preds = []\n    y_test = []\n\n    model.eval()\n    with torch.no_grad():\n        for i, batch in tqdm_notebook(enumerate(test_dataloader)):\n            (questions, lengths), index, label = batch.question_text, batch.id, batch.target\n\n            # sort questions and lengths in descending order\n            indices = torch.argsort(lengths, descending=True)\n\n            questions = questions[indices]\n            index = index[indices]\n            lengths = lengths[indices]\n\n            questions = questions.to(device)\n            meta = torch.from_numpy(test_meta[index]).float().to(device)\n\n            out = model(questions, meta, lengths)\n            \n            # unsort outputs\n            indices = torch.argsort(indices)\n            out = out[indices]\n\n            out = torch.sigmoid(out)\n            pred = out.detach().cpu().numpy()\n            \n            preds.append(pred)\n            y_test.append(label.detach())\n\n    preds = np.concatenate(preds, axis=0).reshape(-1, 1)\n    y_test = torch.cat(y_test, dim=0).reshape(-1, 1)\n    \n    return preds, y_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c66f82d17c65c3ce62958448fe8dbd5fae730f80"},"cell_type":"code","source":"def train(train_dataset, val_dataset, train_dataloader, val_dataloader, train_meta, val_meta, net, question_field, vectors, stoi):  \n    best_train_loss = 1e10\n    best_val_loss = 1e10\n\n    best_epoch = 0\n    epochs = 5\n\n    timer = mlc.time.Timer()\n    logger = mlc.LinewiseCSVWriter('train_log.csv', header=['epoch', 'lr', 'train_loss', 'val_loss'])\n\n    question_field.vocab.set_vectors(stoi, vectors, 300)\n    \n    model = net(question_field.vocab.vectors).to(device)\n\n    optimizer = optim.Adam(model.parameters())\n    criterion = nn.BCEWithLogitsLoss()\n\n    for epoch in range(epochs):\n        print(f'\\nStarting Epoch {epoch}')\n\n        train_loss = 0\n        val_loss = 0\n\n        y_train = []\n        train_preds = []\n\n        timer.add(epoch)\n\n        model.train()\n        for i, batch in tqdm_notebook(enumerate(train_dataloader), total=(int(len(train_dataset) / BATCH_SIZE))):\n            (question, length), index, label = batch.question_text, batch.id, batch.target\n\n            question = question.to(device)\n            meta = torch.from_numpy(train_meta[index]).float().to(device)\n            label = label.to(device).float()\n            \n            out = model(question, meta, length)\n\n            loss = criterion(out, label)\n\n            train_loss += loss.item()\n\n            optimizer.zero_grad()\n\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0)\n\n            optimizer.step()\n\n            y_train.append(label.detach())\n            train_preds.append(out.detach())\n\n        model.eval()\n        with torch.no_grad():\n\n            y_val = []\n            val_preds = []\n\n            for j, batch in tqdm_notebook(enumerate(val_dataloader), total=(int(len(val_dataset) / BATCH_SIZE))):\n                (question, length), index, label = batch.question_text,batch.id, batch.target\n                \n                question = question.to(device)\n                meta = torch.from_numpy(val_meta[index]).float().to(device)\n                label = label.to(device).float()\n                \n                out = model(question, meta, length)\n\n                loss = criterion(out, label)\n\n                val_loss += loss.item()\n\n                optimizer.zero_grad()\n\n                y_val.append(label.detach())\n                val_preds.append(out.detach())\n\n        train_loss /= (i + 1)\n        val_loss /= (j + 1)\n\n        y_train = torch.cat(y_train, dim=0).reshape(-1, 1)\n        y_val = torch.cat(y_val, dim=0).reshape(-1, 1)\n\n        train_preds = torch.cat(train_preds, dim=0).reshape(-1, 1)\n        val_preds = torch.cat(val_preds, dim=0).reshape(-1, 1)\n\n        logger.write([epoch, optimizer.param_groups[0]['lr'], train_loss, val_loss])\n\n        print(f'{timer.fsince(epoch)} | End of Epoch {epoch} | Train Loss: {train_loss} | Val Loss: {val_loss}')\n\n        if val_loss < best_val_loss:\n            best_epoch = epoch\n\n            best_train_loss = train_loss\n            best_val_loss = val_loss\n\n            best_val_preds = val_preds\n\n            save_checkpoint({\n                'model': model.state_dict(),\n                'optimizer': optimizer.state_dict(),\n                'epoch': epoch\n            })\n            \n    message = f'Training Finished | Best Epoch {best_epoch} | Best Train Loss: {best_train_loss} | Best Val Loss: {best_val_loss}'\n                        \n    return best_val_preds, y_val, best_epoch, best_train_loss, best_val_loss, message","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"d173b4afb19215ce533b3bdc7389576492d84208","_kg_hide-output":true},"cell_type":"code","source":"glove_vectors = torchtext.vocab.Vectors('../input/embeddings/glove.840B.300d/glove.840B.300d.txt')#, max_vectors=1000)\n\nfor file in os.listdir('./.vector_cache/'):\n    os.remove(f'./.vector_cache/{file}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"db186fc0a7c92e54f22ea10f6d7f052c92812a6c","_kg_hide-output":true},"cell_type":"code","source":"paragram_vectors = torchtext.vocab.Vectors('../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt')#, max_vectors=1000)\n\nfor file in os.listdir('./.vector_cache/'):\n    os.remove(f'./.vector_cache/{file}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed6461fe2dcba4b08a31844ba4435c79982f1b17"},"cell_type":"code","source":"%%time\nmean_vectors = torch.zeros((len(question_field.vocab.stoi), 300))\n\nfor word, i in tqdm_notebook(question_field.vocab.stoi.items(), total=len(question_field.vocab.stoi)):\n    glove_vector = glove_vectors[word]\n    paragram_vector = paragram_vectors[word]\n    \n    vector = torch.stack([glove_vector, paragram_vector])\n    vector = torch.sum(vector, dim=0).reshape(1, -1) / 2\n    \n    mean_vectors[i] = vector\n    \ndel glove_vectors, paragram_vectors\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a10701a1fd7e3cbcff1bcc845489f07155aca0a6","scrolled":false},"cell_type":"code","source":"val_preds, y_val, _, _, _, message = train(train_dataset, val_dataset, train_dataloader, val_dataloader, x_train_meta, x_val_meta, net, question_field, mean_vectors, question_field.vocab.stoi)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d12da7c01b1280b5acd28b7b4a734b50076cc69"},"cell_type":"code","source":"# del mean_vectors\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3912844c4eafe6c1d899340b4eb634276af9621"},"cell_type":"code","source":"log = pd.read_csv('train_log.csv')\nplt.plot(log['epoch'], log['train_loss'], log['val_loss'])\nplt.show()\n\nbest_threshold, best_val_f1 = choose_threshold(torch.sigmoid(val_preds).cpu().numpy(), y_val.cpu().numpy())\ntest_1_preds, test_1_y_val = test(net, question_field, test_1_dataloader, test_1_meta)\ntest_2_preds, test_2_y_val = test(net, question_field, test_2_dataloader, test_2_meta)\n\n# Remove all saved model files\nfor file in os.listdir('./'):\n    if file.endswith('.pt.tar'):\n        os.remove(f'./{file}')\n\nfor file in os.listdir('./.vector_cache/'):\n    os.remove(f'./.vector_cache/{file}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4fd0678500d5a8479cfd3ab387480c7405cb181b"},"cell_type":"code","source":"print(message)\nprint(best_threshold, best_val_f1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f5c296d79520402657f33f8c237dd74df0423ac"},"cell_type":"code","source":"test_1_preds = (test_1_preds > best_threshold).astype(int)\ntest_2_preds = (test_2_preds > best_threshold).astype(int)\n\ntest_1_f1 = f1_score(test_1_y_val, (test_1_preds > best_threshold).astype(int))\ntest_2_f1 = f1_score(test_2_y_val, (test_2_preds > best_threshold).astype(int))\n\nprint(test_1_f1)\nprint(test_2_f1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"\n\n\n\n\n\n\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}