{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Importing Libraries"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","jupyter":{"outputs_hidden":true},"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport tensorflow as tf\nimport sys\nimport collections\n\nimport operator\n\nsys.path.extend(['../input/bert-joint-baseline/'])\n\nimport bert_utils\nimport modeling \n\nimport tokenization\nimport json\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Splitting the Dataset"},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"on_kaggle_server = os.path.exists('/kaggle')\nnq_test_file = '../input/tensorflow2-question-answering/simplified-nq-test.jsonl' \nnq_train_file = '../input/tensorflow2-question-answering/simplified-nq-train.jsonl'\npublic_dataset = os.path.getsize(nq_test_file)<20_000_000\nprivate_dataset = os.path.getsize(nq_test_file)>=20_000_000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"if True:\n    import importlib\n    importlib.reload(bert_utils)","execution_count":null,"outputs":[]},{"metadata":{"jupyter":{"outputs_hidden":true},"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"config = {'attention_probs_dropout_prob':0.1,\n'hidden_act':'gelu', # 'gelu',\n'hidden_dropout_prob':0.1,\n'hidden_size':1024,\n'initializer_range':0.02,\n'intermediate_size':4096,\n'max_position_embeddings':512,\n'num_attention_heads':16,\n'num_hidden_layers':24,\n'type_vocab_size':2,\n'vocab_size':30522}","execution_count":null,"outputs":[]},{"metadata":{"jupyter":{"outputs_hidden":true},"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"class TDense(tf.keras.layers.Layer):\n    def __init__(self,\n                 output_size,\n                 kernel_initializer=None,\n                 bias_initializer=\"zeros\",\n                **kwargs):\n        super().__init__(**kwargs)\n        self.output_size = output_size\n        self.kernel_initializer = kernel_initializer\n        self.bias_initializer = bias_initializer\n    def build(self,input_shape):\n        dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())\n        if not (dtype.is_floating or dtype.is_complex):\n          raise TypeError(\"Unable to build `TDense` layer with \"\n                          \"non-floating point (and non-complex) \"\n                          \"dtype %s\" % (dtype,))\n        input_shape = tf.TensorShape(input_shape)\n        if tf.compat.dimension_value(input_shape[-1]) is None:\n          raise ValueError(\"The last dimension of the inputs to \"\n                           \"`TDense` should be defined. \"\n                           \"Found `None`.\")\n        last_dim = tf.compat.dimension_value(input_shape[-1])\n        self.input_spec = tf.keras.layers.InputSpec(min_ndim=3, axes={-1: last_dim})\n        self.kernel = self.add_weight(\n            \"kernel\",\n            shape=[self.output_size,last_dim],\n            initializer=self.kernel_initializer,\n            dtype=self.dtype,\n            trainable=True)\n        self.bias = self.add_weight(\n            \"bias\",\n            shape=[self.output_size],\n            initializer=self.bias_initializer,\n            dtype=self.dtype,\n            trainable=True)\n        super(TDense, self).build(input_shape)\n    def call(self,x):\n        return tf.matmul(x,self.kernel,transpose_b=True)+self.bias\n    \ndef mk_model(config):\n    seq_len = config['max_position_embeddings']\n    unique_id  = tf.keras.Input(shape=(1,),dtype=tf.int64,name='unique_id')\n    input_ids   = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')\n    input_mask  = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')\n    segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')\n    BERT = modeling.BertModel(config=config,name='bert')\n    pooled_output, sequence_output = BERT(input_word_ids=input_ids,\n                                          input_mask=input_mask,\n                                          input_type_ids=segment_ids)\n    \n    logits = TDense(2,name='logits')(sequence_output)\n    start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')\n    start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')\n    end_logits   = tf.squeeze(end_logits,  axis=-1,name='end_squeeze')\n    \n    ans_type      = TDense(5,name='ans_type')(pooled_output)\n    return tf.keras.Model([input_ for input_ in [unique_id,input_ids,input_mask,segment_ids] \n                           if input_ is not None],\n                          [unique_id,start_logits,end_logits,ans_type],\n                          name='bert-baseline')    ","execution_count":null,"outputs":[]},{"metadata":{"jupyter":{"outputs_hidden":true},"trusted":true},"cell_type":"code","source":"model= mk_model(config)","execution_count":null,"outputs":[]},{"metadata":{"jupyter":{"outputs_hidden":true},"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Checkpoint"},{"metadata":{"jupyter":{"outputs_hidden":true},"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"cpkt = tf.train.Checkpoint(model=model)\ncpkt.restore('../input/bert-joint-baseline/model_cpkt-1').assert_consumed()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Setting the Flags"},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"class DummyObject:\n    def __init__(self,**kwargs):\n        self.__dict__.update(kwargs)\n\nFLAGS=DummyObject(skip_nested_contexts=True, #True\n                  max_position=50,\n                  max_contexts=48,\n                  max_query_length=64, #64\n                  max_seq_length=512, #512\n                  doc_stride=128,\n                  include_unknowns=0.1,\n                  n_best_size=20, \n                  max_answer_length=30) #30","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"import tqdm\neval_records = \"../input/bert-joint-baseline/nq-test.tfrecords\"\n\nif on_kaggle_server and private_dataset:\n    eval_records='nq-test.tfrecords'\nif not os.path.exists(eval_records):\n    \n    eval_writer = bert_utils.FeatureWriter(\n        filename=os.path.join(eval_records),\n        is_training=False)\n\n    tokenizer = tokenization.FullTokenizer(vocab_file='../input/bert-joint-baseline/vocab-nq.txt', \n                                           do_lower_case=True)\n\n    features = []\n    convert = bert_utils.ConvertExamples2Features(tokenizer=tokenizer,\n                                                   is_training=False,\n                                                   output_fn=eval_writer.process_feature,\n                                                   collect_stat=False)\n\n    n_examples = 0\n    tqdm_notebook= tqdm.tqdm_notebook if not on_kaggle_server else None\n    for examples in bert_utils.nq_examples_iter(input_file=nq_test_file, \n                                           is_training=False,\n                                           tqdm=tqdm_notebook):\n        for example in examples:\n            n_examples += convert(example)\n\n    eval_writer.close()","execution_count":null,"outputs":[]},{"metadata":{"jupyter":{"outputs_hidden":true},"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"seq_length = FLAGS.max_seq_length #config['max_position_embeddings']\nname_to_features = {\n      \"unique_id\": tf.io.FixedLenFeature([], tf.int64),\n      \"input_ids\": tf.io.FixedLenFeature([seq_length], tf.int64),\n      \"input_mask\": tf.io.FixedLenFeature([seq_length], tf.int64),\n      \"segment_ids\": tf.io.FixedLenFeature([seq_length], tf.int64),\n  }\n\ndef _decode_record(record, name_to_features=name_to_features):\n    \"\"\"Decodes a record to a TensorFlow example.\"\"\"\n    example = tf.io.parse_single_example(serialized=record, features=name_to_features)\n\n    # tf.Example only supports tf.int64, but the TPU only supports tf.int32.\n    # So cast all int64 to int32.\n    for name in list(example.keys()):\n        t = example[name]\n        if name != 'unique_id': #t.dtype == tf.int64:\n            t = tf.cast(t, dtype=tf.int64)\n        example[name] = t\n\n    return example\n\ndef _decode_tokens(record):\n    return tf.io.parse_single_example(serialized=record, \n                                      features={\n                                          \"unique_id\": tf.io.FixedLenFeature([], tf.int64),\n                                          \"token_map\" :  tf.io.FixedLenFeature([seq_length], tf.int64)\n                                      })\n      \n","execution_count":null,"outputs":[]},{"metadata":{"jupyter":{"outputs_hidden":true},"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"raw_ds = tf.data.TFRecordDataset(eval_records)\ntoken_map_ds = raw_ds.map(_decode_tokens)\ndecoded_ds = raw_ds.map(_decode_record)\nds = decoded_ds.batch(batch_size=16,drop_remainder=False) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result=model.predict_generator(ds,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"np.savez_compressed('bert-joint-baseline-output.npz',\n                    **dict(zip(['uniqe_id','start_logits','end_logits','answer_type_logits'],\n                               result)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"Span = collections.namedtuple(\"Span\", [\"start_token_idx\", \"end_token_idx\", \"score\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"class ScoreSummary(object):\n  def __init__(self):\n    self.predicted_label = None\n    self.short_span_score = None\n    self.cls_token_score = None\n    self.answer_type_logits = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"class EvalExample(object):\n  \"\"\"Eval data available for a single example.\"\"\"\n  def __init__(self, example_id, candidates):\n    self.example_id = example_id\n    self.candidates = candidates\n    self.results = {}\n    self.features = {}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"def get_best_indexes(logits, n_best_size):\n  \"\"\"Get the n-best logits from a list.\"\"\"\n  index_and_score = sorted(\n      enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)\n  best_indexes = []\n  for i in range(len(index_and_score)):\n    if i >= n_best_size:\n      break\n    best_indexes.append(index_and_score[i][0])\n  return best_indexes\n\ndef top_k_indices(logits,n_best_size,token_map):\n    indices = np.argsort(logits[1:])+1\n    indices = indices[token_map[indices]!=-1]\n    return indices[-n_best_size:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_duplicates(span):\n    start_end = []\n    for s in span:\n        cont = 0\n        if not start_end:\n            start_end.append(Span(s[0], s[1], s[2]))\n            cont += 1\n        else:\n            for i in range(len(start_end)):\n                if start_end[i][0] == s[0] and start_end[i][1] == s[1]:\n                    cont += 1\n        if cont == 0:\n            start_end.append(Span(s[0], s[1], s[2]))\n            \n    return start_end","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_short_long_span(predictions, example):\n    \n    sorted_predictions = sorted(predictions, reverse=True)\n    short_span = []\n    long_span = []\n    for prediction in sorted_predictions:\n        score, _, summary, start_span, end_span = prediction\n        # get scores > zero\n        if score > 0:\n            short_span.append(Span(int(start_span), int(end_span), float(score)))\n\n    short_span = remove_duplicates(short_span)\n\n    for s in range(len(short_span)):\n        for c in example.candidates:\n            start = short_span[s].start_token_idx\n            end = short_span[s].end_token_idx\n           \n            if c[\"top_level\"] and c[\"start_token\"] <= start and c[\"end_token\"] >= end: #c[\"top_level\"] and \n                long_span.append(Span(int(c[\"start_token\"]), int(c[\"end_token\"]), float(short_span[s].score)))\n               \n    long_span = remove_duplicates(long_span)\n    \n    long_span.sort(key = operator.itemgetter(2), reverse=True) \n    \n    if not long_span:\n        long_span = [Span(-1, -1, -10000.0)]\n    if not short_span:\n        short_span = [Span(-1, -1, -10000.0)]\n        \n    \n    return short_span, long_span","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"def compute_predictions(example):\n    \"\"\"Converts an example into an NQEval object for evaluation.\"\"\"\n    predictions = []\n    n_best_size = FLAGS.n_best_size\n    max_answer_length = 1024 #FLAGS.max_answer_length\n    i = 0\n    for unique_id, result in example.results.items():\n        if unique_id not in example.features:\n            raise ValueError(\"No feature found with unique_id:\", unique_id)\n        token_map = np.array(example.features[unique_id][\"token_map\"]) #.int64_list.value\n        start_indexes = top_k_indices(result.start_logits,n_best_size,token_map)\n        if len(start_indexes)==0:\n            continue\n        end_indexes = top_k_indices(result.end_logits,n_best_size,token_map)\n        if len(end_indexes)==0:\n            continue\n        indexes = np.array(list(np.broadcast(start_indexes[None],end_indexes[:,None])))  \n        indexes = indexes[(indexes[:,0]<indexes[:,1])*(indexes[:,1]-indexes[:,0]<max_answer_length)]\n        for _, (start_index,end_index) in enumerate(indexes):  \n            summary = ScoreSummary()\n            summary.short_span_score = (\n                result.start_logits[start_index] +\n                result.end_logits[end_index])\n            summary.cls_token_score = (\n                result.start_logits[0] + result.end_logits[0])\n            summary.answer_type_logits = result.answer_type_logits-result.answer_type_logits.mean()\n            start_span = token_map[start_index]\n            end_span = token_map[end_index] + 1\n\n            # Span logits minus the cls logits seems to be close to the best.\n            score = summary.short_span_score - summary.cls_token_score\n            predictions.append((score, i, summary, start_span, end_span))\n            i += 1 # to break ties\n\n    # Default empty prediction.\n    #score = -10000.0\n    short_span = [Span(-1, -1, -10000.0)]\n    long_span  = [Span(-1, -1, -10000.0)]\n    summary    = ScoreSummary()\n\n    if predictions:\n        short_span, long_span = get_short_long_span(predictions, example)\n      \n    summary.predicted_label = {\n        \"example_id\": int(example.example_id),\n        \"long_answers\": {\n          \"tokens_and_score\": long_span,\n          #\"end_token\": long_span,\n          \"start_byte\": -1,\n          \"end_byte\": -1\n        },\n        #\"long_answer_score\": answer_score,\n        \"short_answers\": {\n          \"tokens_and_score\": short_span,\n          #\"end_token\": short_span,\n          \"start_byte\": -1,\n          \"end_byte\": -1,\n          \"yes_no_answer\": \"NONE\"\n        }\n        #\"short_answer_score\": answer_scores,\n        \n        #\"answer_type_logits\": summary.answer_type_logits.tolist(),\n        #\"answer_type\": int(np.argmax(summary.answer_type_logits))\n       }\n\n    return summary","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"def compute_pred_dict(candidates_dict, dev_features, raw_results,tqdm=None):\n    \"\"\"Computes official answer key from raw logits.\"\"\"\n    raw_results_by_id = [(int(res.unique_id),1, res) for res in raw_results]\n\n    examples_by_id = [(int(k),0,v) for k, v in candidates_dict.items()]\n  \n    features_by_id = [(int(d['unique_id']),2,d) for d in dev_features] \n  \n    # Join examples with features and raw results.\n    examples = []\n    print('merging examples...')\n    merged = sorted(examples_by_id + raw_results_by_id + features_by_id)\n    print('done.')\n    for idx, type_, datum in merged:\n        if type_==0: #isinstance(datum, list):\n            examples.append(EvalExample(idx, datum))\n        elif type_==2: #\"token_map\" in datum:\n            examples[-1].features[idx] = datum\n        else:\n            examples[-1].results[idx] = datum\n\n    # Construct prediction objects.\n    print('Computing predictions...')\n   \n    nq_pred_dict = {}\n    #summary_dict = {}\n    if tqdm is not None:\n        examples = tqdm(examples)\n    for e in examples:\n        summary = compute_predictions(e)\n        #summary_dict[e.example_id] = summary\n        nq_pred_dict[e.example_id] = summary.predicted_label\n    return nq_pred_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"def read_candidates_from_one_split(input_path):\n  \"\"\"Read candidates from a single jsonl file.\"\"\"\n  candidates_dict = {}\n  print(\"Reading examples from: %s\" % input_path)\n  if input_path.endswith(\".gz\"):\n    with gzip.GzipFile(fileobj=tf.io.gfile.GFile(input_path, \"rb\")) as input_file:\n      for index, line in enumerate(input_file):\n        e = json.loads(line)\n        candidates_dict[e[\"example_id\"]] = e[\"long_answer_candidates\"]\n        \n  else:\n    with tf.io.gfile.GFile(input_path, \"r\") as input_file:\n      for index, line in enumerate(input_file):\n        e = json.loads(line)\n        candidates_dict[e[\"example_id\"]] = e[\"long_answer_candidates\"] \n  return candidates_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_candidates(input_pattern):\n  \"\"\"Read candidates with real multiple processes.\"\"\"\n  input_paths = tf.io.gfile.glob(input_pattern)\n  final_dict = {}\n  for input_path in input_paths:\n    final_dict.update(read_candidates_from_one_split(input_path))\n  return final_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"all_results = [bert_utils.RawResult(*x) for x in zip(*result)]\n    \nprint (\"Going to candidates file\")\n\ncandidates_dict = read_candidates('../input/tensorflow2-question-answering/simplified-nq-test.jsonl')\n\nprint (\"setting up eval features\")\n\neval_features = list(token_map_ds)\n\nprint (\"compute_pred_dict\")\n\ntqdm_notebook= tqdm.tqdm_notebook\nnq_pred_dict = compute_pred_dict(candidates_dict,\n                                 eval_features,\n                                 all_results,\n                                 tqdm=tqdm_notebook)\n\npredictions_json = {\"predictions\": list(nq_pred_dict.values())}\n\nprint (\"writing json\")\n\nwith tf.io.gfile.GFile('predictions.json', \"w\") as f:\n    json.dump(predictions_json, f, indent=4)\nprint('done!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answers_df = pd.read_json(\"../working/predictions.json\")\nanswers_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def df_long_index_score(df):\n    answers = []\n    cont = 0\n    for e in df['long_answers']['tokens_and_score']:\n        # if score > 2\n        if e[2] >= 2.5: \n            index = {}\n            index['start'] = e[0]\n            index['end'] = e[1]\n            index['score'] = e[2]\n            answers.append(index)\n            cont += 1\n        # number of answers\n        if cont == 1:\n            break\n            \n    return answers\n\ndef df_short_index_score(df):\n    answers = []\n    cont = 0\n    for e in df['short_answers']['tokens_and_score']:\n        # if score > 2\n        if e[2] >= 6.5:\n            index = {}\n            index['start'] = e[0]\n            index['end'] = e[1]\n            index['score'] = e[2]\n            answers.append(index)\n            cont += 1\n        # number of answers\n        if cont == 1:\n            break\n            \n    return answers\n\ndef df_example_id(df):\n    return df['example_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answers_df['example_id'] = answers_df['predictions'].apply(df_example_id)\n\nanswers_df['long_indexes_and_scores'] = answers_df['predictions'].apply(df_long_index_score)\n\nanswers_df['short_indexes_and_scores'] = answers_df['predictions'].apply(df_short_index_score)\n\nanswers_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answers_df = answers_df.drop(['predictions'], axis=1)\nanswers_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_answer(entry):\n    answer = []\n    for e in entry:\n        answer.append(str(e['start']) + ':'+ str(e['end']))\n    if not answer:\n        answer = \"\"\n    return \", \".join(answer)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"answers_df[\"long_answer\"] = answers_df['long_indexes_and_scores'].apply(create_answer)\nanswers_df[\"short_answer\"] = answers_df['short_indexes_and_scores'].apply(create_answer)\nanswers_df[\"example_id\"] = answers_df['example_id'].apply(lambda q: str(q))\n\nlong_answers = dict(zip(answers_df[\"example_id\"], answers_df[\"long_answer\"]))\nshort_answers = dict(zip(answers_df[\"example_id\"], answers_df[\"short_answer\"]))\n\nanswers_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answers_df = answers_df.drop(['long_indexes_and_scores', 'short_indexes_and_scores'], axis=1)\nanswers_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generating the Submission File"},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/tensorflow2-question-answering/sample_submission.csv\")\n\nlong_prediction_strings = sample_submission[sample_submission[\"example_id\"].str.contains(\"_long\")].apply(lambda q: long_answers[q[\"example_id\"].replace(\"_long\", \"\")], axis=1)\nshort_prediction_strings = sample_submission[sample_submission[\"example_id\"].str.contains(\"_short\")].apply(lambda q: short_answers[q[\"example_id\"].replace(\"_short\", \"\")], axis=1)\n\nsample_submission.loc[sample_submission[\"example_id\"].str.contains(\"_long\"), \"PredictionString\"] = long_prediction_strings\nsample_submission.loc[sample_submission[\"example_id\"].str.contains(\"_short\"), \"PredictionString\"] = short_prediction_strings\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}