{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.insert(0, '../input/tf2bert')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"BERT finetuning runner.\"\"\"\n# Sorry, didn't annotate much yet..\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\n# Many models have been combined here:\n# BERT + BIDAF-ish + UNET-ish = SOMETHING\n# Have not trained it on TPU yet. Doubt it will meet time requirements.\n# It must have many many bugs too. Posting here just to give some ideas. \n\nimport collections\nimport csv\nimport os\nimport modeling\nimport optimization\nimport tokenization\nimport tensorflow as tf\n\n#flags = tf.flags\nflags = tf.compat.v1.flags\n\nFLAGS = flags.FLAGS\n\n## Required parameters\nflags.DEFINE_string(\n \"data_dir\", None,\n \"The input data dir. Should contain the .tsv files (or other data files) \"\n \"for the task.\")\n\nflags.DEFINE_string(\n \"bert_config_file\", None,\n \"The config json file corresponding to the pre-trained BERT model. \"\n \"This specifies the model architecture.\")\n\nflags.DEFINE_string(\"task_name\", None, \"The name of the task to train.\")\n\nflags.DEFINE_string(\"vocab_file\", None,\n \"The vocabulary file that the BERT model was trained on.\")\n\nflags.DEFINE_string(\n \"output_dir\", None,\n \"The output directory where the model checkpoints will be written.\")\n\n## Other parameters\n\nflags.DEFINE_string(\n \"init_checkpoint\", None,\n \"Initial checkpoint (usually from a pre-trained BERT model).\")\n\nflags.DEFINE_bool(\n \"do_lower_case\", True,\n \"Whether to lower case the input text. Should be True for uncased \"\n \"models and False for cased models.\")\n\nflags.DEFINE_integer(\n \"max_seq_length\", 128,\n \"The maximum total input sequence length after WordPiece tokenization. \"\n \"Sequences longer than this will be truncated, and sequences shorter \"\n \"than this will be padded.\")\n\nflags.DEFINE_bool(\"do_train\", False, \"Whether to run training.\")\n\nflags.DEFINE_bool(\"do_eval\", False, \"Whether to run eval on the dev set.\")\n\nflags.DEFINE_bool(\n \"do_predict\", False,\n \"Whether to run the model in inference mode on the test set.\")\n\nflags.DEFINE_integer(\"train_batch_size\", 32, \"Total batch size for training.\")\n\nflags.DEFINE_integer(\"eval_batch_size\", 8, \"Total batch size for eval.\")\n\nflags.DEFINE_integer(\"predict_batch_size\", 8, \"Total batch size for predict.\")\n\nflags.DEFINE_float(\"learning_rate\", 5e-5, \"The initial learning rate for Adam.\")\n\nflags.DEFINE_float(\"num_train_epochs\", 3.0,\n \"Total number of training epochs to perform.\")\n\nflags.DEFINE_float(\n \"warmup_proportion\", 0.1,\n \"Proportion of training to perform linear learning rate warmup for. \"\n \"E.g., 0.1 = 10% of training.\")\n\nflags.DEFINE_integer(\"save_checkpoints_steps\", 1000,\n \"How often to save the model checkpoint.\")\n\nflags.DEFINE_integer(\"iterations_per_loop\", 1000,\n \"How many steps to make in each estimator call.\")\n\nflags.DEFINE_bool(\"use_tpu\", False, \"Whether to use TPU or GPU/CPU.\")\n\ntf.compat.v1.flags.DEFINE_string(\n \"tpu_name\", None,\n \"The Cloud TPU to use for training. This should be either the name \"\n \"used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 \"\n \"url.\")\n\ntf.compat.v1.flags.DEFINE_string(\n \"tpu_zone\", None,\n \"[Optional] GCE zone where the Cloud TPU is located in. If not \"\n \"specified, we will attempt to automatically detect the GCE project from \"\n \"metadata.\")\n\ntf.compat.v1.flags.DEFINE_string(\n \"gcp_project\", None,\n \"[Optional] Project name for the Cloud TPU-enabled project. If not \"\n \"specified, we will attempt to automatically detect the GCE project from \"\n \"metadata.\")\n\ntf.compat.v1.flags.DEFINE_string(\"master\", None, \"[Optional] TensorFlow master URL.\")\n\nflags.DEFINE_integer(\n \"num_tpu_cores\", 8,\n \"Only used if `use_tpu` is True. Total number of TPU cores to use.\")\n\nflags.DEFINE_integer(\n \"cont_len\", 350,\n \"Length allowed for the long sentence in the BERT input. Cont + Ques should be around 500.\")\n\nflags.DEFINE_integer(\n \"ques_len\", 150,\n \"Length allowed for the question sentence in the BERT input.\")\n\n\nclass InputExample(object):\n \"\"\"A single training/test example for simple sequence classification.\"\"\"\n\n def __init__(self, guid, text_a, text_b=None, target_conv3=None, target_conv6 = None, target_present = None):\n \"\"\"Constructs a InputExample.\n Args:\n guid: Unique id for the example.\n text_a: string. The untokenized text of the first sequence. For single\n sequence tasks, only this sequence must be specified.\n text_b: (Optional) string. The untokenized text of the second sequence.\n Only must be specified for sequence pair tasks.\n label: (Optional) string. The label of the example. This should be\n specified for train and dev examples, but not for test examples.\n \"\"\"\n self.guid = guid\n self.text_a = text_a\n self.text_b = text_b\n self.target_conv3 = target_conv3\n self.target_conv6 = target_conv6\n self.target_present = target_present\n self.q_mask = q_mask\n self.c_mask = c_mask\n\nclass PaddingInputExample(object):\n \"\"\"Fake example so the num input examples is a multiple of the batch size.\n When running eval/predict on the TPU, we need to pad the number of examples\n to be a multiple of the batch size, because the TPU requires a fixed batch\n size. The alternative is to drop the last batch, which is bad because it means\n the entire output data won't be generated.\n We use this class instead of `None` because treating `None` as padding\n battches could cause silent errors.\n \"\"\"\n\n\nclass InputFeatures(object):\n \"\"\"A single set of features of data.\"\"\"\n\n def __init__(self,\n input_ids,\n input_mask,\n segment_ids,\n target_conv3,\n target_conv6,\n target_present,\n q_mask,\n c_mask,\n is_real_example=True):\n self.input_ids = input_ids\n self.input_mask = input_mask\n self.segment_ids = segment_ids\n self.target_conv3 = target_conv3\n self.target_conv6 = target_conv6\n self.target_present = target_present\n self.q_mask = q_mask\n self.c_mask = c_mask\n self.is_real_example = is_real_example\n\n\nclass DataProcessor(object):\n \"\"\"Base class for data converters for sequence classification data sets.\"\"\"\n\n def get_train_examples(self, data_dir):\n \"\"\"Gets a collection of `InputExample`s for the train set.\"\"\"\n raise NotImplementedError()\n\n def get_dev_examples(self, data_dir):\n \"\"\"Gets a collection of `InputExample`s for the dev set.\"\"\"\n raise NotImplementedError()\n\n def get_test_examples(self, data_dir):\n \"\"\"Gets a collection of `InputExample`s for prediction.\"\"\"\n raise NotImplementedError()\n\n def get_labels(self):\n \"\"\"Gets the list of labels for this data set.\"\"\"\n raise NotImplementedError()\n\n @classmethod\n def _read_tsv(cls, input_file, quotechar=None):\n \"\"\"Reads a tab separated value file.\"\"\"\n with tf.gfile.Open(input_file, \"r\") as f:\n reader = csv.reader(f, delimiter=\"\\t\", quotechar=quotechar)\n lines = []\n for line in reader:\n lines.append(line)\n return lines\n\n\nclass KeplerProcessor(DataProcessor):\n \"\"\"Processor for the XNLI data set.\"\"\"\n \n def __init__(self):\n self.language = \"zh\"\n \n def get_train_examples(self, data_dir):\n \"\"\"See base class.\"\"\"\n \n examples = []\n train_df = []\n train_df = pd.read_json(path, orient = 'records', lines = True)\n print('Our dataset have {} rows and {} columns'.format(df.shape[0], df.shape[1]))\n gc.collect()\n \n for i_main, row in train.iterrows():\n \n document_text = row['document_text'].split()\n question_text = row['question_text']\n \n for candidate_no, long_answer_candidate in enumerate(row['long_answer_candidates']):\n\n target_conv3 = [0] * FLAGS.cont_len\n target_conv6 = [0] * FLAGS.cont_len\n target_present = [0] * FLAGS.cont_len\n\n q_mask = [1] * FLAGS.ques_len\n c_mask = [1] * FLAGS.cont_len\n\n long_ans_start_tok = long_answer_candidate['start_token']\n long_ans_end_tok = long_answer_candidate['end_token']\n long_cand_length = long_ans_end_tok - long_ans_start_tok\n\n if long_cand_length > FLAGS.cont_len:\n long_sentence = \" \".join(document_text[long_ans_start_tok:long_ans_start_tok + FLAGS.cont_len]) \n else:\n long_sentence = \" \".join(document_text[long_ans_start_tok:long_ans_end_tok])\n for i in range(long_cand_length+1,FLAG.cont_len):\n c_mask[i] = 0\n\n if long_ans_start_tok == row['annotations'][0]['long_answer']['start_token'] and \\\n len(row['annotations'][0]['short_answers']) > 0:\n\n #print(\"this is correct long answer\")\n\n short_answer_start_token = row['annotations'][0]['short_answers'][0]['start_token']\n short_answer_end_token = row['annotations'][0]['short_answers'][0]['end_token']\n short_start_idx = short_answer_start_token-long_ans_start_tok\n short_end_idx = short_answer_end_token-long_ans_start_tok\n\n if short_end_idx < cont_len:\n target_start[short_start_idx] = 1\n target_end[short_end_idx] = 1\n\n for i in range(short_start_idx,short_end_idx):\n target_present[i] = 1\n else:\n smth = \"short answer beyond maximum len\"\n\n ques_length = len(question_text.split()) \n if ques_length < FLAGS.ques_len:\n for i in range(ques_length+1,FLAGS.ques_len):\n q_mask[i] = 0 \n\n guid = \"train-%d\" % (i_main)\n text_a = tokenization.convert_to_unicode(long_sentence)\n text_b = tokenization.convert_to_unicode(question_text)\n target_start = tokenization.convert_to_unicode(target_start)\n target_end = tokenization.convert_to_unicode(target_end)\n target_present = tokenization.convert_to_unicode(target_present)\n q_mask = tokenization.convert_to_unicode(q_mask)\n c_len = tokenization.convert_to_unicode(c_mask) \n\n examples.append(InputExample(guid=guid, text_a=text_a,\\\n text_b=text_b, target_start=target_start, target_end=target_end, \n target_present=target_present, q_mask=q_mask, c_mask=c_mask))\n\n return examples \n\n def get_labels(self):\n \"\"\"See base class.\"\"\"\n return [\"target_conv3\", \"target_conv6\", \"target_present\"]\n\ndef convert_single_example(ex_index, example, label_list, max_seq_length,\n tokenizer):\n \"\"\"Converts a single `InputExample` into a single `InputFeatures`.\"\"\"\n\n if isinstance(example, PaddingInputExample):\n return InputFeatures(\n input_ids=[0] * max_seq_length,\n input_mask=[0] * max_seq_length,\n segment_ids=[0] * max_seq_length,\n target_conv3 = [0]*FLAGS.cont_len,\n target_conv6 = [0]*FLAGS.cont_len,\n target_present = [0]*FLAGS.cont_len,\n q_mask = [0]*FLAGS.ques_len,\n c_mask = [0]*FLAGS.cont_len,\n is_real_example=False)\n\n label_map = {}\n for (i, label) in enumerate(label_list):\n label_map[label] = i\n\n tokens_a = tokenizer.tokenize(example.text_a)\n \n #We need exact length to later build the BIDAF\n tokens_a = tokens_a[0:FLAGS.cont_len]\n \n tokens_b = None\n if example.text_b:\n tokens_b = tokenizer.tokenize(example.text_b)\n #We need exact length to later build the BIDAF\n tokens_b = tokens_b[0:FLAGS.cont_len] \n \n if tokens_b:\n # Modifies `tokens_a` and `tokens_b` in place so that the total\n # length is less than the specified length.\n # Account for [CLS], [SEP], [SEP] with \"- 3\"\n _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)\n else:\n # Account for [CLS] and [SEP] with \"- 2\"\n if len(tokens_a) > max_seq_length - 2:\n tokens_a = tokens_a[0:(max_seq_length - 2)]\n\n # The convention in BERT is:\n # (a) For sequence pairs:\n # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]\n # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1\n # (b) For single sequences:\n # tokens: [CLS] the dog is hairy . [SEP]\n # type_ids: 0 0 0 0 0 0 0\n #\n # Where \"type_ids\" are used to indicate whether this is the first\n # sequence or the second sequence. The embedding vectors for `type=0` and\n # `type=1` were learned during pre-training and are added to the wordpiece\n # embedding vector (and position vector). This is not *strictly* necessary\n # since the [SEP] token unambiguously separates the sequences, but it makes\n # it easier for the model to learn the concept of sequences.\n #\n # For classification tasks, the first vector (corresponding to [CLS]) is\n # used as the \"sentence vector\". Note that this only makes sense because\n # the entire model is fine-tuned.\n tokens = []\n segment_ids = []\n tokens.append(\"[CLS]\")\n segment_ids.append(0)\n for token in tokens_a:\n tokens.append(token)\n segment_ids.append(0)\n tokens.append(\"[SEP]\")\n segment_ids.append(0)\n\n if tokens_b:\n for token in tokens_b:\n tokens.append(token)\n segment_ids.append(1)\n tokens.append(\"[SEP]\")\n segment_ids.append(1)\n\n input_ids = tokenizer.convert_tokens_to_ids(tokens)\n\n # The mask has 1 for real tokens and 0 for padding tokens. Only real\n # tokens are attended to.\n input_mask = [1] * len(input_ids)\n\n # Zero-pad up to the sequence length.\n while len(input_ids) < max_seq_length:\n input_ids.append(0)\n input_mask.append(0)\n segment_ids.append(0)\n\n assert len(input_ids) == max_seq_length\n assert len(input_mask) == max_seq_length\n assert len(segment_ids) == max_seq_length\n\n #The following 3 lines are redundant; just for convention \n label_conv3 = example.target_conv3\n label_conv6 = example.target_conv6\n label_present = example.target_present\n c_mask = example.c_mask\n q_mask = example.q_mask \n \n if ex_index < 5:\n tf.logging.info(\"*** Example ***\")\n tf.logging.info(\"guid: %s\" % (example.guid))\n tf.logging.info(\"tokens: %s\" % \" \".join(\n [tokenization.printable_text(x) for x in tokens]))\n tf.logging.info(\"input_ids: %s\" % \" \".join([str(x) for x in input_ids]))\n tf.logging.info(\"input_mask: %s\" % \" \".join([str(x) for x in input_mask]))\n tf.logging.info(\"segment_ids: %s\" % \" \".join([str(x) for x in segment_ids]))\n tf.logging.info(\"label_start: (id = %d)\" % (label_conv3))\n tf.logging.info(\"label_end: (id = %d)\" % (label_conv6))\n tf.logging.info(\"label_present: (id = %d)\" % (label_present))\n\n feature = InputFeatures(\n input_ids=input_ids,\n input_mask=input_mask,\n segment_ids=segment_ids,\n target_conv3 = label_conv3,\n target_conv6 = label_conv6,\n target_present = label_present,\n c_mask = c_mask,\n q_mask = q_mask,\n is_real_example=True)\n return feature\n\n\ndef file_based_convert_examples_to_features(\n examples, label_list, max_seq_length, tokenizer, output_file):\n \"\"\"Convert a set of `InputExample`s to a TFRecord file.\"\"\"\n\n writer = tf.python_io.TFRecordWriter(output_file)\n\n for (ex_index, example) in enumerate(examples):\n if ex_index % 10000 == 0:\n tf.logging.info(\"Writing example %d of %d\" % (ex_index, len(examples)))\n\n feature = convert_single_example(ex_index, example, label_list,\n max_seq_length, tokenizer)\n\n def create_int_feature(values):\n f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values)))\n return f\n\n features = collections.OrderedDict()\n features[\"input_ids\"] = create_int_feature(feature.input_ids)\n features[\"input_mask\"] = create_int_feature(feature.input_mask)\n features[\"segment_ids\"] = create_int_feature(feature.segment_ids)\n features[\"target_start_ids\"] = create_int_feature([feature.target_conv3])\n features[\"target_end_ids\"] = create_int_feature([feature.target_conv6])\n features[\"target_present_ids\"] = create_int_feature([feature.target_present])\n features[\"q_mask\"] = create_int_feature([feature.q_mask]) \n features[\"c_mask\"] = create_int_festure([feature.c_mask]) \n features[\"is_real_example\"] = create_int_feature(\n [int(feature.is_real_example)])\n\n tf_example = tf.train.Example(features=tf.train.Features(feature=features))\n writer.write(tf_example.SerializeToString())\n writer.close()\n\n\ndef file_based_input_fn_builder(input_file, seq_length, is_training,\n drop_remainder):\n \"\"\"Creates an `input_fn` closure to be passed to TPUEstimator.\"\"\"\n\n name_to_features = {\n \"input_ids\": tf.FixedLenFeature([seq_length], tf.int64),\n \"input_mask\": tf.FixedLenFeature([seq_length], tf.int64),\n \"segment_ids\": tf.FixedLenFeature([seq_length], tf.int64),\n \"target_start_ids\": tf.FixedLenFeature([], tf.int64),\n \"target_end_ids\": tf.FixedLenFeature([], tf.int64),\n \"target_present_ids\": tf.FixedLenFeature([], tf.int64),\n \"c_mask\": tf.FixedLenFeature([], tf.int64),\n \"q_mask\": tf.FixedLenFeature([], tf.int64),\n \"is_real_example\": tf.FixedLenFeature([], tf.int64),\n }\n\n def _decode_record(record, name_to_features):\n \"\"\"Decodes a record to a TensorFlow example.\"\"\"\n example = tf.parse_single_example(record, name_to_features)\n\n # tf.Example only supports tf.int64, but the TPU only supports tf.int32.\n # So cast all int64 to int32.\n for name in list(example.keys()):\n t = example[name]\n if t.dtype == tf.int64:\n t = tf.to_int32(t)\n example[name] = t\n\n return example\n\ndef input_fn(params):\n \"\"\"The actual input function.\"\"\"\n batch_size = params[\"batch_size\"]\n\n # For training, we want a lot of parallel reading and shuffling.\n # For eval, we want no shuffling and parallel reading doesn't matter.\n d = tf.data.TFRecordDataset(input_file)\n if is_training:\n d = d.repeat()\n d = d.shuffle(buffer_size=100)\n\n d = d.apply(\n tf.contrib.data.map_and_batch(\n lambda record: _decode_record(record, name_to_features),\n batch_size=batch_size,\n drop_remainder=drop_remainder))\n\n return d\n\n return input_fn\n\n\ndef _truncate_seq_pair(tokens_a, tokens_b, max_length):\n \"\"\"Truncates a sequence pair in place to the maximum length.\"\"\"\n\n # This is a simple heuristic which will always truncate the longer sequence\n # one token at a time. This makes more sense than truncating an equal percent\n # of tokens from each, since if one sequence is very short then each token\n # that's truncated likely contains more information than a longer sequence.\n while True:\n total_length = len(tokens_a) + len(tokens_b)\n if total_length <= max_length:\n break\n if len(tokens_a) > len(tokens_b):\n tokens_a.pop()\n else:\n tokens_b.pop()\n \ndef masked_softmax(logits, mask, dim):\n \"\"\"\n Takes masked softmax over given dimension of logits. Discards padded entries with e^(-inf).\n Inputs:\n logits: Numpy array. We want to take softmax over dimension dim.\n mask: Numpy array of same shape as logits.\n Has 1s where there's real data in logits, 0 where there's padding\n dim: int. dimension over which to take softmax\n Returns:\n masked_logits: Numpy array same shape as logits.\n This is the same as logits, but with 1e30 subtracted\n (i.e. very large negative number) in the padding locations.\n prob_dist: Numpy array same shape as logits.\n The result of taking softmax over masked_logits in given dimension.\n Should be 0 in padding locations.\n Should sum to 1 over given dimension.\n \"\"\"\n exp_mask = (1 - tf.cast(mask, 'float')) * (-1e30) # -large where there's padding, 0 elsewhere\n masked_logits = tf.add(logits, exp_mask) # where there's padding, set logits to -large\n prob_dist = tf.nn.softmax(masked_logits, dim)\n return masked_logits, prob_dist \n \ndef cnn_output_width(input_width, kernel_size, padding_amount, strides):\n return (input_width - kernel_size + 2*padding_amount) / strides + 1 \n \ndef deconv_output_shape(input_batch_size, input_size_w, output_channel_size, padding):\n output_size_h = 1\n stride = 2\n filter_size_w = 2\n if padding == 'VALID':\n output_size_w = (input_size_w - 1)*stride + filter_size_w\n elif padding == 'SAME':\n output_size_w = (input_size_w - 1)*stride + 1\n else:\n raise ValueError(\"unknown padding\")\n output_shape = tf.stack([input_batch_size, \n output_size_h, output_size_w, \n output_channel_size]) \n return output_shape\n \ndef create_model(bert_config, is_training, input_ids, input_mask, segment_ids,\n target_start, target_end, target_present, q_mask, c_mask, num_labels, use_one_hot_embeddings):\n \"\"\"Creates a classification model.\"\"\"\n model = modeling.BertModel(\n config=bert_config,\n is_training=is_training,\n input_ids=input_ids,\n input_mask=input_mask,\n token_type_ids=segment_ids,\n use_one_hot_embeddings=use_one_hot_embeddings)\n\n # In the demo, we are doing a simple classification task on the entire\n # segment.\n #\n # If you want to use the token-level output, use model.get_sequence_output()\n # instead.\n output_layer = model.get_sequence_output()\n \n output_layer_shape = modeling.get_shape_list(output_layer, expected_rank=3) \n \n batch_size = output_layer[0]\n seq_length = output_layer[1]\n hidden_size = output_layer[2] \n \n hidden_size = output_layer.shape[-1].value \n\n SW_weights = tf.get_variable(\n \"similarity_weights\", [1, 3*hidden_size],\n initializer=tf.contrib.layers.xavier_initializer())\n \n c = output_layer[:,1:FLAGS.cont_len+1,:] #do not count the [CLS]\n q = output_layer[:,FLAGS.cont_len+2:-2,:] #do not count the [SEP] and [SEP]\n \n # Hidden size = 2h by convention \n \n c_expand = tf.expand_dims(c,2) #[B,N,1,2h]\n q_expand = tf.expand_dims(q,1) #[B,1,M,2h]\n c_pointWise_q = c_expand * q_expand #[B,N,M,2h] \n \n c_input = tf.tile(c_expand, [1, 1, tf.shape(q)[1], 1]) #fill in to get same dims\n q_input = tf.tile(q_expand, [1, tf.shape(c)[1], 1, 1])\n \n concat_input = tf.concat([c_input, q_input, c_pointWise_q], -1) # [B,N,M,6h]\n\n similarity=tf.reduce_sum(concat_input * self.S_W, axis=3) #[B,N,M]\n \n # q_mask shape [B,M]\n # c_mask shape [B,N] \n similarity_mask = tf.expand_dims(q_mask, 1) # [B, 1, M]\n \n similarity_mask = tf.tile(similarity_mask, [1,tf.shape(c)[1],1]) # [B, N, M]\n \n _, c2q_dist = masked_softmax(similarity, similarity_mask, 2) # shape (B, N, M). take softmax over q\n \n c2q = tf.matmul(c2q_dist, q) # shape (B, N, 2h)\n \n S_max = tf.reduce_max(similarity, axis=2) # shape (B, N) ; reminder N = cont_len\n \n _, c_dash_dist = masked_softmax(S_max, c_mask, 1) # distribution of shape (B, N)\n \n c_dash_dist_expand = tf.expand_dims(c_dash_dist, 1) # shape (B, 1, N)\n \n c_dash = tf.matmul(c_dash_dist_expand, c) # shape (B, 1, 2h)\n \n c_c2q = c * c2q # shape (B, N, 2h)\n \n c_dash = tf.tile(c_dash, [1,tf.shape(c)[1],1]) # [B, N, 2h] \n \n c_c_dash = c * c_dash # shape (B, N, 2h)\n \n output = tf.concat([c2q, c_c2q, c_c_dash], axis=2) # (B, N, 2h * 3) \n \n output = tf.nn.dropout(output, 0.9)\n \n blended_reps = tf.concat([c, output], axis=2) # (B, N, 8h)\n\n ### ADD MODELING LAYER .. but first add some more data \n \n pooled_output = model.get_pooled_output() # Shape (B, 2h)\n\n pooled_exp = tf.expand_dims(pooled_output, 1) # shape (B, 1, 2h) \n \n pooled_tile = tf.tile(pooled_tile, [1, FLAGS.cont_len, 1]) # shape (B, cont_len, 2h) \n \n model_input = tf.concat([blended_reps, pooled_tile], 2) # shape (B, cont_len, 10h)\n \n # we will go two different routes. targets_conv will come from convolution layers and target_present from lstm..\n # the following is route 1: \n \n fw_cell = tf.nn.rnn_cell.BasicLSTMCell(256)\n bw_cell = tf.nn.rnn_cell.BasicLSTMCell(256)\n rnn_outputs, rnn_state = tf.nn.bidirectional_dynamic_rnn(cell_fw=fw_cell, cell_bw=bw_cell,\n inputs=model_input, sequence_length=FLAGS.cont_len,\n dtype=tf.float64)\n \n rnn_outputs = tf.concat(rnn_outputs, 2) # Shape (B, cont_len, 256*2)\n rnn_outputs = tf.relu(rnn_outputs)\n \n # Now copying from run_nq.py \n rnn_output_weights = tf.get_variable(\n \"rnn_output_w\", [2, 256],\n initializer=tf.truncated_normal_initializer(stddev=0.02))\n rnn_outout_bias = tf.get_variable(\n \"rnn_output_b\", [2], initializer=tf.zeros_initializer())\n \n rnn_outputs = tf.reshape(rnn_outputs, [batch_size*FLAGS.cont_len, hidden_size]) # shape [B*N, 2h] \n \n rnn_logits = tf.matmul(rnn_outputs, rnn_output_weights, transpose_b=True) # shape [B*N, 2]\n \n rnn_logits = tf.nn.bias_add(rnn_logits, rnn_output_bias) # shape [B*N, 2]\n \n rnn_logits = tf.reshape(rnn_logits, [batch_size, FLAGS.cont_len, 2]) #shape [B, N, 2]\n \n rnn_logits = tf.transpose(rnn_logits, [2, 0, 1]) #shape [2, B, N]\n \n unstacked_rnn_logits = tf.unstack(rnn_logits, axis=0)\n\n (start_logits, end_logits) = (unstacked_rnn_logits[0], unstacked_rnn_logits[1])\n \n start_log_probs = tf.nn.log_softmax(start_logits, axis=-1)\n\n end_log_probs = tf.nn.log_softmax(end_logits, axis=-1)\n \n start_loss = -tf.reduce_mean(\n tf.reduce_sum(target_start * start_log_probs, axis=-1))\n \n end_loss = -tf.reduce_mean(\n tf.reduce_sum(target_end * end_log_probs, axis=-1))\n \n # Now Route 2: Convolutions \n # Expand dims to make it a 3D for the convolution:\n conv_input = tf.expand_dims(model_input, axis=1) # Change the shape to [B, 1, cont_len, 5*emb_size] \n \n #U-NET downladder filters \n filter1 = tf.get_variable(\"conv1_filter\", shape=[1, 3, hidden_size*5, 64]) # [h, w, in_size, out_size]\n filter2 = tf.get_variable(\"conv2_filter\", shape=[1, 3, 64, 64])\n filter3 = tf.get_variable(\"conv3_filter\", shape=[1, 3, 64, 128])\n filter4 = tf.get_variable(\"conv4_filter\", shape=[1, 3, 128, 128])\n filter5 = tf.get_variable(\"conv5_filter\", shape=[1, 3, 128, 256])\n filter6 = tf.get_variable(\"conv6_filter\", shape=[1, 3, 256, 256])\n \n #U-NET upladder filters \n up6_filter = tf.get_variable(\"up6_filter\", shape=[1, 2, 256, 256])\n filter7 = tf.get_variable(\"conv3_filter\", shape=[1, 3, 256, 256])\n up7_filter = tf.get_variable(\"up6_filter\", shape=[1, 2, 384, 384])\n filter8 = tf.get_variable(\"conv3_filter\", shape=[1, 3, 448, 448])\n filter9 = tf.get_variable(\"conv3_filter\", shape=[1, 3, 448, 1]) \n \n # Output shapes based on default cont_len 350 \n conv1 = tf.nn.conv2d(conv_input, filter=filter1, strides=[1, 1, 1, 1], padding=\"VALID\") # shape [B, 1, 348, 64]\n conv1 = tf.nn.relu(conv1) \n conv2 = tf.nn.conv2d(conv2, filter=filter2, strides=[1, 1, 1, 1], padding=\"VALID\") # shape [B, 1, 346, 64] \n conv2 = tf.nn.relu(conv2) \n maxp2 = tf.nn.max_pool(conv2, ksize=[1, 1, 2, 1], strides=[1, 1, 2, 1], padding='VALID') # shape [B, 1, 178, 64]\n\n conv3 = tf.nn.conv2d(maxp2, filter=filter3, strides=[1, 1, 1, 1], padding=\"VALID\") # shape [B, 1, 176, 128]\n conv3 = tf.nn.relu(conv3)\n conv4 = tf.nn.conv2d(conv4, filter=filter4, strides=[1, 1, 1, 1], padding=\"VALID\") # shape [B, 1, 174, 128]\n conv4 = tf.nn.relu(conv4)\n maxp4 = tf.nn.max_pool(conv4, ksize=[1, 1, 2, 1], strides=[1, 1, 2, 1], padding='VALID') # shape [B, 1, 87, 128] \n \n conv5 = tf.nn.conv2d(maxp4, filter=filter5, strides=[1, 1, 1, 1], padding=\"VALID\") # shape [B, 1, 85, 256]\n conv5 = tf.nn.relu(conv5)\n conv6 = tf.nn.conv2d(conv6, filter=filter4, strides=[1, 1, 1, 1], padding=\"VALID\") # shape [B, 1, 83, 256]\n conv6 = tf.nn.relu(conv6)\n \n up6_output_shape = deconv_output_shape(conv6.shape[0], conv6.shape[2], conv6.shape[3], \"VALID\") \n conv6_up = tf.nn.conv2d_transpose(conv6, filters = up6_filter, output_shape = up6_output_shape, \n strides = [1, 1, 2, 1], padding = \"VALID\") # shape [B, 1, 166, 256]\n # Convolve until shape is equal to conv4 (174). Use padding = SAME to increase width. \n padding = [[0,0],[0,0],[3,3],[0,0]]\n conv6_padded = tf.pad(conv6,paddings,\"CONSTANT\") # shape [B, 1, 172, 256] \n conv7 = tf.nn.conv2d(conv6_padded, filter=filter7, strides=[1, 1, 1, 1], padding=\"SAME\") # shape [B, 1, 174, 256] \n conv7 = tf.nn.relu(conv7)\n \n conc_4n7 = tf.concat([conv4, conv7], -1) # [B, 1 , 174, 384] \n up7_output_shape = deconv_output_shape(conc_4n7.shape[0], conc_4n7.shape[2], conc_4n7.shape[3], \"VALID\") \n conv7_up = tf.nn.conv2d_transpose(conc_4n7, filters = up7_filter, output_shape = up7_output_shape, \n strides = [1, 1, 2, 1], padding = \"VALID\") # shape [B, 1, 348, 384]\n conc_7n1 = tf.concat([conv7_up, conv1], -1) # [B, 1 , 348, 448]\n conv8 = tf.nn.conv2d(conc_7n1, filter=filter8, strides=[1, 1, 1, 1], padding=\"SAME\") # shape [B, 1, 350, 1]\n \n conv_logits = tf.squeeze(conv8, axis = 3) # shape [B, 1, 350]\n conv_logits = tf.squeeze(conv_logits, axis = 1) # shape [B, cont_len]\n \n conv_preds = tf.nn.sigmoid(conv_logits) \n # WE WILL USE CONV TO PREDICT THE WORDS WITHIN THE SHORT ANSWER SPAN: \n conv_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=target_present, logits=conv_logits) \n \n #Also add loss for answer type, as in run_nq:\n conv_q_typ = tf.nn.max_pool(conv6, ksize=[1, 1, 4, 1], strides=[1, 1, 4, 1], padding='VALID') # shape [B, 1, 22, 256] \n conv_q_typ = tf.squeeze(conv_q_typ, axis = 1) # shape [B, 22, 256]\n conv_q_typ = tf.reshape(conv_q_typ, [batch_size, conv_q_typ.shape[1]*conv_q_typ.shape[2]]) #shape [B, 22*256] \n\n answer_type_hidden_size = conv_q_typ.shape[-1].value\n\n num_answer_types = 5 # YES, NO, UNKNOWN, SHORT, LONG\n \n answer_type_output_weights = tf.get_variable(\n \"answer_type_output_weights\", [num_answer_types, answer_type_hidden_size],\n initializer=tf.truncated_normal_initializer(stddev=0.02))\n\n answer_type_output_bias = tf.get_variable(\n \"answer_type_output_bias\", [num_answer_types],\n initializer=tf.zeros_initializer())\n \n answer_type_logits = tf.matmul(\n conv_q_typ, answer_type_output_weights, transpose_b=True)\n \n answer_type_logits = tf.nn.bias_add(answer_type_logits,\n answer_type_output_bias)\n \n with tf.variable_scope(\"loss\"): \n total_loss = rnn_loss + conv_loss\n \n return (total_loss, start_loss, end_loss, start_logits, end_logits, conv_preds, answer_type_logits) \n \n\ndef model_fn_builder(bert_config, num_labels, init_checkpoint, learning_rate,\n num_train_steps, num_warmup_steps, use_tpu,\n use_one_hot_embeddings):\n \"\"\"Returns `model_fn` closure for TPUEstimator.\"\"\"\n\n # This is the most confusing one. Note that “labels” are not passed on by the model_fn_builder. \n # They are actually passed on inside tpu_estimator when it calls the model_fn. We don’t see how. \n # Apparently we need to treat labels as per example, not per batch (to be confirmed). \n \n def model_fn(features, labels, mode, params): # pylint: disable=unused-argument\n \"\"\"The `model_fn` for TPUEstimator.\"\"\"\n\n tf.logging.info(\"*** Features ***\")\n for name in sorted(features.keys()):\n tf.logging.info(\" name = %s, shape = %s\" % (name, features[name].shape))\n\n input_ids = features[\"input_ids\"]\n input_mask = features[\"input_mask\"]\n segment_ids = features[\"segment_ids\"]\n target_start_ids = features[\"target_start_ids\"]\n target_end_ids = features[\"target_end_ids\"]\n target_present_ids = features[\"target_present_ids\"]\n q_mask = features[\"q_mask\"]\n c_mask = features[\"c_mask\"] \n \n is_real_example = None\n if \"is_real_example\" in features:\n is_real_example = tf.cast(features[\"is_real_example\"], dtype=tf.float32)\n else:\n is_real_example = tf.ones(tf.shape(label_ids), dtype=tf.float32)\n\n is_training = (mode == tf.estimator.ModeKeys.TRAIN)\n\n (total_loss, start_loss, end_loss, start_logits, end_logits, conv_preds, answer_type_logits) = create_model(\n bert_config, is_training, input_ids, input_mask, segment_ids, target_start_ids,\n target_end_ids, target_present_ids, q_mask, c_mask, num_labels, use_one_hot_embeddings)\n\n def compute_label_loss(logits, labels):\n one_hot_labels = tf.one_hot(\n labels, depth=len(AnswerType), dtype=tf.float32)\n log_probs = tf.nn.log_softmax(logits, axis=-1)\n loss = -tf.reduce_mean(\n tf.reduce_sum(one_hot_labels * log_probs, axis=-1))\n return loss\n \n answer_type_loss = compute_label_loss(answer_type_logits, answer_types)\n \n total_loss = total_loss + answer_type_loss\n \n tvars = tf.trainable_variables()\n initialized_variable_names = {}\n scaffold_fn = None\n if init_checkpoint:\n (assignment_map, initialized_variable_names\n ) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint)\n if use_tpu:\n\n def tpu_scaffold():\n tf.train.init_from_checkpoint(init_checkpoint, assignment_map)\n return tf.train.Scaffold()\n\n scaffold_fn = tpu_scaffold\n else:\n tf.train.init_from_checkpoint(init_checkpoint, assignment_map)\n\n tf.logging.info(\"**** Trainable Variables ****\")\n for var in tvars:\n init_string = \"\"\n if var.name in initialized_variable_names:\n init_string = \", *INIT_FROM_CKPT*\"\n tf.logging.info(\" name = %s, shape = %s%s\", var.name, var.shape,\n init_string)\n\n output_spec = None\n if mode == tf.estimator.ModeKeys.TRAIN:\n\n train_op = optimization.create_optimizer(\n total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu)\n\n output_spec = tf.contrib.tpu.TPUEstimatorSpec(\n mode=mode,\n loss=total_loss,\n train_op=train_op,\n scaffold_fn=scaffold_fn)\n elif mode == tf.estimator.ModeKeys.PREDICT:\n\n output_spec = tf.contrib.tpu.TPUEstimatorSpec(\n mode=mode,\n predictions={\"probabilities\": probabilities},\n scaffold_fn=scaffold_fn)\n else:\n raise ValueError(\"Only TRAIN and PREDICT modes are supported: %s\" %\n (mode)) \n\n return output_spec\n\n return model_fn\n\n\n# This function is not used by this file but is still used by the Colab and\n# people who depend on it.\ndef input_fn_builder(features, seq_length, is_training, drop_remainder):\n \"\"\"Creates an `input_fn` closure to be passed to TPUEstimator.\"\"\"\n\n all_input_ids = []\n all_input_mask = []\n all_segment_ids = []\n all_target_start_ids = []\n all_target_end_ids = []\n all_target_present_ids = []\n all_q_mask = []\n all_c_mask = [] \n\n for feature in features:\n all_input_ids.append(feature.input_ids)\n all_input_mask.append(feature.input_mask)\n all_segment_ids.append(feature.segment_ids)\n all_target_start_ids.append(feature.target_start)\n all_target_end_ids.append(feature.target_end)\n all_target_present_ids.append(feature.target_present)\n all_q_mask.append(feature.q_mask)\n all_c_mask.append(feature.c_mask) \n \n def input_fn(params):\n \"\"\"The actual input function.\"\"\"\n batch_size = params[\"batch_size\"]\n\n num_examples = len(features)\n\n # This is for demo purposes and does NOT scale to large data sets. We do\n # not use Dataset.from_generator() because that uses tf.py_func which is\n # not TPU compatible. The right way to load data is with TFRecordReader.\n d = tf.data.Dataset.from_tensor_slices({\n \"input_ids\":\n tf.constant(\n all_input_ids, shape=[num_examples, seq_length],\n dtype=tf.int32),\n \"input_mask\":\n tf.constant(\n all_input_mask,\n shape=[num_examples, seq_length],\n dtype=tf.int32),\n \"segment_ids\":\n tf.constant(\n all_segment_ids,\n shape=[num_examples, seq_length],\n dtype=tf.int32),\n \"target_start_ids\":\n tf.constant(\n all_target_start_ids,\n shape=[num_examples, seq_length],\n dtype=tf.int32),\n \"segment_ids\":\n tf.constant(\n all_target_end_ids,\n shape=[num_examples,seq_length],\n dtype=tf.int32),\n \"segment_ids\":\n tf.constant(\n all_target_present_ids,\n shape=[num_examples, seq_length],\n dtype=tf.int32),\n })\n\n if is_training:\n d = d.repeat()\n d = d.shuffle(buffer_size=100)\n\n d = d.batch(batch_size=batch_size, drop_remainder=drop_remainder)\n return d\n\n return input_fn\n\n\n# This function is not used by this file but is still used by the Colab and\n# people who depend on it.\ndef convert_examples_to_features(examples, label_list, max_seq_length,\n tokenizer):\n \"\"\"Convert a set of `InputExample`s to a list of `InputFeatures`.\"\"\"\n\n features = []\n for (ex_index, example) in enumerate(examples):\n if ex_index % 10000 == 0:\n tf.logging.info(\"Writing example %d of %d\" % (ex_index, len(examples)))\n\n feature = convert_single_example(ex_index, example, label_list,\n max_seq_length, tokenizer)\n\n features.append(feature)\n return features\n\n\ndef main(_):\n tf.logging.set_verbosity(tf.logging.INFO)\n\n processors = {\n \"\": KeplerProcessor,\n }\n\n tokenization.validate_case_matches_checkpoint(FLAGS.do_lower_case,\n FLAGS.init_checkpoint)\n\n if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_predict:\n raise ValueError(\n \"At least one of `do_train`, `do_eval` or `do_predict' must be True.\")\n\n bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)\n\n if FLAGS.max_seq_length > bert_config.max_position_embeddings:\n raise ValueError(\n \"Cannot use sequence length %d because the BERT model \"\n \"was only trained up to sequence length %d\" %\n (FLAGS.max_seq_length, bert_config.max_position_embeddings))\n\n tf.gfile.MakeDirs(FLAGS.output_dir)\n\n task_name = FLAGS.task_name.lower()\n\n if task_name not in processors:\n raise ValueError(\"Task not found: %s\" % (task_name))\n\n processor = processors[task_name]()\n\n label_list = processor.get_labels()\n\n tokenizer = tokenization.FullTokenizer(\n vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case)\n\n tpu_cluster_resolver = None\n if FLAGS.use_tpu and FLAGS.tpu_name:\n tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(\n FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)\n\n is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2\n run_config = tf.contrib.tpu.RunConfig(\n cluster=tpu_cluster_resolver,\n master=FLAGS.master,\n model_dir=FLAGS.output_dir,\n save_checkpoints_steps=FLAGS.save_checkpoints_steps,\n tpu_config=tf.contrib.tpu.TPUConfig(\n iterations_per_loop=FLAGS.iterations_per_loop,\n num_shards=FLAGS.num_tpu_cores,\n per_host_input_for_training=is_per_host))\n\n train_examples = None\n num_train_steps = None\n num_warmup_steps = None\n if FLAGS.do_train:\n train_examples = processor.get_train_examples(FLAGS.data_dir)\n num_train_steps = int(\n len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs)\n num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion)\n\n model_fn = model_fn_builder(\n bert_config=bert_config,\n num_labels=len(label_list),\n init_checkpoint=FLAGS.init_checkpoint,\n learning_rate=FLAGS.learning_rate,\n num_train_steps=num_train_steps,\n num_warmup_steps=num_warmup_steps,\n use_tpu=FLAGS.use_tpu,\n use_one_hot_embeddings=FLAGS.use_tpu)\n\n # If TPU is not available, this will fall back to normal Estimator on CPU\n # or GPU.\n estimator = tf.contrib.tpu.TPUEstimator(\n use_tpu=FLAGS.use_tpu,\n model_fn=model_fn,\n config=run_config,\n train_batch_size=FLAGS.train_batch_size,\n eval_batch_size=FLAGS.eval_batch_size,\n predict_batch_size=FLAGS.predict_batch_size)\n\n if FLAGS.do_train:\n train_file = os.path.join(FLAGS.output_dir, \"train.tf_record\")\n file_based_convert_examples_to_features(\n train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file)\n tf.logging.info(\"***** Running training *****\")\n tf.logging.info(\" Num examples = %d\", len(train_examples))\n tf.logging.info(\" Batch size = %d\", FLAGS.train_batch_size)\n tf.logging.info(\" Num steps = %d\", num_train_steps)\n train_input_fn = file_based_input_fn_builder(\n input_file=train_file,\n seq_length=FLAGS.max_seq_length,\n is_training=True,\n drop_remainder=True)\n estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)\n\n if FLAGS.do_predict:\n predict_examples = processor.get_test_examples(FLAGS.data_dir)\n num_actual_predict_examples = len(predict_examples)\n if FLAGS.use_tpu:\n # TPU requires a fixed batch size for all batches, therefore the number\n # of examples must be a multiple of the batch size, or else examples\n # will get dropped. So we pad with fake examples which are ignored\n # later on.\n while len(predict_examples) % FLAGS.predict_batch_size != 0:\n predict_examples.append(PaddingInputExample())\n\n predict_file = os.path.join(FLAGS.output_dir, \"predict.tf_record\")\n file_based_convert_examples_to_features(predict_examples, label_list,\n FLAGS.max_seq_length, tokenizer,\n predict_file)\n\n tf.logging.info(\"***** Running prediction*****\")\n tf.logging.info(\" Num examples = %d (%d actual, %d padding)\",\n len(predict_examples), num_actual_predict_examples,\n len(predict_examples) - num_actual_predict_examples)\n tf.logging.info(\" Batch size = %d\", FLAGS.predict_batch_size)\n\n predict_drop_remainder = True if FLAGS.use_tpu else False\n predict_input_fn = file_based_input_fn_builder(\n input_file=predict_file,\n seq_length=FLAGS.max_seq_length,\n is_training=False,\n drop_remainder=predict_drop_remainder)\n\n result = estimator.predict(input_fn=predict_input_fn)\n\n output_predict_file = os.path.join(FLAGS.output_dir, \"test_results.tsv\")\n with tf.gfile.GFile(output_predict_file, \"w\") as writer:\n num_written_lines = 0\n tf.logging.info(\"***** Predict results *****\")\n for (i, prediction) in enumerate(result):\n probabilities = prediction[\"probabilities\"]\n if i >= num_actual_predict_examples:\n break\n output_line = \"\\t\".join(str(class_probability) \n for class_probability in probabilities) + \"\\n\"\n writer.write(output_line)\n num_written_lines += 1\n assert num_written_lines == num_actual_predict_examples\n\n\nif __name__ == \"__main__\":\n flags.mark_flag_as_required(\"data_dir\")\n flags.mark_flag_as_required(\"task_name\")\n flags.mark_flag_as_required(\"vocab_file\")\n flags.mark_flag_as_required(\"bert_config_file\")\n flags.mark_flag_as_required(\"output_dir\")\n tf.app.run()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}