{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport sys\nsys.path.append('../input/bert-joint-baseline/')\nimport bert_utils\nimport tokenization\n\nimport os\non_kaggle_server = os.path.exists('/kaggle')\nif not on_kaggle_server:\n    sys.path.append('../input/preprocess/')\nimport bert_preprocess as preprocess\nimport json\nimport collections\nimport itertools\nimport tqdm\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nq_test_file = '../input/tensorflow2-question-answering/simplified-nq-test.jsonl'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\neval_records='nq-test_v1.tfrecords'\nif True or not os.path.exists(eval_records):\n    eval_writer = bert_utils.FeatureWriter(\n        filename=os.path.join(eval_records),\n        is_training=False)\n\n    tokenizer = tokenization.FullTokenizer(vocab_file='../input/bert-joint-baseline/vocab-nq.txt', \n                                           do_lower_case=True)\n\n    features = []\n    convert = bert_utils.ConvertExamples2Features(tokenizer=tokenizer,\n                                                  is_training=False,\n                                                  output_fn=eval_writer.process_feature,\n                                                  collect_stat=False)\n\n    n_examples = 0\n    tqdm_notebook= None\n    for examples in bert_utils.nq_examples_iter(input_file=nq_test_file, \n                                                tqdm=tqdm_notebook,\n                                                is_training=False):\n        for example in examples:\n            n_examples += convert(example)\n\n    eval_writer.close()\n    print('number of test examples: %d, written to file: %d' % (n_examples,eval_writer.num_features))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\neval_records='nq-test_v2.tfrecords'\nif True or not os.path.exists(eval_records):\n    eval_writer = preprocess.FeatureWriter(filename=os.path.join(eval_records),is_training=False)\n\n    tokenizer = tokenization.FullTokenizer(vocab_file='../input/bert-joint-baseline/vocab-nq.txt', \n                                           do_lower_case=True)\n\n    convert = preprocess.JSON2Features(tokenizer=tokenizer)\n\n    tqdm_notebook= None # tqdm.tqdm_notebook\n    for line in preprocess.file_iter(input_file=nq_test_file, tqdm=tqdm_notebook):\n        examples = convert(line)\n        for example in examples:\n            eval_writer.process_feature(example)\n\n    eval_writer.close()\n    print('number of test examples written to file: %d' % eval_writer.num_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_dataset(file_name):\n    raw_data = tf.data.TFRecordDataset(file_name)\n    seq_length = 512 \n    name_to_features = {\n        \"unique_id\":   tf.io.FixedLenFeature([],           tf.int64),\n        \"input_ids\":   tf.io.FixedLenFeature([seq_length], tf.int64),\n        \"input_mask\":  tf.io.FixedLenFeature([seq_length], tf.int64),\n        \"segment_ids\": tf.io.FixedLenFeature([seq_length], tf.int64),\n        \"token_map\":   tf.io.FixedLenFeature([seq_length], tf.int64),\n    }\n\n    def _decode_record(record, name_to_features=name_to_features):\n        \"\"\"Decodes a record to a TensorFlow example.\"\"\"\n        example = tf.io.parse_single_example(serialized=record, features=name_to_features)\n\n        # tf.Example only supports tf.int64, but the TPU only supports tf.int32.\n        # So cast all int64 to int32.\n        for name in list(example.keys()):\n            t = example[name]\n            if name != 'unique_id': #t.dtype == tf.int64:\n                t = tf.cast(t, dtype=tf.int32)\n            example[name] = t\n\n        return example\n    decoded_data = raw_data.map(_decode_record)\n    return list(decoded_data)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%time data1 = read_dataset('nq-test_v1.tfrecords')\n%time data2 = read_dataset('nq-test_v2.tfrecords')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def neq(d1,d2,return_key=False):\n    if d1.keys()!=d2.keys(): return True\n    for k,v in d1.items():\n        if (v.numpy()!=d2[k].numpy()).any():\n            if return_key:\n                return k\n            return True\n    return False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d1_neq_d2 = np.array([neq(d1,d2) for d1,d2 in zip(data1,data2)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = d1_neq_d2.nonzero()[0]\nidx","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}