{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n!pip install ../input/tf-20/tensorflow_gpu-2.0.0-cp36-cp36m-manylinux2010_x86_64.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/tensorflow2-question-answering","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/tf2qa-sub/tf2qa_sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/tf-115-dependencies","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n!pip install ../input/tf-115-dependencies/natural_questions-1.0.4-py2.py3-none-any.whl --no-dependencies","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = '../input/tf2qa-sub/tf2qa_sub'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n#GPU_id = '4,5,6,7,0,1,2,3'\nGPU_id = '0'\nos.environ['CUDA_VISIBLE_DEVICES'] = GPU_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nprint('tensorflow',tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ngpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n  try:\n    # Currently, memory growth needs to be the same across GPUs\n    for gpu in gpus:\n      tf.config.experimental.set_memory_growth(gpu, True)\n    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n  except RuntimeError as e:\n    # Memory growth must be set before GPUs have been initialized\n    print(e)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append(f\"{PATH}/models-2.0\")\nsys.path.append(f\"{PATH}/jb_2.0\")\nimport gzip\nimport random\nimport os\nfrom official.nlp.bert import input_pipeline\nfrom nq_lib import create_example_from_jsonl,CreateTFExampleFn,read_nq_examples\nfrom nq_lib import FeatureWriter,convert_examples_to_features,read_candidates,compute_pred_dict\nfrom nq_lib import FLAGS\nfrom run_jb import get_strategy,read_metas,predict_jb_customized\nimport tokenization\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport json\nimport os\nimport pickle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_DIR = '../input/tensorflow2-question-answering'\nMODEL_DIR='../input/tf2qa-sub/tf2qa_sub'\nBERT_FILE='../input/tf2qa-sub/tf2qa_sub/bert_config.json'\nBATCH_SIZE = 32\nCKPT = 'ctl_step_5158.ckpt-12'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Preprocess"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_test():\n  \n  \n  input_jsonl = f'{INPUT_DIR}/simplified-nq-test.jsonl'\n  output_tfrecord = 'simplified-nq-test.tfrecords'\n  meta_path = 'simplified-nq-test.meta'\n  vocab_file = f'{MODEL_DIR}/vocab-nq.txt'\n  \n  if os.path.exists(meta_path):\n      print(meta_path,'exists.')\n      return\n  tokenizer = tokenization.FullTokenizer(\n      vocab_file=vocab_file, do_lower_case=FLAGS.do_lower_case)\n\n  eval_examples = read_nq_examples(\n      input_file=input_jsonl, is_training=False)\n  eval_writer = FeatureWriter(\n      filename=output_tfrecord,\n      is_training=False)\n\n  real_examples = []\n  all_examples = []\n  \n  def append_feature(feature,is_padding=False):\n    if not is_padding:\n      real_examples.append(1)\n    eval_writer.process_feature(feature)\n    all_examples.append(1)\n    return len(real_examples)\n  \n  num_spans_to_ids = convert_examples_to_features(\n      examples=eval_examples,\n      tokenizer=tokenizer,\n      is_training=False,\n      output_fn=append_feature,\n      predict_batch_size = BATCH_SIZE)\n  eval_writer.close()\n  \n  num_orig_examples = len(eval_examples)\n  num_real_examples = len(real_examples)\n  num_all_examples = len(all_examples)\n  \n  del eval_examples,real_examples,all_examples\n  meta_data = {\n      \"task_type\": \"jb_nq\",\n      \"num_orig_examples\": num_orig_examples,\n      \"num_real_examples\": num_real_examples,\n      \"num_all_examples\": num_all_examples,\n      \"max_query_length\": FLAGS.max_query_length,\n      \"max_seq_length\": FLAGS.max_seq_length,\n      \"doc_stride\": FLAGS.doc_stride,\n  }\n\n  with tf.io.gfile.GFile(meta_path, \"w\") as writer:\n    writer.write(json.dumps(meta_data, indent=4) + \"\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ncreate_test()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_jb():\n  \n  bert_config_file= BERT_FILE\n  model_dir=MODEL_DIR\n  input_meta_data_path='simplified-nq-test.meta'\n  predict_batch_size=BATCH_SIZE\n  predict_file='simplified-nq-test.tfrecords'\n  result_path=f'sub_{CKPT}'\n  use_checkpoint=f'{MODEL_DIR}/{CKPT}' \n  strategy_type = 'mirror'\n  tpu = False\n  \n  strategy = get_strategy(strategy_type,tpu)\n  input_meta_data = read_metas(input_meta_data_path)\n  num_eval_examples = input_meta_data['num_orig_examples']\n  dataset_size = input_meta_data['num_all_examples']\n\n  print(\"***** Running predictions *****\")\n  print(\"  Num orig examples = %d\" % num_eval_examples)\n  print(\"  Num split examples = %d\" % dataset_size)\n  print(\"  Batch size = %d\" % predict_batch_size)\n\n  all_results = predict_jb_customized(strategy, input_meta_data, \n    bert_config_file,\n    predict_file,\n    model_dir,\n    use_tpu = tpu,\n    predict_batch_size = predict_batch_size,\n    use_checkpoint = use_checkpoint)\n\n  with open(f'{result_path}.p','wb') as f:\n    pickle.dump(all_results,f)\n    \n  del all_results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\npredict_jb() # BS = 32","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Post Process"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_eval_nq_dataset(file_path, seq_length):\n  \"\"\"Creates input dataset from (tf)records files for eval.\"\"\"\n  name_to_features = {\n      'unique_ids': tf.io.FixedLenFeature([], tf.int64),\n      'input_ids': tf.io.FixedLenFeature([seq_length], tf.int64),\n      'input_mask': tf.io.FixedLenFeature([seq_length], tf.int64),\n      'segment_ids': tf.io.FixedLenFeature([seq_length], tf.int64),\n      'token_map': tf.io.FixedLenFeature([seq_length], tf.int64), \n  }\n  input_fn = input_pipeline.file_based_input_fn_builder(file_path, name_to_features)\n  return input_fn()\n\ndef post_process():\n  raw_prediction_file = f'sub_{CKPT}.p'\n  predict_file = f'{INPUT_DIR}/simplified-nq-test.jsonl'\n  predict_tfrecord = 'simplified-nq-test.tfrecords'\n  \n  with open(raw_prediction_file,'rb') as f:\n    all_results = pickle.load(f)\n  candidates_dict = read_candidates(predict_file) # json file to be predicted\n  eval_features = [\n      r for r in create_eval_nq_dataset(predict_tfrecord,FLAGS.max_seq_length)\n  ]\n  print(len(eval_features),len(all_results))\n  results = [r._asdict() for c,r in enumerate(all_results) if c<len(eval_features)]\n  nq_pred_dict,nbest_summary_dict = compute_pred_dict(candidates_dict, eval_features,results)\n  output = {\"predictions\": list(nq_pred_dict.values())}\n\n  return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\noutput = post_process()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Write submission"},{"metadata":{"trusted":false},"cell_type":"code","source":"def create_short_answer(entry):\n    if entry[\"short_answers_score\"] < short_thr: \n        return \"\"\n    \n    answer = []    \n    for short_answer in entry[\"short_answers\"]:\n        if short_answer[\"start_token\"] > -1:\n            answer.append(str(short_answer[\"start_token\"]) + \":\" + str(short_answer[\"end_token\"]))\n    if entry[\"yes_no_answer\"] != \"NONE\":\n        answer.append(entry[\"yes_no_answer\"])\n    return \" \".join(answer)\n\ndef create_long_answer(entry):\n    if entry[\"long_answer_score\"] < long_thr: \n        return \"\"\n\n    answer = []\n    if entry[\"long_answer\"][\"start_token\"] > -1:\n        answer.append(str(entry[\"long_answer\"][\"start_token\"]) + \":\" + str(entry[\"long_answer\"][\"end_token\"]))\n    return \" \".join(answer)\n\ndef write_sub(test_answers_df):\n  path = f'{INPUT_DIR}/sample_submission.csv'\n  test_answers_df[\"long_answer_score\"] = test_answers_df[\"predictions\"].apply(lambda q: q[\"long_answer_score\"])\n  test_answers_df[\"short_answer_score\"] = test_answers_df[\"predictions\"].apply(lambda q: q[\"short_answers_score\"])\n\n  test_answers_df[\"long_answer\"] = test_answers_df[\"predictions\"].apply(create_long_answer)\n  test_answers_df[\"short_answer\"] = test_answers_df[\"predictions\"].apply(create_short_answer)\n  test_answers_df[\"example_id\"] = test_answers_df[\"predictions\"].apply(lambda q: str(q[\"example_id\"]))\n\n  long_answers = dict(zip(test_answers_df[\"example_id\"], test_answers_df[\"long_answer\"]))\n  short_answers = dict(zip(test_answers_df[\"example_id\"], test_answers_df[\"short_answer\"]))\n\n  sample_submission = pd.read_csv(path)\n\n  long_prediction_strings = sample_submission[sample_submission[\"example_id\"].str.contains(\"_long\")].apply(lambda q: long_answers[q[\"example_id\"].replace(\"_long\", \"\")], axis=1)\n  short_prediction_strings = sample_submission[sample_submission[\"example_id\"].str.contains(\"_short\")].apply(lambda q: short_answers[q[\"example_id\"].replace(\"_short\", \"\")], axis=1)\n\n  sample_submission.loc[sample_submission[\"example_id\"].str.contains(\"_long\"), \"PredictionString\"] = long_prediction_strings\n  sample_submission.loc[sample_submission[\"example_id\"].str.contains(\"_short\"), \"PredictionString\"] = short_prediction_strings\n\n  sample_submission.to_csv('submission.csv',index=False)\n  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"%%time\n\ndf = pd.DataFrame(output)\nlong_thr = 2.7368\nshort_thr = 6.8347\n\nwrite_sub(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"pd.read_csv('submission.csv').count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"FLAGS.__dict__","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.7"}},"nbformat":4,"nbformat_minor":1}