{"cells":[{"metadata":{},"cell_type":"markdown","source":"This version uses the WTA (Winner takes all) summary both for short and long answers."},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\n\nimport sys\nsys.path.append('../input/bert-baseline-pre-and-post-process/')\n\nimport preprocessv5 as preprocess\nimport postprocessv6 as postprocess\nimport to_pklv5 as to_pkl\nimport pkl_to_tfrecordsv5 as pkl_to_tfrecords\n\nimport json\nimport tqdm\n\nimport absl\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.getpid()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use_wta = True\n\non_kaggle_server = os.path.exists('/kaggle')\nnq_test_file = '../input/tensorflow2-question-answering/simplified-nq-test.jsonl' \npublic_dataset = os.path.getsize(nq_test_file)<20_000_000\nprivate_dataset = os.path.getsize(nq_test_file)>=20_000_000\nmodel_path = '../input/tpu-2020-01-22/'\n\nfor k in ['on_kaggle_server','nq_test_file','public_dataset','private_dataset']:\n    print(k,globals()[k],sep=': ')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.saved_model.load(model_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"to_pkl.jsonl_to_pkl(source=nq_test_file,output='features.pkl',\n                vocab=model_path +'assets/vocab-nq.txt',\n                max_contexts=-1,lower_case=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pkl_to_tfrecords._convert(source='features.pkl',output='all.tfrecords',meta_data='meta_data',shuffle=False,shuffle_size=0,yield_segment_variant='nolabels')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def input_fn(input_file_pattern,seq_length=512,batch_size=4):\n    def mk_labels(ex):          \n        qlen = ex.pop('question_len')\n        dlen = ex.pop('data_len')\n        input_mask = tf.sequence_mask(dlen,seq_length,dtype=tf.int32)\n        ex['input_mask']  = input_mask\n        ex['segment_ids'] = tf.minimum(input_mask,1-tf.sequence_mask(qlen,seq_length,dtype=tf.int32))\n        return ex\n\n    name_to_features = {\n        'input_ids'   : tf.io.FixedLenFeature([seq_length], tf.int64),\n        'question_len': tf.io.FixedLenFeature([], tf.int64),\n        'data_len'    : tf.io.FixedLenFeature([], tf.int64),\n    }\n    name_to_features['unique_id']   = tf.io.FixedLenFeature([2], tf.int64)\n\n    def decode(record):\n        ex = tf.io.parse_single_example(record, name_to_features)\n        for k,v in ex.items():\n            if k!='unique_id':\n                ex[k] = tf.cast(v,tf.int32)\n        return ex\n\n    input_files = tf.io.gfile.glob(input_file_pattern)        \n    d = tf.data.TFRecordDataset(input_files)\n    d = d.map(decode)\n    d = d.batch(batch_size,drop_remainder=False)\n    #d = d.map(mk_labels)\n    d = d.prefetch(128)\n    return d\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def output_fn():\n    def _output_fn(unique_id,model_output,n_keep=100):\n        pos_logits,ans_logits,long_mask,short_mask,cross = model_output\n\n        long_span_logits =  pos_logits\n        mask = tf.cast(tf.expand_dims(long_mask,-1),long_span_logits.dtype)\n\n        long_span_logits = long_span_logits-10000*mask \n        long_p = tf.nn.softmax(long_span_logits,axis=1)\n\n        short_span_logits = pos_logits\n        short_span_logits -= 10000*tf.cast(tf.expand_dims(short_mask,-1),short_span_logits.dtype)\n        start_logits,end_logits = short_span_logits[:,:,0],short_span_logits[:,:,1]\n\n        batch_size,seq_length = short_span_logits.shape[0],short_span_logits.shape[1]\n        seq = tf.range(seq_length)\n        i_leq_j_mask = tf.cast(tf.expand_dims(seq,1)>tf.expand_dims(seq,0),short_span_logits.dtype)\n        i_leq_j_mask = tf.expand_dims(i_leq_j_mask,0)\n\n        logits  = tf.expand_dims(start_logits,2)+tf.expand_dims(end_logits,1)+cross\n        logits -= 10000*i_leq_j_mask\n        logits  = tf.reshape(logits, [batch_size,seq_length*seq_length])\n        short_p = tf.nn.softmax(logits)\n        indices = tf.argsort(short_p,axis=1,direction='DESCENDING')[:,:n_keep]\n        short_p = tf.gather(short_p,indices,batch_dims=1)\n\n        return dict(unique_id = unique_id,\n                    ans_logits= ans_logits,\n                    long_p    = long_p,\n                    short_p   = short_p,\n                    short_p_indices = indices)\n    return _output_fn\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d = input_fn('all.tfrecords',batch_size=64) \nif public_dataset:\n    d = d.take(3)\nif not on_kaggle_server:\n    d = tqdm.notebook.tqdm(d)\nresults = []\noutput = output_fn() \nfor b in d:\n    unique_id = b.pop('unique_id').numpy()\n    b = [b['data_len'],b['input_ids'],b['question_len']]\n    # print(b.keys())\n    #pos_logits,ans_logits,mask_0,mask_1 = \n    out_dict = output(unique_id,model(b,training=False))\n    for k,v in out_dict.items():\n            if isinstance(v,tf.Tensor):\n                out_dict[k] = v.numpy()\n    results.append(out_dict)\n\nraw_results = postprocess.read_rawresult(results)\n#    pos_logits,ans_logits = pos_logits.numpy(),ans_logits.numpy()\n#    result = postprocess.to_rawresult(unique_id=unique_id,\n#                                      pos_logits=pos_logits,\n#                                      ans_logits=ans_logits)\n#    raw_results.extend([postprocess.RawResult(*x) for x in zip(*result)])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iterator = postprocess.pickle_iter('features.pkl')\nif not on_kaggle_server:\n    iterator = tqdm.notebook.tqdm(iterator)\nrecords = postprocess.read_features(iterator)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"examples = postprocess.compute_examples(raw_results,records)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#e2p = postprocess.ExampleToProb(keep_threshold=0.1,null_prob_threshold=1e-4)\nSummary = postprocess.WTASummary #if use_wta else postprocessv3.ProbSummary\nsummary = Summary(min_vote_prob=0.1)\npredictions = [summary(e) for e in tqdm.notebook.tqdm(examples)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"index = pd.read_csv('../input/tensorflow2-question-answering/sample_submission.csv').example_id\n\nsubmission = postprocess.create_submission_df(predictions,index=index,\n                                                long_threshold=0.94 if use_wta else 0.77,\n                                                short_threshold=0.94 if use_wta else 0.77 ,\n                                                yes_no_threshold=0.6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## ! head submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head(10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}