{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is an attempt on using Bidaf as final layer on BERT. I used code from:\n\nhttps://www.kaggle.com/negedng/bert-embeddings-with-tensorflow-2-0-example\n\nhttps://www.kaggle.com/ragnar123/exploratory-data-analysis-and-baseline\n\nhttps://www.kaggle.com/petrov/first-long-paragraph\n\nhttps://towardsdatascience.com/nlp-building-a-question-answering-model-ed0529a68c54\n\nhttps://www.kaggle.com/hiromoon166/load-bert-fine-tuning-model\n\nThe Bidaf model was simplified as follows:\nBert is not trainable. Separate bert for question and context. etc...\n\nI suggest building a separate model for YES or NO answers. Not done yet.\n\nResults not meaningful yet. Trained on only a handful samples. \n\nInterestingly the score is worse than the first long paragraph kernel, although the output should be 99% same.\n\nThere are lots of redundant packages in this kernel. The code is admittedly quite messy. It could be made way more concise."},{"metadata":{"trusted":true},"cell_type":"code","source":"startex = 95\nendex = startex + 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport sys\nimport random\nimport keras\nimport tensorflow as tf\nimport json\nimport gc\nfrom tqdm import tqdm\n\nsys.path.insert(0, '../input/pretrained-bert-including-scripts/uncased_L-12_H-768_A-12/uncased_L-12_H-768_A-12')\nsys.path.insert(0, '../input/pretrained-bert-including-scripts/master/bert-master')\n!cp -r '../input/kerasbert/keras_bert' '/kaggle/working'\n!cp -r '../input/pretrained-bert-including-scripts/uncased_L-12_H-768_A-12/uncased_L-12_H-768_A-12/vocab.txt' '/kaggle/working'\n!cp -r '../input/bert-and-bidaf/model_bidaf.h5' '/kaggle/working'\n\nsys.path.insert(0, '../input/copy-tokenization')\n!cp -r '../input/copy-tokenization/tokenization.py' '/kaggle/working'\n\nBERT_PRETRAINED_DIR = '../input/pretrained-bert-including-scripts/uncased_L-12_H-768_A-12/uncased_L-12_H-768_A-12'\nprint('***** BERT pretrained directory: {} *****'.format(BERT_PRETRAINED_DIR))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tokenization","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import Model       # Keras is the new high level API for TensorFlow\nfrom tensorflow.python.ops.rnn_cell import DropoutWrapper\nfrom tensorflow.python.ops import variable_scope as vs\nfrom tensorflow.keras.layers import GRU, Bidirectional\nfrom tensorflow.keras.layers import Layer\nfrom tensorflow.keras.layers import Dense","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras_bert.keras_bert.bert import get_model\nfrom keras_bert.keras_bert.tokenizer import Tokenizer\nfrom keras_bert.keras_bert.loader import load_trained_model_from_checkpoint\nfrom keras.optimizers import Adam\nadam = Adam(lr=2e-5,decay=0.01)\nprint('begin_build')\nconfig_file = os.path.join(BERT_PRETRAINED_DIR, 'bert_config.json')\ncheckpoint_file = os.path.join(BERT_PRETRAINED_DIR, 'bert_model.ckpt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_to_unicode(text):\n    \"\"\"Converts `text` to Unicode (if it's not already), assuming utf-8 input.\"\"\"\n    if isinstance(text, str):\n        return text\n    elif isinstance(text, bytes):\n        return text.decode(\"utf-8\", \"ignore\")\n    else:\n        raise ValueError(\"Unsupported string type: %s\" % (type(text)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import collections\nimport unicodedata\n\ndef load_vocab(vocab_file):\n    \"\"\"Loads a vocabulary file into a dictionary.\"\"\"\n    vocab = collections.OrderedDict()\n    index = 0\n    with tf.gfile.GFile(vocab_file, \"r\") as reader:\n        while True:\n            token = convert_to_unicode(reader.readline())\n            if not token:\n                break\n        token = token.strip()\n        vocab[token] = index\n        index += 1\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cont_len = 512\nques_len = 126\ndict_path = os.path.join(BERT_PRETRAINED_DIR, 'vocab.txt')\ntokenizer = tokenization.FullTokenizer(vocab_file=dict_path, do_lower_case=True)\n\n#voc_dict = load_vocab(dict_path)\n#tokenizer = Tokenizer(voc_dict)\n\ncont_model = load_trained_model_from_checkpoint(config_file, checkpoint_file, training=False,seq_len=cont_len)\nques_model = load_trained_model_from_checkpoint(config_file, checkpoint_file, training=False,seq_len=ques_len)\n\ncont_model.trainable = False\nques_model.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_masks(tokens, max_seq_length):\n    \"\"\"Mask for padding\"\"\"\n    if len(tokens)>max_seq_length:\n        print(\"Token length more than max seq length!\")\n        return max_seq_length*[1]\n    return [1]*len(tokens) + [0] * (max_seq_length - len(tokens))\n\n\ndef get_segments(tokens, max_seq_length):\n    \"\"\"Segments: 0 for the first sequence, 1 for the second\"\"\"\n    if len(tokens)>max_seq_length:\n        #raise IndexError(\"Token length more than max seq length!\")\n        return max_seq_length*[1]\n    segments = []\n    current_segment_id = 0\n    for token in tokens:\n        segments.append(current_segment_id)\n        if token == \"[SEP]\":\n            current_segment_id = 1\n    return segments + [0] * (max_seq_length - len(tokens))\n\n\ndef convert_by_vocab(vocab, items):\n    \"\"\"Converts a sequence of [tokens|ids] using the vocab.\"\"\"\n    output = []\n    for item in items:\n        output.append(vocab[item])\n    return output\n\ndef get_ids(tokens, tokenizer, max_seq_length):\n    \"\"\"Token ids from Tokenizer vocab\"\"\"\n    token_ids = tokenizer.convert_tokens_to_ids(tokens)\n    if len(tokens)>max_seq_length:\n        return token_ids[:max_seq_length]\n    input_ids = token_ids + [0] * (max_seq_length-len(token_ids))\n    return input_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '/kaggle/input/tensorflow2-question-answering/'\ntrain_path = 'simplified-nq-train.jsonl'\ntest_path = 'simplified-nq-test.jsonl'\nsample_submission_path = 'sample_submission.csv'\n\ndef read_data(path, sample = True, chunksize = 30000):\n    if sample == True:\n        df = []\n        with open(path, 'rt') as reader:\n            for i in range(chunksize):\n                df.append(json.loads(reader.readline()))\n        df = pd.DataFrame(df)\n        print('Our sampled dataset have {} rows and {} columns'.format(df.shape[0], df.shape[1]))\n    else:\n        df = pd.read_json(path, orient = 'records', lines = True)\n        print('Our dataset have {} rows and {} columns'.format(df.shape[0], df.shape[1]))\n        gc.collect()\n    return df\n\ntrain = read_data(path+train_path, sample = True)\ntest = read_data(path+test_path, sample = False)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class QA_Layer_partA(Layer):\n    def __init__(self,**kwargs):\n        super(QA_Layer_partA, self).__init__(**kwargs)\n        self.supports_masking = False\n        init_op = tf.global_variables_initializer\n    def build(self, input_shape):\n        #inputs: [q, qmask, c, cmask] in that order\n        #input shapes: q.shape = (batch,seq_len,emb_size) = (batch, ques_len, 768)\n        print(input_shape)\n        self.S_W = self.add_weight(name = 'S_W', shape = (768*3, ), initializer='uniform', trainable=True)\n        init_op = tf.global_variables_initializer\n        super(QA_Layer_partA, self).build(input_shape)\n        \n    def call(self, ccmqqm):\n        #input must be a list of four list(ccmqqm):\n        #cont_embs, cont_mask, ques_embs, ques_mask     \n        c = ccmqqm[0]\n        c_mask = ccmqqm[1]\n        q = ccmqqm[2]\n        q_mask = ccmqqm[3]\n        \n        # Calculating similarity matrix\n        c_expand = tf.expand_dims(c,2)  #[batch,N,1,2h] ; 2h = bert emb size\n        q_expand = tf.expand_dims(q,1)  #[batch,1,M,2h]\n        c_pointWise_q = c_expand * q_expand  #[batch,N,M,2h]\n\n        c_input = tf.tile(c_expand, [1, 1, tf.shape(q)[1], 1])\n        q_input = tf.tile(q_expand, [1, tf.shape(c)[1], 1, 1])\n\n        concat_input = tf.concat([c_input, q_input, c_pointWise_q], -1) # [batch,N,M,6h]\n        print(\"concat_in\", concat_input.shape)\n        similarity=tf.reduce_sum(concat_input * self.S_W, axis=-1)  #[batch,N,M]\n        print(\"similarity\", similarity.shape)\n        # Calculating context to question attention\n        similarity_mask = tf.expand_dims(q_mask, 1) # shape (batch_size, 1, M)\n        print(\"sim mask\", similarity_mask.shape)\n        exp_mask_c2q = (1 - tf.cast(similarity_mask, 'float')) * (-1e30) # -large where there's padding, 0 elsewhere\n        print(\"exp mask\", exp_mask_c2q.shape)\n        masked_logits_c2q = tf.add(similarity, exp_mask_c2q) # where there's padding, set logits to -large\n        c2q_dist = tf.nn.softmax(masked_logits_c2q, 1)  # dim = 1\n                \n        # Use attention distribution to take weighted sum of values\n        c2q = tf.matmul(c2q_dist, q) # shape (batch_size, N, vec_size)\n        \n        # Calculating question to context attention c_dash\n        S_max = tf.reduce_max(similarity, axis=2) # shape (batch, N)\n\n        exp_mask_S = (1 - tf.cast(c_mask, 'float')) * (-1e30) # -large where there's padding, 0 elsewhere\n        masked_logits_S = tf.add(S_max, exp_mask_S) # where there's padding, set logits to -large\n        c_dash_dist = tf.nn.softmax(masked_logits_S, 1)  # dim = 1\n        \n        c_dash_dist_expand = tf.expand_dims(c_dash_dist, 1) # shape (batch, 1, N)\n        c_dash = tf.matmul(c_dash_dist_expand, c) # shape (batch_size, 1, vec_size)\n        \n        c_c2q = c * c2q # shape (batch, N, vec_size)\n        print(\"c_c2q\", c_c2q.shape)\n        c_c_dash = c * c_dash # shape (batch, N, vec_size)\n        print(\"cdash\", c_c_dash.shape)\n        # concatenate the output\n        output = tf.concat([c2q, c_c2q, c_c_dash], axis=2) # (batch_size, N, vec_size * 3)\n\n        # Apply dropout\n        attn_output = tf.nn.dropout(output, 0.9)\n\n        blended_reps = tf.concat([c, attn_output], axis=2) #attn out has len same as c_mask\n        print(\"blended\", blended_reps.shape)\n        \n        return blended_reps\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class QA_Layer_partB(Layer):\n    def __init__(self,**kwargs):\n        super(QA_Layer_partB, self).__init__(**kwargs)\n        self.supports_masking = False\n    def build(self, input_shape):\n        #inputs: [logits, cmask] in that order\n        #input shapes: q.shape = (batch,seq_len,emb_size) = (batch, ques_len, 768)\n        #input shapes: q.shape = (batch,seq_len,emb_size) = (batch, cont_len, 768)\n        print(input_shape)\n        super(QA_Layer_partB, self).build(input_shape)\n    def call(self, lcm):\n        #input must be a list of two list(logits, cmask):\n        logits_start = lcm[0] \n        c_mask = lcm[1]\n        logits_start = tf.squeeze(logits_start, axis=[2]) # shape (batch_size, seq_len)\n        exp_mask_start = (1 - tf.cast(c_mask, 'float')) * (-1e30) # -large where there's padding, 0 elsewhere\n        masked_logits_start = tf.add(logits_start, exp_mask_start) # where there's padding, set logits to -large\n        start_prob = tf.nn.softmax(masked_logits_start, 1)  # dim = 1\n        return start_prob","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model for short sentences.\n#Xplainin: So, this is a quite simplified version of: \n#https://github.com/priya-dwivedi/cs224n-Squad-Project/blob/master/code/modules.py\n#Priya explains that it's better to make bert trainable, add ques and context together and feed as such.\n#Here we don't. But again, we are using bert embs, which is hopefully better than their glove embs. We'll see.\n\ncont_embs = tf.keras.layers.Input(shape=(cont_len, 768, ), dtype=tf.float32,\n                                       name=\"cont_embs\")\ncont_masks = tf.keras.layers.Input(shape=(cont_len,), dtype=tf.int32,\n                                   name=\"cont_masks\")\nques_embs = tf.keras.layers.Input(shape=(ques_len, 768, ), dtype=tf.float32,\n                                       name=\"ques_embs\")\nques_masks = tf.keras.layers.Input(shape=(ques_len,), dtype=tf.int32,\n                                   name=\"ques_masks\")\n\nbidaf_model_partA = QA_Layer_partA()\nbidaf_startlogits = QA_Layer_partB()\nbidaf_endlogits = QA_Layer_partB()\nbidaf_presentlogits = QA_Layer_partB()\n\nblended_reps = bidaf_model_partA([cont_embs, cont_masks, ques_embs, ques_masks])\n\nblended_reps_final = Bidirectional(GRU(128, kernel_initializer = 'glorot_uniform', return_sequences = True, dropout = 0.1))(blended_reps)\nblended_reps_final = Bidirectional(GRU(128, kernel_initializer = 'glorot_uniform', return_sequences = True, dropout = 0.1))(blended_reps_final)\n\nlogits_start = GRU(1, kernel_initializer = 'glorot_uniform', return_sequences = True, dropout = 0.1)(blended_reps_final)\nlogits_end = GRU(1, kernel_initializer = 'glorot_uniform', return_sequences = True, dropout = 0.1)(blended_reps_final)\nlogits_present = GRU(1, kernel_initializer = 'glorot_uniform', return_sequences = True, dropout = 0.1)(blended_reps_final)\n\nstart_prob = bidaf_startlogits([logits_start, cont_masks])\nend_prob = bidaf_endlogits([logits_end, cont_masks])\npresent_prob = bidaf_presentlogits([logits_present, cont_masks])\n\nlist_of_inputs = [cont_embs, cont_masks, ques_embs, ques_masks]\ndist_probs = [start_prob, end_prob, present_prob]\n\nmodel_bidaf = Model(inputs=list_of_inputs, outputs=dist_probs)\n\nprint(model_bidaf.summary())\n\nmodel_bidaf.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model\n\ncont_embs_output  = cont_model.layers[-6].output\ncont_bert = Model(inputs=cont_model.input, outputs=cont_embs_output)\ncont_bert.compile(loss='binary_crossentropy', optimizer=adam)\ncont_bert.summary()\n\nques_embs_output  = ques_model.layers[-6].output\nques_bert = Model(inputs=ques_model.input, outputs=ques_embs_output)\nques_bert.compile(loss='binary_crossentropy', optimizer=adam)\nques_bert.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_bidaf.load_weights('../input/bert-and-bidaf/model_bidaf.h5') #works only with cpu","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#target present is an auxilliary target. not used for predictions.\n\nwith tf.Session() as sess:\n    sess.run(tf.global_variables_initializer())\n     \n    for i_main in range(startex,endex):\n        \n        sample_weight = []\n\n        row = train.iloc[i_main]\n\n        if i_main == 3:\n            print('row no ', i_main)\n            print('annotation short ans', row['annotations'][0]['short_answers'])\n        document_text = row['document_text'].split()\n        question_text = row['question_text']\n\n        ques_ans_tokens = tokenizer.tokenize(question_text)\n\n        ques_ids = np.asarray([get_ids(ques_ans_tokens, tokenizer, ques_len)])\n        ques_masks = np.asarray([get_masks(ques_ans_tokens, ques_len)])\n\n        batch_of_cont_embs = []\n        batch_of_cont_masks = []\n        batch_of_ques_embs = []\n        batch_of_ques_masks = []\n\n        batch_of_target_st = []\n        batch_of_target_en = []\n        batch_of_target_pr = []\n\n        for candidate_no, long_answer_candidate in enumerate(row['long_answer_candidates']):\n\n            target_start = [0] * cont_len\n            target_end = [0] * cont_len\n            target_present = [0] * cont_len\n\n            long_ans_start_tok = long_answer_candidate['start_token']\n            long_ans_end_tok = long_answer_candidate['end_token']\n            long_sentence = \" \".join(document_text[long_ans_start_tok:long_ans_end_tok])\n\n            if long_ans_start_tok == row['annotations'][0]['long_answer']['start_token'] and \\\n                len(row['annotations'][0]['short_answers']) > 0:\n\n                #print(\"this is correct long answer\")\n\n                short_answer_start_token = row['annotations'][0]['short_answers'][0]['start_token']\n                short_answer_end_token = row['annotations'][0]['short_answers'][0]['end_token']\n                short_start_idx = short_answer_start_token-long_ans_start_tok\n                short_end_idx = short_answer_end_token-long_ans_start_tok\n\n                if short_end_idx < cont_len:\n                    target_start[short_start_idx] = 1\n                    target_end[short_end_idx] = 1\n                    sample_weight.append(900)\n                    for i in range(short_start_idx,short_end_idx):\n                        target_present[i] = 1\n                else:\n                    print(\"short answer beyond maximum len\")\n                    sample_weight.append(1)\n            else:\n                sample_weight.append(1)\n\n            long_ans_tokens = tokenizer.tokenize(long_sentence)\n            long_ids = np.asarray([get_ids(long_ans_tokens, tokenizer, cont_len)])\n            cont_masks = np.asarray([get_masks(long_ans_tokens, cont_len)])\n\n            cont_embs = cont_bert.predict([long_ids,cont_masks],verbose=1,batch_size=1)\n            ques_embs = ques_bert.predict([ques_ids,ques_masks],verbose=1,batch_size=1)\n\n            batch_of_cont_embs.append(np.squeeze(cont_embs))\n            batch_of_cont_masks.append(np.squeeze(cont_masks))\n            batch_of_ques_embs.append(np.squeeze(ques_embs))\n            batch_of_ques_masks.append(np.squeeze(ques_masks))\n\n            batch_of_target_st.append(target_start)\n            batch_of_target_en.append(target_end)\n            batch_of_target_pr.append(target_present)\n\n            #we will train after every line due to the massive size of embs.\n        \n        sample_weight = np.asarray(sample_weight)\n        \n        xcont_embs = np.asarray(batch_of_cont_embs)\n        xcont_masks = np.asarray(batch_of_cont_masks)\n        xques_embs = np.asarray(batch_of_ques_embs)\n        xques_masks = np.array(batch_of_ques_masks)\n\n        print(xcont_embs.shape)\n        print(xques_embs.shape)\n\n        ytarget_st = np.asarray(batch_of_target_st)\n        ytarget_en = np.asarray(batch_of_target_en)\n        ytarget_pr = np.asarray(batch_of_target_pr)\n\n        train_x = [xcont_embs, xcont_masks, xques_embs, xques_masks]\n        train_y = [ytarget_st, ytarget_en, ytarget_pr]\n\n\n        model_bidaf.fit(train_x, train_y, batch_size=1, epochs=1,sample_weight=[sample_weight,sample_weight,sample_weight])\n        model_bidaf.save_weights(\"model_bidaf.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def bidaf_pred(row):\n    \n    threshold = 0.01 #random selection\n\n    document_text = row['document_text'].split()\n    question_text = row['question_text']\n    \n    ques_ans_tokens = tokenizer.tokenize(question_text)\n\n    ques_ids = np.asarray([get_ids(ques_ans_tokens, tokenizer, ques_len)])\n    ques_masks = np.asarray([get_masks(ques_ans_tokens, ques_len)])\n    \n    highest_combined_score = 0\n    best_short_tokens = (0,0)\n    best_long_tokens = (0,0)\n    \n    for candidate_no, long_answer_candidate in enumerate(row['long_answer_candidates']):\n\n        long_ans_start_tok = long_answer_candidate['start_token']\n        long_ans_end_tok = long_answer_candidate['end_token']\n        long_sentence = \" \".join(document_text[long_ans_start_tok:long_ans_end_tok])\n\n        batch_of_cont_embs = []\n        batch_of_cont_masks = []\n        batch_of_ques_embs = []\n        batch_of_ques_masks = []\n\n        cont_embs = cont_bert.predict([long_ids,cont_masks],verbose=1,batch_size=1)\n        ques_embs = ques_bert.predict([ques_ids,ques_masks],verbose=1,batch_size=1)\n\n        batch_of_cont_embs.append(np.squeeze(cont_embs))\n        batch_of_cont_masks.append(np.squeeze(cont_masks))\n        batch_of_ques_embs.append(np.squeeze(ques_embs))\n        batch_of_ques_masks.append(np.squeeze(ques_masks))\n    \n        xcont_embs = np.asarray(batch_of_cont_embs)\n        xcont_masks = np.asarray(batch_of_cont_masks)\n        xques_embs = np.asarray(batch_of_ques_embs)\n        xques_masks = np.array(batch_of_ques_masks)\n        \n        output = model_bidaf.predict([xcont_embs, xcont_masks, xques_embs, xques_masks], batch_size = 1)\n    \n        start_pred_scores = output[0]\n        end_pred_scores = output[1]\n        present_pred_scores = output[2]\n\n        start_tok_pred = np.argmax(start_pred_scores, axis=1)[0]\n        end_tok_pred = np.argmax(end_pred_scores, axis=1)[0]\n        present_tok_pred = np.argmax(present_pred_scores, axis=1)[0]\n\n        start_pred_score = start_pred_scores[0][start_tok_pred]\n        end_pred_score = end_pred_scores[0][end_tok_pred]\n        present_pred_score = present_pred_scores[0][present_tok_pred]\n        \n        #print(start_pred_score)\n        #print(start_tok_pred)\n        \n        if start_pred_score + end_pred_score > highest_combined_score and start_pred_score > threshold and end_pred_score > threshold:\n            best_short_tokens = (start_tok_pred + long_ans_start_tok, end_tok_pred + long_ans_start_tok)\n            best_long_tokens = (long_ans_start_tok, long_ans_end_tok)\n            print(\"found one!\")\n\n    return best_short_tokens, best_long_tokens\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_TOTAL = 346\n\ndef get_joined_tokens(answer: dict) -> str:\n    return '%d:%d' % (answer['start_token'], answer['end_token'])\n\ndef get_pred(json_data: dict, count) -> dict:\n    ret = {'short': 'YES', 'long': ''}\n    candidates = json_data['long_answer_candidates']\n    \n    paragraphs = []\n    tokens = json_data['document_text'].split(' ')\n    for cand in candidates:\n        start_token = tokens[cand['start_token']]\n        if start_token == '<P>' and cand['top_level'] and cand['end_token']-cand['start_token']>35:\n            break\n    else:\n        cand = candidates[0]\n    \n    ret['long'] = get_joined_tokens(cand)\n\n    best_short_tokens = (0,0)\n    best_long_tokens = (0,0)\n    \n    if count < 100: #it takes a long time, so we just show few samples here.\n        best_short_tokens, best_long_tokens = bidaf_pred(json_data)\n\n    #if bidaf doesn't return good pred, it falls back to the first paragraph method.\n    if best_short_tokens != (0,0):\n        ret['short'] = '%d:%d' % (best_short_tokens[0], best_short_tokens[1])\n        ret['long'] = '%d:%d' % (best_long_tokens[0], best_long_tokens[1])\n    \n    id_ = str(json_data['example_id'])\n    ret = {id_+'_'+k: v for k, v in ret.items()} \n    return ret\n\npreds = dict()\n\nwith open(path + test_path, 'r') as f:\n    count = 0\n    for line in tqdm(f, total=TEST_TOTAL):\n        count += 1\n        json_data = json.loads(line) \n        prediction = get_pred(json_data, count)\n        preds.update(prediction)\n            \nsubmission = pd.read_csv(path + 'sample_submission.csv')\nsubmission['PredictionString'] = submission['example_id'].map(lambda x: preds[x])\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}