{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Handle data","metadata":{}},{"cell_type":"code","source":"!pip install -q transformers","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:26:26.615098Z","iopub.execute_input":"2021-10-25T08:26:26.615581Z","iopub.status.idle":"2021-10-25T08:26:56.298121Z","shell.execute_reply.started":"2021-10-25T08:26:26.615459Z","shell.execute_reply":"2021-10-25T08:26:56.296828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Import**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport sys\nimport random\nfrom tqdm import tqdm\nimport re\nimport string\nimport os\nimport shutil\nimport json\nfrom transformers import AutoTokenizer, TFBertMainLayer, TFBertForPreTraining, BertConfig, TFBertModel\nimport tensorflow as tf\nfrom tensorflow.keras.losses import sparse_categorical_crossentropy as sce","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:27:17.034518Z","iopub.execute_input":"2021-10-25T08:27:17.034889Z","iopub.status.idle":"2021-10-25T08:27:24.86189Z","shell.execute_reply.started":"2021-10-25T08:27:17.034854Z","shell.execute_reply":"2021-10-25T08:27:24.860717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f_test = '../input/tensorflow2-question-answering/simplified-nq-test.jsonl'\nf_train = '../input/tensorflow2-question-answering/simplified-nq-train.jsonl'\nnum_train_samples = 307372\nnum_test_samples = 346","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:27:24.863783Z","iopub.execute_input":"2021-10-25T08:27:24.864206Z","iopub.status.idle":"2021-10-25T08:27:24.869792Z","shell.execute_reply.started":"2021-10-25T08:27:24.864161Z","shell.execute_reply":"2021-10-25T08:27:24.868532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_id_df(filename=f_test):\n    list_id = []\n    with open(filename) as f:\n        progress = tqdm(f)  \n        for sam_count, line in enumerate(progress):\n            data = json.loads(line)\n            example_id = str(data['example_id'])\n            doc = {'example_id':example_id}\n            list_id.append(doc)\n    list_id_df = pd.DataFrame(list_id)\n    return list_id_df ","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:27:25.616229Z","iopub.execute_input":"2021-10-25T08:27:25.616649Z","iopub.status.idle":"2021-10-25T08:27:25.623442Z","shell.execute_reply.started":"2021-10-25T08:27:25.616617Z","shell.execute_reply":"2021-10-25T08:27:25.622184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AnswerType = {\n    'NO_ANSWER': 0,\n    'YES': 1,\n    'NO': 2,\n    'SHORT' : 3,\n    'LONG' : 4\n}\n\nAnswerTypeRev = {\n    0: 'NO_ANSWER',\n    1: 'YES',\n    2: 'NO',\n    3: 'SHORT',\n    4: 'LONG'\n}","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:27:28.722892Z","iopub.execute_input":"2021-10-25T08:27:28.72334Z","iopub.status.idle":"2021-10-25T08:27:28.729751Z","shell.execute_reply.started":"2021-10-25T08:27:28.723308Z","shell.execute_reply":"2021-10-25T08:27:28.727757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_data(data, tokenizer, debug=False): \n    progress = tqdm(data, total=len(data))\n    x1 = []\n    x2 = []\n    x3 = []\n    y = []\n    for sam in progress:\n        tokenized_sam = tokenizer.encode_plus(sam['question'], sam['context'], \n                                              padding='max_length',\n                                              truncation=True,\n                                              max_length=512,\n                                              add_special_tokens=True)\n        \n        x1.append(tf.cast(tokenized_sam['input_ids'], tf.int32))\n        x2.append(tf.cast(tokenized_sam['token_type_ids'], tf.int32))\n        x3.append(tf.cast(tokenized_sam['attention_mask'], tf.int32))\n\n        y.append([sam['start'], sam['stop'], AnswerType[sam['target']]])\n\n    x1 = tf.convert_to_tensor(x1)\n    x2 = tf.convert_to_tensor(x2)\n    x3 = tf.convert_to_tensor(x3)\n\n    y = tf.convert_to_tensor(y)\n    return x1, x2, x3, y","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:27:33.001611Z","iopub.execute_input":"2021-10-25T08:27:33.001991Z","iopub.status.idle":"2021-10-25T08:27:33.011388Z","shell.execute_reply.started":"2021-10-25T08:27:33.001944Z","shell.execute_reply":"2021-10-25T08:27:33.010248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_strategy():\n    try:\n        tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n        print('Running on TPU ', tpu_cluster_resolver.cluster_spec().as_dict()['worker'])\n        tf.config.experimental_connect_to_cluster(tpu_cluster_resolver)\n        tf.tpu.experimental.initialize_tpu_system(tpu_cluster_resolver)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu_cluster_resolver)\n    except ValueError as e:\n        print(e)\n        print('No TPU detected')\n        tpu = None\n        strategy = tf.distribute.get_strategy()\n    return strategy","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:27:34.711215Z","iopub.execute_input":"2021-10-25T08:27:34.711636Z","iopub.status.idle":"2021-10-25T08:27:34.718889Z","shell.execute_reply.started":"2021-10-25T08:27:34.711604Z","shell.execute_reply":"2021-10-25T08:27:34.717385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mergeInstanceResult(test_res, list_test_ins):\n    for i in range(len(list_test_ins)):\n        ins_res = test_res[i]\n        start = np.argmax(ins_res[0])\n        stop = np.argmax(ins_res[1])\n        target = np.argmax(ins_res[2])\n\n        start_score = ins_res[0][start]\n        stop_score = ins_res[1][stop]\n        target_score = ins_res[2][target]\n\n        start_CLS = ins_res[0][0]\n        stop_CLS = ins_res[1][0]\n\n\n        list_test_ins[i]['start'] = start \n        list_test_ins[i]['stop'] = stop\n        list_test_ins[i]['target'] = target \n\n        list_test_ins[i]['start_score'] = start_score\n        list_test_ins[i]['stop_score'] = stop_score\n        list_test_ins[i]['target_score'] = target_score\n\n        list_test_ins[i]['start_CLS'] = start_CLS\n        list_test_ins[i]['stop_CLS'] = stop_CLS\n    return list_test_ins","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:27:38.644455Z","iopub.execute_input":"2021-10-25T08:27:38.645012Z","iopub.status.idle":"2021-10-25T08:27:38.66287Z","shell.execute_reply.started":"2021-10-25T08:27:38.644964Z","shell.execute_reply":"2021-10-25T08:27:38.661889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mergeDocumentRes(ins_df, val_id_df, threshold=0.0001, stride=128, debug=False):\n    STRIDE = stride\n    list_doc_lan = []\n    for idx, doc in val_id_df.iterrows():\n        doc_id = doc['example_id']\n        ins_of_doc = ins_df.loc[ins_df['example_id'] == doc_id]\n        \n        start_ins = ins_of_doc.loc[ins_of_doc['start'] != 0]\n        stop_ins = ins_of_doc.loc[ins_of_doc['stop'] != 0]\n        all_non_zero = pd.concat([start_ins,stop_ins]).drop_duplicates()\n        \n        best_start = -1\n        best_stop = -1\n        best_target = 0\n        best_score = threshold\n                    \n        for idx_ins, ins in all_non_zero.iterrows():\n            ins_start = int(ins['start'])\n            ins_stop = int(ins['stop'])\n            ins_target = int(ins['target'])\n            \n            part_start = ins['part_start']\n            \n            real_start = int(ins_start + part_start)\n            real_stop = int(ins_stop + part_start)\n            \n            s_start = ins['start_score']\n            s_stop = ins['stop_score']\n            \n            cls_start = ins['start_CLS']\n            cls_stop = ins['stop_CLS']\n            \n            if real_stop > real_start:   \n                if s_start - cls_start + s_stop - cls_stop > best_score:\n                    best_score = s_start - cls_start + s_stop - cls_stop\n                    best_start = real_start\n                    best_stop = real_stop\n                    best_target = ins_target\n\n        doc_lan = {}\n        doc_lan['example_id'] = doc_id\n        doc_lan['start'] = best_start\n        doc_lan['stop'] = best_stop\n        doc_lan['target'] = best_target\n        doc_lan['score'] = best_score\n        \n        if debug:\n            if idx == 101:\n                print(doc_lan)\n        \n        list_doc_lan.append(doc_lan)\n    \n    list_doc_lan_df = pd.DataFrame(list_doc_lan)\n    return list_doc_lan_df","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:27:40.633263Z","iopub.execute_input":"2021-10-25T08:27:40.633679Z","iopub.status.idle":"2021-10-25T08:27:40.647125Z","shell.execute_reply.started":"2021-10-25T08:27:40.633647Z","shell.execute_reply":"2021-10-25T08:27:40.645572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Get instances (html tags cleaned) for long answer predict**","metadata":{}},{"cell_type":"code","source":"cleanr = re.compile('<.*?>')\ndef clean_html(raw_html):\n    cleantext = re.sub(cleanr, '<tag>', raw_html)\n    return cleantext\n\ndef parseDataClean(filename=f_test, is_val=True, drop_noanswer_rate = 0.95, drop_null_instances_rate = 0.98, debug=False):\n    INSTANCE_WORDS_LEN = 500 \n    STRIDE = 128 \n    num, count_drop, count_yes_no, count_long, count_short, count_no_answer = 0, 0, 0, 0, 0, 0\n    list_instances = []\n\n    with open(filename) as f:\n        progress = tqdm(f)  \n        for sam_count, line in enumerate(progress):\n            data = json.loads(line)\n            example_id = str(data['example_id'])\n\n\n            doc_text_raw = data['document_text']\n            doc_text_tag = clean_html(doc_text_raw) # change all html tags to the form <tag>\n            doc_tag_split = doc_text_tag.split()\n\n            lan_start, lan_stop, san_start, san_stop = -1, -1, -1, -1\n\n            clean_doc = list(filter(('<tag>').__ne__, doc_tag_split))\n\n            question = data['question_text'] # question\n\n            len_ques = len(question.split())\n            part_len = INSTANCE_WORDS_LEN - len_ques \n\n            num_ins = (len(clean_doc) - part_len)//STRIDE + 1\n\n            for part_id in range(num_ins + 1):\n                part_start = part_id*STRIDE\n                part_stop = min(len(clean_doc), part_id*STRIDE + part_len)\n\n                part_split = clean_doc[part_start:part_stop]\n\n                part = ' '.join(part_split)\n                \n                instance = {'example_id': example_id, 'part_start': part_start, 'part_stop': part_stop,\n                            'question': question,'context': part, \n                            'start': 0, 'stop': 0, 'target': 'NO_ANSWER'}\n                list_instances.append(instance)\n    return list_instances","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:28:00.698021Z","iopub.execute_input":"2021-10-25T08:28:00.698565Z","iopub.status.idle":"2021-10-25T08:28:00.713851Z","shell.execute_reply.started":"2021-10-25T08:28:00.698517Z","shell.execute_reply":"2021-10-25T08:28:00.712338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getMapping(set_id, filename=f_test):\n    list_cand_maps = []\n    with open(filename) as f:\n        progress = tqdm(f)  \n        for sam_count, line in enumerate(progress):\n                \n            data = json.loads(line)\n            example_id = str(data['example_id'])\n\n            if example_id in set_id:\n                doc_text_raw = data['document_text']\n                doc_text_raw = clean_html(doc_text_raw) # change all html tags to the form <tag>\n                doc_text_split = doc_text_raw.split()\n\n                clean_doc = list(filter(('<tag>').__ne__, doc_text_split))\n\n                list_candidates = data['long_answer_candidates']\n                list_new_candidates = []\n                for cand in list_candidates:\n                    cand_start = cand['start_token']\n                    cand_stop = cand['end_token']\n                    \n                    num_tag_bef_start = doc_text_split[0:cand_start].count('<tag>')\n                    num_tag_bef_stop = doc_text_split[0:cand_stop].count('<tag>')\n                \n                    new_start = cand_start - num_tag_bef_start\n                    new_stop = cand_stop - num_tag_bef_stop\n                    \n                    new_cand = {}\n                    new_cand['end_token'] = new_stop\n                    new_cand['start_token'] = new_start\n                    \n                    list_new_candidates.append(new_cand)\n                sample = {}\n                sample['example_id'] = str(example_id)\n                sample['new_candidates'] = list_new_candidates\n                sample['old_candidates'] = list_candidates\n                \n                list_cand_maps.append(sample)\n    return list_cand_maps","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:28:03.301373Z","iopub.execute_input":"2021-10-25T08:28:03.301841Z","iopub.status.idle":"2021-10-25T08:28:03.312657Z","shell.execute_reply.started":"2021-10-25T08:28:03.301808Z","shell.execute_reply":"2021-10-25T08:28:03.311166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(model_name, debug=False):\n    encoder = TFBertModel.from_pretrained(model_name)\n    \n    tokenizer = AutoTokenizer.from_pretrained(model_name)\n    \n    tags = ['``', '\\'\\'', '--']\n\n    special_tokens_dict = {'additional_special_tokens': tags}\n\n    num_added_toks = tokenizer.add_special_tokens(special_tokens_dict)\n    \n    encoder.resize_token_embeddings(len(tokenizer))\n\n    NUM_TARGET = 5\n    class MyQAModel(tf.keras.Model):\n        def __init__(self, *inputs, **kwargs):\n            super().__init__(*inputs, **kwargs)            \n            self.bert = encoder\n\n            self.start_logits = tf.keras.layers.Dense(1)\n            self.stop_logits = tf.keras.layers.Dense(1)\n            \n            self.target = tf.keras.layers.Dense(NUM_TARGET)\n\n        def call(self, inputs, **kwargs):\n            bert_res=self.bert(inputs[0], \n                               token_type_ids=inputs[1], \n                               attention_mask=inputs[2]\n                               )\n            dropout_res1 = bert_res[0]\n\n            start_logits = tf.squeeze(self.start_logits(dropout_res1), -1)\n            dropout_res2 = bert_res[0]\n\n            stop_logits = tf.squeeze(self.stop_logits(dropout_res2), -1)\n            dropout_res3 = bert_res[1]\n            \n            targets = self.target(dropout_res3)\n            \n            paddings = tf.constant([[0, 0,], [0, 512-NUM_TARGET]])\n            targets = tf.pad(targets, paddings)\n            \n            res = tf.stack([start_logits, stop_logits, targets], axis=1)\n            return res\n        \n    model = MyQAModel()\n    return model ","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:28:27.173267Z","iopub.execute_input":"2021-10-25T08:28:27.173694Z","iopub.status.idle":"2021-10-25T08:28:27.186298Z","shell.execute_reply.started":"2021-10-25T08:28:27.173662Z","shell.execute_reply":"2021-10-25T08:28:27.184606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getRawInstanceResults(list_test, verbose = True, debug = False):  \n    if verbose:\n        print('Getting raw result for all the instances generated from test file')\n        \n    model_name = '../input/tensorflow-question-answer-fine-data'\n    tokenizer = AutoTokenizer.from_pretrained(model_name)\n    \n    tags = ['``', '\\'\\'', '--']\n\n    special_tokens_dict = {'additional_special_tokens': tags}\n\n    num_added_toks = tokenizer.add_special_tokens(special_tokens_dict)\n    print(num_added_toks)\n    print(len(tokenizer))\n    \n    x_test1, x_test2, x_test3, y_test = preprocess_data(list_test, tokenizer)\n    if verbose:\n        print(\"Finish tokenizing \", len(list_test), \" data for the first model\")\n        print(x_test1.shape)\n    \n    if verbose:\n        print(\"Preparing model\")\n        \n    strategy = get_strategy()\n    with strategy.scope():\n        testModel = build_model(model_name)\n        x = np.ones([1, 512], dtype=int)\n        testModel.predict([x, x, x])\n        testModel.load_weights('../input/model1/weights-02.h5')\n        optAdam = tf.keras.optimizers.Adam(learning_rate=0.00005)\n        lossSCE = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n        metricSCA = tf.keras.metrics.SparseCategoricalAccuracy()\n        testModel.compile(optimizer=optAdam, loss=lossSCE, metrics=[metricSCA])\n    \n    if verbose:\n        print(\"Finish loading pretrained weights for the model\")\n        \n    test_res = testModel.predict([x_test1, x_test2, x_test3], verbose=1)\n    \n    if verbose:\n        print(\"Finish calculating raw result, get an array of size: \", test_res.shape)\n    return test_res\n","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:28:59.5209Z","iopub.execute_input":"2021-10-25T08:28:59.521257Z","iopub.status.idle":"2021-10-25T08:28:59.53331Z","shell.execute_reply.started":"2021-10-25T08:28:59.521226Z","shell.execute_reply":"2021-10-25T08:28:59.531591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getSubmissionLan(doc_res_df, doc_cand_df, threshold=0.0001, debug=False):\n    doc_res_df.example_id = doc_res_df.example_id.astype(str)\n    doc_cand_df.example_id = doc_cand_df.example_id.astype(str)\n    if debug:\n        print(doc_res_df.dtypes)\n        print(doc_cand_df.dtypes)\n\n    combine_df = pd.merge(doc_res_df, doc_cand_df, on='example_id')\n    lines = []\n    for id, doc in combine_df.iterrows():\n\n        example_id = doc['example_id']\n        long_id = str(example_id) + '_long'\n        short_id = str(example_id) + '_short'\n\n        line_long = {}\n        line_long['example_id'] = long_id\n\n        an_start = int(doc['start'])\n        an_stop = int(doc['stop'])\n        an_target = doc['target']\n        an_score = doc['score']\n        # print(an_start, an_stop, an_target, an_score)\n        lan_start, lan_stop = -1, -1\n\n        # find long answer \n        if an_start > 0 and an_stop > 0:\n            candidates = doc['new_candidates']\n            an_range = [*range(an_start, an_stop + 1, 1)]\n\n            best_inter = 0.5\n            shortest = 10000000000000\n            best_id = 0\n            for cidx, cand in enumerate(candidates):\n                c_start = int(cand['start_token'])\n                c_stop = int(cand['end_token'])\n\n                c_range = [*range(c_start, c_stop + 1, 1)]\n                inter = len(list(set(an_range)&set(c_range)))\n            \n                if float(inter) > best_inter:\n                    best_id = cidx\n                    best_inter = inter\n                    shortest = len(c_range)\n                elif inter == best_inter:\n                    if shortest > len(c_range):\n                        best_id = cidx\n                        shortest = len(c_range)\n\n            real_candidates = doc['old_candidates']\n            lan_start = real_candidates[best_id]['start_token']\n            lan_stop = real_candidates[best_id]['end_token']\n\n            if debug:\n                if id == 101:\n                    print(lan_start, lan_stop)\n\n        if lan_start > 0 and lan_stop > 0 and an_target != 0:\n            long_string = str(lan_start) + ':' + str(lan_stop)\n        else:\n            long_string = ''\n\n\n        line_long['PredictionString'] = long_string\n        lines.append(line_long)\n\n    lines_df = pd.DataFrame(lines)\n    sorted_df = lines_df.sort_values('example_id')\n    return sorted_df","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:29:05.583248Z","iopub.execute_input":"2021-10-25T08:29:05.583644Z","iopub.status.idle":"2021-10-25T08:29:05.598781Z","shell.execute_reply.started":"2021-10-25T08:29:05.583613Z","shell.execute_reply":"2021-10-25T08:29:05.597514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Process short answer**","metadata":{}},{"cell_type":"code","source":"def getSanCandidate(sub, filename=f_test, debug=False):\n    INSTANCE_WORDS_LEN = 500 \n    STRIDE = 256 \n\n    list_doc_lan_res = []\n    for rowid, row in sub.iterrows():\n        example_id = str(row['example_id']).replace('_long',\"\")\n        lan_start, lan_stop = -1, -1\n\n        if str(row['PredictionString']) != '':\n            tokens = str(row['PredictionString']).split(':')\n            lan_start = int(tokens[0])\n            lan_stop = int(tokens[1]) \n            \n        sam = {'example_id': example_id, 'lan_start': lan_start, 'lan_stop': lan_stop}\n        list_doc_lan_res.append(sam)\n        \n    list_doc_lan_res_df = pd.DataFrame(list_doc_lan_res)\n\n    set_id = set(list_doc_lan_res_df['example_id'].values.tolist())\n\n    list_san_ins = []\n\n    with open(filename) as f:\n        progress = tqdm(f)  \n        for sam_count, line in enumerate(progress):\n            data = json.loads(line)\n            example_id = str(data['example_id'])\n            if example_id in set_id:\n                # get lan result \n                ans = list_doc_lan_res_df.loc[list_doc_lan_res_df['example_id']==example_id]\n                lan_start, lan_stop = -1, -1\n                for rowid, row in ans.iterrows():\n                    lan_start = row['lan_start']\n                    lan_stop = row['lan_stop']\n                if debug:\n                    print(example_id, lan_start, lan_stop)\n                doc_text = data['document_text']\n                doc_text_split = doc_text.split()\n                question = data['question_text']\n                \n                if lan_start > -1 and lan_stop > -1:\n                    if lan_stop - lan_start <= INSTANCE_WORDS_LEN:\n                        offset = (INSTANCE_WORDS_LEN - (lan_stop - lan_start))//2 \n                        part_start = max(0,lan_start - offset)\n                        part_stop = min(lan_stop + offset, len(doc_text_split))\n                        part_split = doc_text_split[part_start:part_stop]\n                        context = ' '.join(part_split)\n                        ins = {'example_id': example_id, 'part_start': part_start, 'part_stop': part_stop, \n                               'question': question, 'context': context, 'start': 0, 'stop': 0, 'target': 'NO_ANSWER'}\n                        list_san_ins.append(ins) \n                        if debug:\n                            print(ins)\n                    else: \n                    # in case found long answer is longer than context length limit then split the long answer into small parts\n                    # and slide with stride 256\n                        part_length = INSTANCE_WORDS_LEN\n                        num_parts = (lan_stop - lan_start - INSTANCE_WORDS_LEN)//STRIDE + 1\n                        for part_id in range(num_parts + 1):\n                            part_start = lan_start + part_id*STRIDE\n                            part_stop = min(len(doc_text_split), lan_start + part_id*STRIDE + part_length)\n                            part_split = doc_text_split[part_start:part_stop]\n                    \n                            context = ' '.join(part_split)\n                            ins = {'example_id': example_id, 'part_start': part_start, 'part_stop': part_stop, \n                               'question': question, 'context': context, 'start': 0, 'stop': 0, 'target': 'NO_ANSWER'}\n                            list_san_ins.append(ins)\n                            if debug:\n                                print(ins)\n    return list_san_ins            \n","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:29:15.345051Z","iopub.execute_input":"2021-10-25T08:29:15.345568Z","iopub.status.idle":"2021-10-25T08:29:15.374661Z","shell.execute_reply.started":"2021-10-25T08:29:15.345522Z","shell.execute_reply":"2021-10-25T08:29:15.373252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model_san(tokenizer_san, model_name_san, debug=False):\n    config = BertConfig()\n    if debug:\n        print(config)\n    encoder = TFBertModel.from_pretrained(model_name_san)\n    encoder.resize_token_embeddings(len(tokenizer_san))\n\n    NUM_TARGET = 5\n    class MyQAModel(tf.keras.Model):\n        def __init__(self, *inputs, **kwargs):\n            super().__init__(*inputs, **kwargs)            \n            self.bert = encoder\n            self.start_logits = tf.keras.layers.Dense(1)\n            self.stop_logits = tf.keras.layers.Dense(1)\n            \n            self.target = tf.keras.layers.Dense(NUM_TARGET)\n\n        def call(self, inputs, **kwargs):\n            bert_res=self.bert(inputs[0], \n                               token_type_ids=inputs[1], \n                               attention_mask=inputs[2]\n                               )\n            \n            dropout_res1 = bert_res[0]\n\n            start_logits = tf.squeeze(self.start_logits(dropout_res1), -1)\n\n            dropout_res2 = bert_res[0]\n\n            stop_logits = tf.squeeze(self.stop_logits(dropout_res2), -1)\n\n            dropout_res3 = bert_res[1]\n            \n            targets = self.target(dropout_res3)\n            \n            paddings = tf.constant([[0, 0,], [0, 512-NUM_TARGET]])\n            targets = tf.pad(targets, paddings)\n            \n            res = tf.stack([start_logits, stop_logits, targets], axis=1)\n            return res\n        \n    model = MyQAModel()\n    return model ","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:29:19.535577Z","iopub.execute_input":"2021-10-25T08:29:19.53594Z","iopub.status.idle":"2021-10-25T08:29:19.548249Z","shell.execute_reply.started":"2021-10-25T08:29:19.535908Z","shell.execute_reply":"2021-10-25T08:29:19.546732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getSanRawRes(list_san_ins, verbose=1):\n    print(\"Getting raw result for short answer instance generated from found long answers\")\n    \n    model_name_san = '../input/tensorflow-question-answer-fine-data'\n\n    tokenizer_san = AutoTokenizer.from_pretrained(model_name_san)\n\n    tags_san = ['<Dd>', '<Dl>', '<Dt>', '<H1>', '<H2>', '<H3>', '<Li>', '<Ol>', '<P>', '<Table>', '<Td>', '<Th>', '<Tr>', '<Ul>',\n            '</Dd>', '</Dl>', '</Dt>', '</H1>', '</H2>', '</H3>', '</Li>', '</Ol>', '</P>', '</Table>', '</Td>', '</Th>', '</Tr>', '</Ul>',\n            '<Th_colspan=', '</Th_colspan=', '``', '\\'\\'', '--']\n\n    special_tokens_dict_san = {'additional_special_tokens': tags_san}\n\n    num_added_toks_san = tokenizer_san.add_special_tokens(special_tokens_dict_san)\n    print(\"Short answer vocab size: \", len(tokenizer_san))\n    \n    x_san1, x_san2, x_san3, y_san = preprocess_data(list_san_ins, tokenizer_san)\n    print(\"Finish tokenizing \", len(list_san_ins), \" instances for short answer candidates\")\n    print(x_san1.shape)\n    \n    strategy_san = get_strategy()\n    with strategy_san.scope():\n        sanModel = create_model_san(tokenizer_san, model_name_san)\n        x = np.ones([1, 512], dtype=int)\n        sanModel.predict([x, x, x])\n        sanModel.load_weights('../input/model1/weights-14.h5')\n        optAdam = tf.keras.optimizers.Adam(learning_rate=0.00005)\n        lossSCE = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n        metricSCA = tf.keras.metrics.SparseCategoricalAccuracy()\n        sanModel.compile(optimizer=optAdam, loss=lossSCE, metrics=[metricSCA])\n    \n    if verbose:\n        print(\"Finish loading pretrained weights for the model for short answer\")\n        \n    test_res = sanModel.predict([x_san1, x_san2, x_san3], verbose=1)\n    \n    if verbose:\n        print(\"Finish calculating raw result, get an array of size: \", test_res.shape)\n    return test_res","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:29:23.754372Z","iopub.execute_input":"2021-10-25T08:29:23.754832Z","iopub.status.idle":"2021-10-25T08:29:23.76724Z","shell.execute_reply.started":"2021-10-25T08:29:23.754801Z","shell.execute_reply":"2021-10-25T08:29:23.765393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getSanSubmission(doc_res_df, threshold=0.0001, debug=False):\n    doc_res_df.example_id = doc_res_df.example_id.astype(str)\n    lines = []\n    for id, doc in doc_res_df.iterrows():\n        example_id = doc['example_id']\n        short_id = str(example_id) + '_short'\n\n        line_short = {}\n        line_short['example_id'] = short_id\n\n        an_start = int(doc['start'])\n        an_stop = int(doc['stop'])\n        an_target = int(doc['target'])\n        an_score = float(doc['score'])\n\n        if an_start > 0 and an_stop > 0 and an_target != 4 and an_stop - an_start < 30:\n            short_string = str(an_start) + ':' + str(an_stop)\n        else:\n            short_string = ''\n\n        if an_target == 1 or an_target == 2:\n            short_string = AnswerTypeRev[an_target]\n\n\n        line_short['PredictionString'] = short_string\n        lines.append(line_short)\n\n    lines_df = pd.DataFrame(lines)\n    sorted_df = lines_df.sort_values('example_id')\n    return sorted_df","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:45:34.712249Z","iopub.execute_input":"2021-10-25T08:45:34.712695Z","iopub.status.idle":"2021-10-25T08:45:34.724137Z","shell.execute_reply.started":"2021-10-25T08:45:34.712663Z","shell.execute_reply":"2021-10-25T08:45:34.72259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def refineLan(sub, list_mapping_df, debug=False):\n    newsub = []\n    for rowid, row in sub.iterrows():\n        if 'long' in str(row['example_id']):\n            example_id = str(row['example_id']).replace('_long',\"\")\n            \n            longid = str(row['example_id'])\n            longStr = str(row['PredictionString'])\n            \n            lan_start, lan_stop = -1, -1\n\n            if str(row['PredictionString']) != '':\n                tokens = str(row['PredictionString']).split(':')\n                lan_start = int(tokens[0])\n                lan_stop = int(tokens[1])\n            \n            # find corresponding short answer \n            san_start, san_stop = -1, -1\n            \n            sanid = str(example_id) + '_short'\n            san = sub.loc[sub['example_id'] == sanid].iloc[0]\n            sanStr = str(san['PredictionString'])\n            \n            \n            if sanStr != '' and sanStr != 'YES' and sanStr != 'NO':\n                tokensans = sanStr.split(':')\n                san_start = int(tokensans[0])\n                san_stop = int(tokensans[1])\n                \n                if san_start < lan_start or san_stop > lan_stop: # san is not in lan \n                    # find candidate list of this example \n                    cands = list_mapping_df.loc[list_mapping_df['example_id'] == example_id].iloc[0]['old_candidates']\n                    \n                    an_range = [*range(san_start, san_stop + 1, 1)]\n                    best_inter = 0.5\n                    shortest = 10000000000000\n                    best_id = 0\n                    for cidx, cand in enumerate(cands):\n                        c_start = int(cand['start_token'])\n                        c_stop = int(cand['end_token'])\n\n                        c_range = [*range(c_start, c_stop + 1, 1)]\n                        inter = len(list(set(an_range)&set(c_range)))\n\n                        if float(inter) > best_inter:\n                            best_id = cidx\n                            best_inter = inter\n                            shortest = len(c_range)\n                        elif inter == best_inter:\n                            if shortest > len(c_range):\n                                best_id = cidx\n                                shortest = len(c_range)\n\n                    lan_start = cands[best_id]['start_token']\n                    lan_stop = cands[best_id]['end_token']\n                    longStr = str(lan_start) + \":\" + str(lan_stop)\n                    \n            longline = {'example_id': longid, 'PredictionString': longStr}\n            shortline = {'example_id': sanid, 'PredictionString': sanStr}\n            newsub.append(longline)\n            newsub.append(shortline)\n    newsubdf = pd.DataFrame(newsub)\n    newsubsorted = newsubdf.sort_values('example_id')\n    return newsubsorted","metadata":{"execution":{"iopub.status.busy":"2021-10-25T09:44:19.471103Z","iopub.execute_input":"2021-10-25T09:44:19.471674Z","iopub.status.idle":"2021-10-25T09:44:19.488649Z","shell.execute_reply.started":"2021-10-25T09:44:19.471634Z","shell.execute_reply":"2021-10-25T09:44:19.487055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **From here on is for test**","metadata":{}},{"cell_type":"code","source":"list_id_df = get_id_df()","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:29:30.398123Z","iopub.execute_input":"2021-10-25T08:29:30.398525Z","iopub.status.idle":"2021-10-25T08:29:30.908573Z","shell.execute_reply.started":"2021-10-25T08:29:30.398459Z","shell.execute_reply":"2021-10-25T08:29:30.906602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set_id = set(list_id_df['example_id'].values.tolist())\nlan_map = getMapping(set_id)","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:29:32.87265Z","iopub.execute_input":"2021-10-25T08:29:32.873046Z","iopub.status.idle":"2021-10-25T08:29:47.505914Z","shell.execute_reply.started":"2021-10-25T08:29:32.87299Z","shell.execute_reply":"2021-10-25T08:29:47.504738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_mappings_df = pd.DataFrame(lan_map)\nlist_mappings_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:29:54.414636Z","iopub.execute_input":"2021-10-25T08:29:54.415035Z","iopub.status.idle":"2021-10-25T08:29:54.619379Z","shell.execute_reply.started":"2021-10-25T08:29:54.415003Z","shell.execute_reply":"2021-10-25T08:29:54.618186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_all_ins= parseDataClean(f_test)\nall_ins_res = getRawInstanceResults(list_all_ins)","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:29:58.777356Z","iopub.execute_input":"2021-10-25T08:29:58.777886Z","iopub.status.idle":"2021-10-25T08:37:39.221818Z","shell.execute_reply.started":"2021-10-25T08:29:58.77785Z","shell.execute_reply":"2021-10-25T08:37:39.220466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_fine_res_all_ins = mergeInstanceResult(all_ins_res, list_all_ins)\nfine_res_all_ins_df = pd.DataFrame(list_fine_res_all_ins)","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:43:21.097794Z","iopub.execute_input":"2021-10-25T08:43:21.098187Z","iopub.status.idle":"2021-10-25T08:43:21.477528Z","shell.execute_reply.started":"2021-10-25T08:43:21.098155Z","shell.execute_reply":"2021-10-25T08:43:21.476449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fine_res_all_ins_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:43:23.184374Z","iopub.execute_input":"2021-10-25T08:43:23.18479Z","iopub.status.idle":"2021-10-25T08:43:23.207207Z","shell.execute_reply.started":"2021-10-25T08:43:23.184757Z","shell.execute_reply":"2021-10-25T08:43:23.205908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"docAnsDf = mergeDocumentRes(fine_res_all_ins_df, list_id_df)","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:43:27.601473Z","iopub.execute_input":"2021-10-25T08:43:27.601915Z","iopub.status.idle":"2021-10-25T08:43:31.222019Z","shell.execute_reply.started":"2021-10-25T08:43:27.601873Z","shell.execute_reply":"2021-10-25T08:43:31.220929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"docAnsDf.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:43:33.027299Z","iopub.execute_input":"2021-10-25T08:43:33.027764Z","iopub.status.idle":"2021-10-25T08:43:33.041383Z","shell.execute_reply.started":"2021-10-25T08:43:33.027718Z","shell.execute_reply":"2021-10-25T08:43:33.03987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subLan = getSubmissionLan(docAnsDf, list_mappings_df)","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:43:35.93506Z","iopub.execute_input":"2021-10-25T08:43:35.93541Z","iopub.status.idle":"2021-10-25T08:43:36.181414Z","shell.execute_reply.started":"2021-10-25T08:43:35.935379Z","shell.execute_reply":"2021-10-25T08:43:36.180305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subLan.head(20)","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:43:41.795393Z","iopub.execute_input":"2021-10-25T08:43:41.79584Z","iopub.status.idle":"2021-10-25T08:43:41.809755Z","shell.execute_reply.started":"2021-10-25T08:43:41.79581Z","shell.execute_reply":"2021-10-25T08:43:41.808507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_san_ins = getSanCandidate(subLan, debug=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:43:48.666044Z","iopub.execute_input":"2021-10-25T08:43:48.666451Z","iopub.status.idle":"2021-10-25T08:43:49.40264Z","shell.execute_reply.started":"2021-10-25T08:43:48.666395Z","shell.execute_reply":"2021-10-25T08:43:49.400676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sanRawRes = getSanRawRes(list_san_ins)","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:43:52.806104Z","iopub.execute_input":"2021-10-25T08:43:52.806532Z","iopub.status.idle":"2021-10-25T08:44:22.409776Z","shell.execute_reply.started":"2021-10-25T08:43:52.80649Z","shell.execute_reply":"2021-10-25T08:44:22.408553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_fine_res_san_ins = mergeInstanceResult(sanRawRes, list_san_ins)\nfine_res_san_ins_df = pd.DataFrame(list_fine_res_san_ins)","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:45:40.270837Z","iopub.execute_input":"2021-10-25T08:45:40.271257Z","iopub.status.idle":"2021-10-25T08:45:40.290356Z","shell.execute_reply.started":"2021-10-25T08:45:40.27121Z","shell.execute_reply":"2021-10-25T08:45:40.289061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fine_res_san_ins_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:45:42.292414Z","iopub.execute_input":"2021-10-25T08:45:42.292823Z","iopub.status.idle":"2021-10-25T08:45:42.314678Z","shell.execute_reply.started":"2021-10-25T08:45:42.29279Z","shell.execute_reply":"2021-10-25T08:45:42.313029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"docSanAnsDf = mergeDocumentRes(fine_res_san_ins_df, list_id_df)","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:45:51.159324Z","iopub.execute_input":"2021-10-25T08:45:51.159759Z","iopub.status.idle":"2021-10-25T08:45:53.762509Z","shell.execute_reply.started":"2021-10-25T08:45:51.159728Z","shell.execute_reply":"2021-10-25T08:45:53.761214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"docSanAnsDf.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:47:02.255239Z","iopub.execute_input":"2021-10-25T08:47:02.255668Z","iopub.status.idle":"2021-10-25T08:47:02.270643Z","shell.execute_reply.started":"2021-10-25T08:47:02.255636Z","shell.execute_reply":"2021-10-25T08:47:02.269272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subSan = getSanSubmission(docSanAnsDf, threshold=0.2)","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:47:05.248923Z","iopub.execute_input":"2021-10-25T08:47:05.249718Z","iopub.status.idle":"2021-10-25T08:47:05.34295Z","shell.execute_reply.started":"2021-10-25T08:47:05.249604Z","shell.execute_reply":"2021-10-25T08:47:05.341783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subSan.head(20)","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:47:25.486807Z","iopub.execute_input":"2021-10-25T08:47:25.487161Z","iopub.status.idle":"2021-10-25T08:47:25.501838Z","shell.execute_reply.started":"2021-10-25T08:47:25.487131Z","shell.execute_reply":"2021-10-25T08:47:25.500465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.concat([subLan, subSan])\nsub_sorted = sub.sort_values('example_id')","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:47:31.001994Z","iopub.execute_input":"2021-10-25T08:47:31.002431Z","iopub.status.idle":"2021-10-25T08:47:31.012623Z","shell.execute_reply.started":"2021-10-25T08:47:31.002352Z","shell.execute_reply":"2021-10-25T08:47:31.011094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_sorted.head(20)","metadata":{"execution":{"iopub.status.busy":"2021-10-25T08:47:34.689927Z","iopub.execute_input":"2021-10-25T08:47:34.690275Z","iopub.status.idle":"2021-10-25T08:47:34.706427Z","shell.execute_reply.started":"2021-10-25T08:47:34.690244Z","shell.execute_reply":"2021-10-25T08:47:34.7051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"refineSub = refineLan(sub, list_mappings_df, debug=True)","metadata":{"execution":{"iopub.status.busy":"2021-10-25T09:44:45.573427Z","iopub.execute_input":"2021-10-25T09:44:45.573798Z","iopub.status.idle":"2021-10-25T09:44:46.003531Z","shell.execute_reply.started":"2021-10-25T09:44:45.573767Z","shell.execute_reply":"2021-10-25T09:44:46.002568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"refineSub.to_csv('./submission.csv', \n                  index=False, \n                  columns=['example_id', 'PredictionString'])","metadata":{"execution":{"iopub.status.busy":"2021-10-25T09:44:48.106009Z","iopub.execute_input":"2021-10-25T09:44:48.106408Z","iopub.status.idle":"2021-10-25T09:44:48.120179Z","shell.execute_reply.started":"2021-10-25T09:44:48.106379Z","shell.execute_reply":"2021-10-25T09:44:48.118755Z"},"trusted":true},"execution_count":null,"outputs":[]}]}