{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np # linear algebra\nimport pandas as pd \nimport gc\nimport os\nimport json\nimport collections\nfrom time import time\nfrom tqdm import tqdm as tqdm_base\nimport os\nimport sys\ndef tqdm(*args, **kwargs):\n    if hasattr(tqdm_base, '_instances'):\n        for instance in list(tqdm_base._instances):\n            tqdm_base._decr_instances(instance)\n    return tqdm_base(*args, **kwargs)\nimport gc\nprint(tf.__version__)\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Input data files are available in the \"../input/\" directory.\nIS_KAGGLE = True\nINPUT_DIR = \"/kaggle/input/\"\n\n# The original Bert Joint Baseline data.\nclass_model_dir = os.path.join(INPUT_DIR, \"tf-3labels\")\n\n# This nq dir contains all files for publicly use.\nTFNQ_DIR = os.path.join(INPUT_DIR, \"tensorflow2-question-answering\")\n\nTrans_ = os.path.join(INPUT_DIR, 'tf-asnq/src/transformers')\n\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n#for dirname, _, filenames in os.walk(INPUT_DIR):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\nsys.path.append(Trans_)\n# Any results you write to the current directory are saved as output.\n\nfrom transformers import TFRobertaForSequenceClassification, RobertaTokenizer,RobertaConfig\nfrom transformers.data.processors.utils import DataProcessor, InputExample, InputFeatures","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"test_path = os.path.join(TFNQ_DIR,'simplified-nq-test.jsonl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(test_path,'rt') as reader:\n    df = [] \n    for line in tqdm(reader):\n        exemple = json.loads(line)\n        id = int(exemple[\"example_id\"])\n        candidates = exemple[\"long_answer_candidates\"]\n        question= exemple[\"question_text\"]\n        _text = exemple['document_text'].split()\n        for candidate in candidates:\n            local_df = []\n            local_df.append(id)\n            local_df.append(question)\n            start = candidate['start_token']\n            stop = candidate['end_token']\n            text__ = ' '.join(_text[start:min(start+512,stop)])\n            local_df.append(text__)\n            local_df.append(f'{start}:{stop}')\n            df.append(local_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame(df,columns=['example_id','question','text','start_stop'])\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 512\nmax_length = MAX_SEQUENCE_LENGTH\ntask='asnq'\nlabel_list=[\"1\",\"2\",\"3\"]\noutput_mode=\"classification\"\npad_on_left=False\npad_token=0\npad_token_segment_id=0\nmask_padding_with_zero=True\n\nlabel_map = {label: i for i, label in enumerate(label_list)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_fn():\n    return TFRobertaForSequenceClassification.from_pretrained(class_model_dir,config =RobertaConfig.from_pretrained(class_model_dir))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = model_fn()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in df:\n    print(i)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = RobertaTokenizer.from_pretrained(class_model_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def gen_data():\n    for index, row in df.iterrows():\n        inputs = tokenizer.encode_plus(row.question,row.text,add_special_tokens=True,max_length=max_length,)\n        input_ids = inputs[\"input_ids\"]\n        attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)\n        padding_length = max_length - len(input_ids)\n        input_ids = ([pad_token] * padding_length) + input_ids\n        attention_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask\n        yield (input_ids,attention_mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = tf.data.Dataset.from_generator( \n     gen_data, \n     (tf.int64, tf.int64), \n     (tf.TensorShape([None]), tf.TensorShape([None])))\n\ndataset = dataset.map(lambda x,y : {'input_ids': x,'attention_mask': y,},num_parallel_calls=32)\ndataset = dataset.batch(128,drop_remainder=False)\ndataset = dataset.prefetch(100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start = time()\npreds=np.array([])\nfor line in tqdm(dataset):\n    preds= np.concatenate((preds,tf.argmax(model(line, training=False)[0],1).numpy()),0)\nend=time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(preds)==len(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['preds'] = preds.tolist()\npreds=None\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_true = df[df.preds==2]\ndf=None\ngc.collect()\ndf_true\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_true_df=[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in df_true.example_id.unique():\n    local_df = df_true[df_true.example_id==i]\n    def gen_data():\n        for index, row in local_df .iterrows():\n            inputs = tokenizer.encode_plus(row.question,row.text,add_special_tokens=True,max_length=max_length,)\n            input_ids = inputs[\"input_ids\"]\n            attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)\n            padding_length = max_length - len(input_ids)\n            input_ids = ([pad_token] * padding_length) + input_ids\n            attention_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask\n            yield (input_ids,attention_mask)\n    dataset = tf.data.Dataset.from_generator( gen_data, (tf.int64, tf.int64), (tf.TensorShape([None]), tf.TensorShape([None])))\n\n    dataset = dataset.map(lambda x,y : {'input_ids': x,'attention_mask': y,},num_parallel_calls=32)\n    dataset = dataset.batch(128,drop_remainder=False)\n    dataset = dataset.prefetch(100)\n    preds=np.array([])\n    for line in tqdm(dataset):\n        preds= np.concatenate((preds,model(line, training=False)[0].numpy()[:,2]),0)\n    true_one = local_df.iloc[np.argmax(preds,0)]\n    new_true_df.append([true_one.example_id,true_one.start_stop])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=None\ndf_true = None\ntokenizer=None\ngc.collect()\nnew_true_df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_true_df = pd.DataFrame(new_true_df,columns =['example_id','preds'])\nnew_true_df['example_id'] = new_true_df['example_id'].apply(lambda q: str(q)+\"_long\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/tensorflow2-question-answering/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.PredictionString = sample_submission.join(new_true_df.set_index('example_id'),on='example_id').preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.PredictionString = sample_submission.PredictionString.apply(lambda x: '' if type(x)==float  else x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}